Repository: brimsec/zq
Pull request #1922 (2021-01-04 22:38:02)

Move ztests/formats/path/* into zio/zeekio/ztests/


Patch:
diff --git a/cmd/ast/ast.go b/cmd/ast/ast.go
index 77fbf3106..ab44afae2 100644
--- a/cmd/ast/ast.go
+++ b/cmd/ast/ast.go
@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"strings"
 
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/zql"
 	"github.com/mccanne/charm"
 	"github.com/peterh/liner"
@@ -170,7 +171,7 @@ func parsePEGjs(z string) (string, error) {
 }
 
 func parseProc(z string) (string, error) {
-	proc, err := zql.ParseProc(z)
+	proc, err := compiler.ParseProc(z)
 	if err != nil {
 		return "", err
 	}
diff --git a/cmd/zapi/cmd/get/get.go b/cmd/zapi/cmd/get/get.go
index 7b7c4a46c..7fdbbf1e2 100644
--- a/cmd/zapi/cmd/get/get.go
+++ b/cmd/zapi/cmd/get/get.go
@@ -15,10 +15,10 @@ import (
 	"github.com/brimsec/zq/api/client"
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/nano"
 	"github.com/brimsec/zq/zbuf"
-	"github.com/brimsec/zq/zql"
 	"github.com/mccanne/charm"
 )
 
@@ -170,7 +170,7 @@ func (c *Command) Run(args []string) error {
 
 // parseExpr creates an api.SearchRequest to be used with the client.
 func parseExpr(spaceID api.SpaceID, expr string) (*api.SearchRequest, error) {
-	search, err := zql.ParseProc(expr)
+	search, err := compiler.ParseProc(expr)
 	if err != nil {
 		return nil, err
 	}
@@ -188,7 +188,7 @@ func parseExpr(spaceID api.SpaceID, expr string) (*api.SearchRequest, error) {
 // parseExprWithChunk creates an api.WorkerChunkRequest to be used with the client.
 func parseExprWithChunk(expr string, chunkPath string) (*api.WorkerChunkRequest, error) {
 	// This is only for testing using the -chunk flag
-	search, err := zql.ParseProc(expr)
+	search, err := compiler.ParseProc(expr)
 	if err != nil {
 		return nil, err
 	}
diff --git a/cmd/zapi/cmd/index/create.go b/cmd/zapi/cmd/index/create.go
index 77128478f..09b9b804c 100644
--- a/cmd/zapi/cmd/index/create.go
+++ b/cmd/zapi/cmd/index/create.go
@@ -7,7 +7,7 @@ import (
 
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/brimsec/zq/zql"
+	"github.com/brimsec/zq/compiler"
 	"github.com/mccanne/charm"
 )
 
@@ -70,7 +70,7 @@ func (c *CreateCmd) Run(args []string) error {
 		OutputFile: c.outputFile,
 	}
 	if c.zql != "" {
-		_, err := zql.ParseProc(c.zql)
+		_, err := compiler.ParseProc(c.zql)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/zapi/cmd/post/flags.go b/cmd/zapi/cmd/post/flags.go
index e0f8c134a..c38e61655 100644
--- a/cmd/zapi/cmd/post/flags.go
+++ b/cmd/zapi/cmd/post/flags.go
@@ -7,7 +7,7 @@ import (
 	"github.com/brimsec/zq/api/client"
 	"github.com/brimsec/zq/ast"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/brimsec/zq/zql"
+	"github.com/brimsec/zq/compiler"
 )
 
 type postFlags struct {
@@ -30,7 +30,7 @@ func (f *postFlags) Init() error {
 		return err
 	}
 	if f.shaper != "" {
-		ast, err := zql.ParseProc(f.shaper)
+		ast, err := compiler.ParseProc(f.shaper)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/zq/zq.go b/cmd/zq/zq.go
index 308ab6a8e..7ab62b75e 100644
--- a/cmd/zq/zq.go
+++ b/cmd/zq/zq.go
@@ -10,13 +10,13 @@ import (
 	"github.com/brimsec/zq/cli/inputflags"
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cli/procflags"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/pkg/rlimit"
 	"github.com/brimsec/zq/pkg/s3io"
 	"github.com/brimsec/zq/pkg/signalctx"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/mccanne/charm"
 )
 
@@ -135,7 +135,7 @@ func (c *Command) Run(args []string) error {
 		zqlSrc = paths[0]
 		paths = paths[1:]
 	}
-	query, err := zql.ParseProc(zqlSrc)
+	query, err := compiler.ParseProc(zqlSrc)
 	if err != nil {
 		return fmt.Errorf("parse error: %s", err)
 	}
diff --git a/compiler/compiler.go b/compiler/compiler.go
new file mode 100644
index 000000000..6573d431a
--- /dev/null
+++ b/compiler/compiler.go
@@ -0,0 +1,50 @@
+package compiler
+
+import (
+	"github.com/brimsec/zq/ast"
+	"github.com/brimsec/zq/pkg/joe"
+	"github.com/brimsec/zq/zql"
+	"github.com/mitchellh/mapstructure"
+)
+
+// ParseProc() is an entry point for use from external go code,
+// mostly just a wrapper around Parse() that casts the return value.
+func ParseProc(query string, opts ...zql.Option) (ast.Proc, error) {
+	parsed, err := zql.Parse("", []byte(query), opts...)
+	if err != nil {
+		return nil, err
+	}
+	return ast.UnpackMap(nil, parsed)
+}
+
+func ParseExpression(expr string) (ast.Expression, error) {
+	m, err := zql.Parse("", []byte(expr), zql.Entrypoint("Expr"))
+	if err != nil {
+		return nil, err
+	}
+	node := joe.Convert(m)
+	ex, err := ast.UnpackExpression(node)
+	if err != nil {
+		return nil, err
+	}
+	c := &mapstructure.DecoderConfig{
+		TagName: "json",
+		Result:  ex,
+		Squash:  true,
+	}
+	dec, err := mapstructure.NewDecoder(c)
+	if err != nil {
+		return nil, err
+	}
+	return ex, dec.Decode(m)
+}
+
+// MustParseProc is functionally the same as ParseProc but panics if an error
+// is encountered.
+func MustParseProc(query string) ast.Proc {
+	proc, err := ParseProc(query)
+	if err != nil {
+		panic(err)
+	}
+	return proc
+}
diff --git a/filter/fieldnamefinder.go b/filter/fieldnamefinder.go
index 1d415aeae..1aa2fe0b8 100644
--- a/filter/fieldnamefinder.go
+++ b/filter/fieldnamefinder.go
@@ -81,7 +81,10 @@ type fieldNameIterInfo struct {
 
 func (f *fieldNameIter) init(t *zng.TypeRecord) {
 	f.buf = f.buf[:0]
-	f.stack = append(f.stack[:0], fieldNameIterInfo{t.Columns, 0})
+	f.stack = f.stack[:0]
+	if len(t.Columns) > 0 {
+		f.stack = append(f.stack, fieldNameIterInfo{t.Columns, 0})
+	}
 }
 
 func (f *fieldNameIter) done() bool {
@@ -89,13 +92,13 @@ func (f *fieldNameIter) done() bool {
 }
 
 func (f *fieldNameIter) next() []byte {
-	// Step into records.
+	// Step into non-empty records.
 	for {
 		info := &f.stack[len(f.stack)-1]
 		col := info.columns[info.offset]
 		f.buf = append(f.buf, "."+col.Name...)
 		t, ok := zng.AliasedType(col.Type).(*zng.TypeRecord)
-		if !ok {
+		if !ok || len(t.Columns) == 0 {
 			break
 		}
 		f.stack = append(f.stack, fieldNameIterInfo{t.Columns, 0})
diff --git a/ppl/archive/index/rule.go b/ppl/archive/index/rule.go
index 7bcd35f8f..d6eb84e10 100644
--- a/ppl/archive/index/rule.go
+++ b/ppl/archive/index/rule.go
@@ -5,12 +5,12 @@ import (
 	"fmt"
 
 	"github.com/brimsec/zq/ast"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/field"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/zngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 )
 
 type RuleKind string
@@ -73,7 +73,7 @@ func UnmarshalRule(b []byte) (Rule, error) {
 
 func NewZqlRule(prog, name string, keys []field.Static) (Rule, error) {
 	// make sure it compiles
-	if _, err := zql.ParseProc(prog); err != nil {
+	if _, err := compiler.ParseProc(prog); err != nil {
 		return Rule{}, err
 	}
 	return Rule{
@@ -158,7 +158,7 @@ func (r Rule) fieldProc() (ast.Proc, error) {
 }
 
 func (r Rule) zqlProc() (ast.Proc, error) {
-	return zql.ParseProc(r.ZQL)
+	return compiler.ParseProc(r.ZQL)
 }
 
 func (r Rule) String() string {
diff --git a/ppl/cmd/zar/map/command.go b/ppl/cmd/zar/map/command.go
index 226650f9d..3680ecc38 100644
--- a/ppl/cmd/zar/map/command.go
+++ b/ppl/cmd/zar/map/command.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cli/procflags"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/emitter"
 	"github.com/brimsec/zq/pkg/iosrc"
@@ -20,7 +21,6 @@ import (
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/mccanne/charm"
 )
 
@@ -71,7 +71,7 @@ func (c *Command) Run(args []string) error {
 	if len(args) == 0 {
 		return errors.New("zar map needs input arguments")
 	}
-	query, err := zql.ParseProc(args[0])
+	query, err := compiler.ParseProc(args[0])
 	if err != nil {
 		return err
 	}
diff --git a/ppl/cmd/zar/zq/command.go b/ppl/cmd/zar/zq/command.go
index 7fdbf9646..23fe77fc1 100644
--- a/ppl/cmd/zar/zq/command.go
+++ b/ppl/cmd/zar/zq/command.go
@@ -9,13 +9,13 @@ import (
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cli/procflags"
 	"github.com/brimsec/zq/cli/searchflags"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/pkg/rlimit"
 	"github.com/brimsec/zq/pkg/signalctx"
 	"github.com/brimsec/zq/ppl/archive"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/mccanne/charm"
 )
 
@@ -70,7 +70,7 @@ func (c *Command) Run(args []string) error {
 		return err
 	}
 
-	query, err := zql.ParseProc(args[0])
+	query, err := compiler.ParseProc(args[0])
 	if err != nil {
 		return err
 	}
diff --git a/ppl/zqd/handlers.go b/ppl/zqd/handlers.go
index f338d64e3..83c3b852e 100644
--- a/ppl/zqd/handlers.go
+++ b/ppl/zqd/handlers.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/brimsec/zq/api"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/pcap"
 	"github.com/brimsec/zq/pkg/ctxio"
 	"github.com/brimsec/zq/ppl/zqd/ingest"
@@ -18,7 +19,6 @@ import (
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zqe"
-	"github.com/brimsec/zq/zql"
 	"github.com/gorilla/mux"
 	"go.uber.org/zap"
 )
@@ -80,7 +80,7 @@ func handleASTPost(c *Core, w http.ResponseWriter, r *http.Request) {
 	if !request(c, w, r, &req) {
 		return
 	}
-	proc, err := zql.ParseProc(req.ZQL)
+	proc, err := compiler.ParseProc(req.ZQL)
 	if err != nil {
 		respondError(c, w, r, zqe.ErrInvalid(err))
 		return
diff --git a/ppl/zqd/ingest/pcap.go b/ppl/zqd/ingest/pcap.go
index 2b49ac3b7..21d94bdac 100644
--- a/ppl/zqd/ingest/pcap.go
+++ b/ppl/zqd/ingest/pcap.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/brimsec/zq/api"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/iosrc"
@@ -27,7 +28,6 @@ import (
 	"github.com/brimsec/zq/zio/ndjsonio"
 	"github.com/brimsec/zq/zio/zngio"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 )
 
 //go:generate go run ../../../zio/ndjsonio/typegenerator -o ./suricata.go -package ingest -var suricataTC ./suricata-types.json
@@ -43,7 +43,7 @@ type PcapOp interface {
 	Snap() <-chan struct{}
 }
 
-var suricataTransform = zql.MustParseProc("rename ts=timestamp")
+var suricataTransform = compiler.MustParseProc("rename ts=timestamp")
 
 type ClearableStore interface {
 	storage.Storage
diff --git a/ppl/zqd/storage/filestore/filestore.go b/ppl/zqd/storage/filestore/filestore.go
index 5991a3a89..82a5b69e3 100644
--- a/ppl/zqd/storage/filestore/filestore.go
+++ b/ppl/zqd/storage/filestore/filestore.go
@@ -11,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/brimsec/zq/api"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/pkg/bufwriter"
 	"github.com/brimsec/zq/pkg/fs"
@@ -25,7 +26,6 @@ import (
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zqe"
-	"github.com/brimsec/zq/zql"
 	"go.uber.org/zap"
 	"golang.org/x/sync/semaphore"
 )
@@ -38,7 +38,7 @@ const (
 var (
 	ErrWriteInProgress = errors.New("another write is already in progress")
 
-	zngWriteProc = zql.MustParseProc("sort -r ts")
+	zngWriteProc = compiler.MustParseProc("sort -r ts")
 )
 
 func Load(path iosrc.URI, logger *zap.Logger) (*Storage, error) {
diff --git a/python/brim/src/zqext.go b/python/brim/src/zqext.go
index ae24a215f..03f0c6278 100644
--- a/python/brim/src/zqext.go
+++ b/python/brim/src/zqext.go
@@ -6,12 +6,12 @@ import (
 	"context"
 	"errors"
 
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/emitter"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 )
 
 // result converts an error into response structure expected
@@ -47,7 +47,7 @@ func doZqlFileEval(inquery, inpath, informat, outpath, outformat string) (err er
 	if outpath == "-" {
 		outpath = "/dev/stdout"
 	}
-	query, err := zql.ParseProc(inquery)
+	query, err := compiler.ParseProc(inquery)
 	if err != nil {
 		return err
 	}
diff --git a/ztests/formats/csv/hello.yaml b/zio/csvio/ztests/record-flat.yaml
similarity index 100%
rename from ztests/formats/csv/hello.yaml
rename to zio/csvio/ztests/record-flat.yaml
diff --git a/ztests/formats/csv/nested.yaml b/zio/csvio/ztests/record-nested.yaml
similarity index 100%
rename from ztests/formats/csv/nested.yaml
rename to zio/csvio/ztests/record-nested.yaml
diff --git a/ztests/formats/autodetect/bzng-gz.yaml b/zio/detector/ztests/bzng-gz.yaml
similarity index 100%
rename from ztests/formats/autodetect/bzng-gz.yaml
rename to zio/detector/ztests/bzng-gz.yaml
diff --git a/ztests/formats/autodetect/ndjson-gz.yaml b/zio/detector/ztests/ndjson-gz.yaml
similarity index 100%
rename from ztests/formats/autodetect/ndjson-gz.yaml
rename to zio/detector/ztests/ndjson-gz.yaml
diff --git a/ztests/formats/autodetect/ndjson-nonewline.yaml b/zio/detector/ztests/ndjson-nonewline.yaml
similarity index 100%
rename from ztests/formats/autodetect/ndjson-nonewline.yaml
rename to zio/detector/ztests/ndjson-nonewline.yaml
diff --git a/ztests/formats/autodetect/ndjson.yaml b/zio/detector/ztests/ndjson.yaml
similarity index 100%
rename from ztests/formats/autodetect/ndjson.yaml
rename to zio/detector/ztests/ndjson.yaml
diff --git a/ztests/formats/autodetect/zeek-gz.yaml b/zio/detector/ztests/zeek-gz.yaml
similarity index 100%
rename from ztests/formats/autodetect/zeek-gz.yaml
rename to zio/detector/ztests/zeek-gz.yaml
diff --git a/ztests/formats/autodetect/zeek-nonewline.yaml b/zio/detector/ztests/zeek-nonewline.yaml
similarity index 100%
rename from ztests/formats/autodetect/zeek-nonewline.yaml
rename to zio/detector/ztests/zeek-nonewline.yaml
diff --git a/ztests/formats/autodetect/zeek.yaml b/zio/detector/ztests/zeek.yaml
similarity index 100%
rename from ztests/formats/autodetect/zeek.yaml
rename to zio/detector/ztests/zeek.yaml
diff --git a/ztests/formats/autodetect/zjson-gz.yaml b/zio/detector/ztests/zjson-gz.yaml
similarity index 100%
rename from ztests/formats/autodetect/zjson-gz.yaml
rename to zio/detector/ztests/zjson-gz.yaml
diff --git a/ztests/formats/autodetect/zjson-nonewline.yaml b/zio/detector/ztests/zjson-nonewline.yaml
similarity index 100%
rename from ztests/formats/autodetect/zjson-nonewline.yaml
rename to zio/detector/ztests/zjson-nonewline.yaml
diff --git a/ztests/formats/autodetect/zjson.yaml b/zio/detector/ztests/zjson.yaml
similarity index 100%
rename from ztests/formats/autodetect/zjson.yaml
rename to zio/detector/ztests/zjson.yaml
diff --git a/ztests/formats/autodetect/zng-gz.yaml b/zio/detector/ztests/zng-gz.yaml
similarity index 100%
rename from ztests/formats/autodetect/zng-gz.yaml
rename to zio/detector/ztests/zng-gz.yaml
diff --git a/ztests/formats/autodetect/zng-nonewline.yaml b/zio/detector/ztests/zng-nonewline.yaml
similarity index 100%
rename from ztests/formats/autodetect/zng-nonewline.yaml
rename to zio/detector/ztests/zng-nonewline.yaml
diff --git a/ztests/formats/autodetect/zng-rdwr-comp.yaml b/zio/detector/ztests/zng-rdwr-comp.yaml
similarity index 100%
rename from ztests/formats/autodetect/zng-rdwr-comp.yaml
rename to zio/detector/ztests/zng-rdwr-comp.yaml
diff --git a/ztests/formats/autodetect/zng.yaml b/zio/detector/ztests/zng.yaml
similarity index 100%
rename from ztests/formats/autodetect/zng.yaml
rename to zio/detector/ztests/zng.yaml
diff --git a/ztests/formats/no-path/1.yaml b/zio/zeekio/no-path.yaml
similarity index 100%
rename from ztests/formats/no-path/1.yaml
rename to zio/zeekio/no-path.yaml
diff --git a/ztests/formats/path/path.yaml b/zio/zeekio/path.yaml
similarity index 100%
rename from ztests/formats/path/path.yaml
rename to zio/zeekio/path.yaml
diff --git a/ztests/formats/path/path2.yaml b/zio/zeekio/path2.yaml
similarity index 100%
rename from ztests/formats/path/path2.yaml
rename to zio/zeekio/path2.yaml
diff --git a/zql/parser-support.go b/zql/parser-support.go
index c60f2cbaf..a851b67ac 100644
--- a/zql/parser-support.go
+++ b/zql/parser-support.go
@@ -5,54 +5,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-
-	"github.com/brimsec/zq/ast"
-	"github.com/brimsec/zq/pkg/joe"
-	"github.com/mitchellh/mapstructure"
 )
 
-// ParseProc() is an entry point for use from external go code,
-// mostly just a wrapper around Parse() that casts the return value.
-func ParseProc(query string, opts ...Option) (ast.Proc, error) {
-	parsed, err := Parse("", []byte(query), opts...)
-	if err != nil {
-		return nil, err
-	}
-	return ast.UnpackMap(nil, parsed)
-}
-
-func ParseExpression(expr string) (ast.Expression, error) {
-	m, err := Parse("", []byte(expr), Entrypoint("Expr"))
-	if err != nil {
-		return nil, err
-	}
-	node := joe.Convert(m)
-	ex, err := ast.UnpackExpression(node)
-	if err != nil {
-		return nil, err
-	}
-	c := &mapstructure.DecoderConfig{
-		TagName: "json",
-		Result:  ex,
-		Squash:  true,
-	}
-	dec, err := mapstructure.NewDecoder(c)
-	if err != nil {
-		return nil, err
-	}
-	return ex, dec.Decode(m)
-}
-
-// MustParseProc is functionally the same as ParseProc but panics if an error
-// is encountered.
-func MustParseProc(query string) ast.Proc {
-	proc, err := ParseProc(query)
-	if err != nil {
-		panic(err)
-	}
-	return proc
-}
-
 func makeChain(first interface{}, restIn interface{}, op string) interface{} {
 	rest, ok := restIn.([]interface{})
 	if !ok || len(rest) == 0 {


Test patch:
diff --git a/compiler/compile_test.go b/compiler/compile_test.go
index 46dbc5145..ec09c4f2b 100644
--- a/compiler/compile_test.go
+++ b/compiler/compile_test.go
@@ -5,7 +5,6 @@ import (
 
 	"github.com/brimsec/zq/ast"
 	"github.com/brimsec/zq/field"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -75,7 +74,7 @@ func TestComputeColumns(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.zql, func(t *testing.T) {
-			query, err := zql.ParseProc(tc.zql)
+			query, err := ParseProc(tc.zql)
 			require.NoError(t, err)
 			// apply ReplaceGroupByProcDurationWithKey here because computeColumns
 			// will be applied after it when it is plugged into compilation.
@@ -150,7 +149,7 @@ func TestExpressionFields(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.expr, func(t *testing.T) {
-			e, err := zql.ParseExpression(tc.expr)
+			e, err := ParseExpression(tc.expr)
 			require.NoError(t, err)
 			f := expressionFields(e)
 			assert.Equal(t, tc.expected, f)
@@ -186,7 +185,7 @@ func TestBooleanExpressionFields(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.expr, func(t *testing.T) {
-			parsed, err := zql.ParseProc(tc.expr)
+			parsed, err := ParseProc(tc.expr)
 			require.NoError(t, err)
 			f := booleanExpressionFields(parsed.(*ast.FilterProc).Filter)
 			assert.Equal(t, tc.expected, f)
@@ -289,7 +288,7 @@ func TestParallelizeFlowgraph(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.zql, func(t *testing.T) {
-			query, err := zql.ParseProc(tc.zql)
+			query, err := ParseProc(tc.zql)
 			require.NoError(t, err)
 
 			ok := IsParallelizable(query.(*ast.SequentialProc), sf(tc.orderField), false)
@@ -298,7 +297,7 @@ func TestParallelizeFlowgraph(t *testing.T) {
 			parallelized, ok := Parallelize(query.(*ast.SequentialProc), 2, sf(tc.orderField), false)
 			require.Equal(t, ok, tc.zql != tc.expected)
 
-			expected, err := zql.ParseProc(tc.expected)
+			expected, err := ParseProc(tc.expected)
 			require.NoError(t, err)
 
 			// If the parallelized flowgraph includes a groupby, then adjust the expected AST by setting
@@ -324,12 +323,12 @@ func TestParallelizeFlowgraph(t *testing.T) {
 		orderField := "ts"
 		query := "* | cut ts, y, z | put x=y | rename y=z"
 		dquery := "(filter * | cut ts, y, z | put x=y | rename y=z; filter * | cut ts, y, z | put x=y | rename y=z)"
-		program, err := zql.ParseProc(query)
+		program, err := ParseProc(query)
 		require.NoError(t, err)
 		parallelized, ok := Parallelize(program.(*ast.SequentialProc), 2, sf(orderField), false)
 		require.True(t, ok)
 
-		expected, err := zql.ParseProc(dquery)
+		expected, err := ParseProc(dquery)
 		require.NoError(t, err)
 
 		// We can't express a pass proc in zql, so add it to the AST this way.
@@ -357,7 +356,7 @@ func TestSetGroupByProcInputSortDir(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.zql, func(t *testing.T) {
-			query, err := zql.ParseProc(tc.zql)
+			query, err := ParseProc(tc.zql)
 			require.NoError(t, err)
 			ReplaceGroupByProcDurationWithKey(query)
 			outputSorted := setGroupByProcInputSortDir(query, sf(tc.inputSortField), 1)
diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go
index cff0a6ec5..93d9b46bf 100644
--- a/compiler/compiler_test.go
+++ b/compiler/compiler_test.go
@@ -16,7 +16,6 @@ import (
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -39,7 +38,7 @@ func TestCompileParents(t *testing.T) {
 		sources = append(sources, &proctest.RecordPuller{R: r})
 	}
 	t.Run("read two sources", func(t *testing.T) {
-		query, err := zql.ParseProc("(filter *; filter *) | filter *")
+		query, err := compiler.ParseProc("(filter *; filter *) | filter *")
 		require.NoError(t, err)
 
 		leaves, err := compiler.Compile(nil, query, pctx, sources)
@@ -52,7 +51,7 @@ func TestCompileParents(t *testing.T) {
 	})
 
 	t.Run("too few parents", func(t *testing.T) {
-		query, err := zql.ParseProc("(filter *; filter *; filter *) | filter *")
+		query, err := compiler.ParseProc("(filter *; filter *; filter *) | filter *")
 		require.NoError(t, err)
 
 		query.(*ast.SequentialProc).Procs = query.(*ast.SequentialProc).Procs[1:]
@@ -62,7 +61,7 @@ func TestCompileParents(t *testing.T) {
 	})
 
 	t.Run("too many parents", func(t *testing.T) {
-		query, err := zql.ParseProc("* | (filter *; filter *) | filter *")
+		query, err := compiler.ParseProc("* | (filter *; filter *) | filter *")
 		require.NoError(t, err)
 		_, err = compiler.Compile(nil, query, pctx, sources)
 		require.Error(t, err)
@@ -85,7 +84,7 @@ func TestCompileMergeDone(t *testing.T) {
 	pctx := &proc.Context{Context: context.Background(), TypeContext: zctx}
 	r := tzngio.NewReader(bytes.NewReader([]byte(input)), zctx)
 	src := &proctest.RecordPuller{R: r}
-	query, err := zql.ParseProc("(filter * ; head 1) | head 3")
+	query, err := compiler.ParseProc("(filter * ; head 1) | head 3")
 	require.NoError(t, err)
 
 	seq, ok := query.(*ast.SequentialProc)
diff --git a/driver/driver_test.go b/driver/driver_test.go
index 7c26553d2..72fee1605 100644
--- a/driver/driver_test.go
+++ b/driver/driver_test.go
@@ -5,11 +5,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -27,7 +27,7 @@ func TestMuxDriver(t *testing.T) {
 #0:record[_path:string,ts:time]
 0:[conn;1425565514.419939;]`
 
-	query, err := zql.ParseProc("(tail 1; tail 1)")
+	query, err := compiler.ParseProc("(tail 1; tail 1)")
 	assert.NoError(t, err)
 
 	t.Run("muxed into one writer", func(t *testing.T) {
diff --git a/driver/parallel_test.go b/driver/parallel_test.go
index 5aff8d76e..337342cd8 100644
--- a/driver/parallel_test.go
+++ b/driver/parallel_test.go
@@ -10,13 +10,13 @@ import (
 	"time"
 
 	"github.com/brimsec/zq/api"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/field"
 	"github.com/brimsec/zq/pkg/nano"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -116,7 +116,7 @@ func TestParallelOrder(t *testing.T) {
 	t.Parallel()
 
 	// Use `v!=3` to trigger & verify empty rank handling in orderedWaiter.
-	query, err := zql.ParseProc("v!=3")
+	query, err := compiler.ParseProc("v!=3")
 	require.NoError(t, err)
 
 	var buf bytes.Buffer
@@ -185,7 +185,7 @@ func (m *scannerCloseMS) SourceFromRequest(context.Context, *api.WorkerChunkRequ
 // TestScannerClose verifies that any open ScannerCloser's will be closed soon
 // after the MultiRun call finishes.
 func TestScannerClose(t *testing.T) {
-	query, err := zql.ParseProc("* | head 1")
+	query, err := compiler.ParseProc("* | head 1")
 	require.NoError(t, err)
 
 	var buf bytes.Buffer
diff --git a/expr/expr_test.go b/expr/expr_test.go
index dff20a29c..792bdc60e 100644
--- a/expr/expr_test.go
+++ b/expr/expr_test.go
@@ -14,7 +14,6 @@ import (
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -44,7 +43,7 @@ func parseOneRecord(zngsrc string) (*zng.Record, error) {
 }
 
 func compileExpr(s string) (expr.Evaluator, error) {
-	parsed, err := zql.ParseExpression(s)
+	parsed, err := compiler.ParseExpression(s)
 	if err != nil {
 		return nil, err
 	}
diff --git a/filter/fieldnamefinder_test.go b/filter/fieldnamefinder_test.go
index da84ea6d7..678c519d1 100644
--- a/filter/fieldnamefinder_test.go
+++ b/filter/fieldnamefinder_test.go
@@ -9,11 +9,12 @@ import (
 )
 
 func TestFieldNameIter(t *testing.T) {
-	const typeString = "record[r1:record[r2:record[s:string,r3:record[t:time]],a:array[int64],r4:record[i:ip]]]"
+	const typeString = "record[r1:record[r2:record[s:string,r3:record[t:time]],a:array[int64],r4:record[i:ip]],empty:record[]]"
 	typ, err := resolver.NewContext().LookupByName(typeString)
 	require.NoError(t, err)
 	var f fieldNameIter
 	f.init(typ.(*zng.TypeRecord))
+	require.False(t, f.done())
 	require.Exactly(t, "r1.r2.s", string(f.next()))
 	require.False(t, f.done())
 	require.Exactly(t, "r1.r2.r3.t", string(f.next()))
@@ -21,5 +22,14 @@ func TestFieldNameIter(t *testing.T) {
 	require.Exactly(t, "r1.a", string(f.next()))
 	require.False(t, f.done())
 	require.Exactly(t, "r1.r4.i", string(f.next()))
+	require.False(t, f.done())
+	require.Exactly(t, "empty", string(f.next()))
+	require.True(t, f.done())
+}
+func TestFieldNameIterEmptyTopLevelRecord(t *testing.T) {
+	typ, err := resolver.NewContext().LookupByName("record[]")
+	require.NoError(t, err)
+	var f fieldNameIter
+	f.init(typ.(*zng.TypeRecord))
 	require.True(t, f.done())
 }
diff --git a/filter/filter_test.go b/filter/filter_test.go
index 8ee8bcc44..d6bd99445 100644
--- a/filter/filter_test.go
+++ b/filter/filter_test.go
@@ -11,7 +11,6 @@ import (
 	"github.com/brimsec/zq/zcode"
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -44,7 +43,7 @@ func runCasesHelper(t *testing.T, tzng string, cases []testcase, expectBufferFil
 		t.Run(c.filter, func(t *testing.T) {
 			t.Helper()
 
-			proc, err := zql.ParseProc(c.filter)
+			proc, err := compiler.ParseProc(c.filter)
 			require.NoError(t, err, "filter: %q", c.filter)
 			filterExpr := proc.(*ast.FilterProc).Filter
 
@@ -499,11 +498,27 @@ func TestFilters(t *testing.T) {
 		{"rec.str", true},
 	})
 
+	// Test searching an empty top-level record
+	tzng = `
+#0:record[]
+0:[]`
+	runCases(t, tzng, []testcase{
+		{"empty", false},
+	})
+
+	// Test searching an empty nested record
+	tzng = `
+#0:record[empty:record[]]
+0:[[]]`
+	runCases(t, tzng, []testcase{
+		{"empty", true},
+	})
+
 }
 
 func TestBadFilter(t *testing.T) {
 	t.Parallel()
-	proc, err := zql.ParseProc(`s =~ \xa8*`)
+	proc, err := compiler.ParseProc(`s =~ \xa8*`)
 	require.NoError(t, err)
 	f := compiler.NewFilter(resolver.NewContext(), proc.(*ast.FilterProc).Filter)
 	_, err = f.AsFilter()
diff --git a/microindex/microindex_test.go b/microindex/microindex_test.go
index 4eb59783f..6ba2bb1bf 100644
--- a/microindex/microindex_test.go
+++ b/microindex/microindex_test.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/microindex"
 	"github.com/brimsec/zq/pkg/iosrc"
@@ -16,7 +17,6 @@ import (
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -131,7 +131,7 @@ func TestCompare(t *testing.T) {
 			runtest(t, desc, "==", c.value, c.eql)
 		}
 	})
-	r, err := driver.NewReader(context.Background(), zql.MustParseProc("sort ts"), resolver.NewContext(), reader(records))
+	r, err := driver.NewReader(context.Background(), compiler.MustParseProc("sort ts"), resolver.NewContext(), reader(records))
 	require.NoError(t, err)
 	asc := buildAndOpen(t, r, microindex.Keys("ts"), microindex.Order(zbuf.OrderAsc))
 	t.Run("Ascending", func(t *testing.T) {
diff --git a/pkg/test/internal.go b/pkg/test/internal.go
index c429d88db..f0f31b14f 100644
--- a/pkg/test/internal.go
+++ b/pkg/test/internal.go
@@ -6,13 +6,13 @@ import (
 	"io/ioutil"
 	"strings"
 
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/emitter"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 )
 
 type Internal struct {
@@ -47,7 +47,7 @@ func newEmitter(ofmt string) (*emitter.Bytes, error) {
 }
 
 func (i *Internal) Run() (string, error) {
-	program, err := zql.ParseProc(i.Query)
+	program, err := compiler.ParseProc(i.Query)
 	if err != nil {
 		return "", fmt.Errorf("parse error: %s (%s)", err, i.Query)
 	}
diff --git a/ppl/zqd/handlers_test.go b/ppl/zqd/handlers_test.go
index c53cb46bb..84dab4feb 100644
--- a/ppl/zqd/handlers_test.go
+++ b/ppl/zqd/handlers_test.go
@@ -19,6 +19,7 @@ import (
 
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/api/client"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/nano"
@@ -31,7 +32,6 @@ import (
 	"github.com/brimsec/zq/zio/ndjsonio"
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -79,7 +79,7 @@ func TestSearchNoCtrl(t *testing.T) {
 	_, err = conn.LogPostReaders(context.Background(), sp.ID, nil, strings.NewReader(src))
 	require.NoError(t, err)
 
-	parsed, err := zql.ParseProc("*")
+	parsed, err := compiler.ParseProc("*")
 	require.NoError(t, err)
 	proc, err := json.Marshal(parsed)
 	require.NoError(t, err)
@@ -173,7 +173,7 @@ func TestSearchError(t *testing.T) {
 	_, err = conn.LogPostReaders(context.Background(), sp.ID, nil, strings.NewReader(src))
 	require.NoError(t, err)
 
-	parsed, err := zql.ParseProc("*")
+	parsed, err := compiler.ParseProc("*")
 	require.NoError(t, err)
 	proc, err := json.Marshal(parsed)
 	require.NoError(t, err)
@@ -948,7 +948,7 @@ func indexSearch(t *testing.T, conn *client.Connection, space api.SpaceID, index
 // space, returning the tzng results along with a slice of all control
 // messages that were received.
 func search(t *testing.T, conn *client.Connection, space api.SpaceID, prog string) (string, []interface{}) {
-	parsed, err := zql.ParseProc(prog)
+	parsed, err := compiler.ParseProc(prog)
 	require.NoError(t, err)
 	proc, err := json.Marshal(parsed)
 	require.NoError(t, err)
@@ -981,7 +981,7 @@ func tzngCopy(t *testing.T, prog string, in string, outFormat string) string {
 	buf := bytes.NewBuffer(nil)
 	w, err := detector.LookupWriter(zio.NopCloser(buf), zio.WriterOpts{Format: outFormat})
 	require.NoError(t, err)
-	p := zql.MustParseProc(prog)
+	p := compiler.MustParseProc(prog)
 	err = driver.Copy(context.Background(), w, p, zctx, r, driver.Config{})
 	require.NoError(t, err)
 	return buf.String()
diff --git a/ppl/zqd/ingest/logtailer_test.go b/ppl/zqd/ingest/logtailer_test.go
index 5e7a1a21f..9ba75dd64 100644
--- a/ppl/zqd/ingest/logtailer_test.go
+++ b/ppl/zqd/ingest/logtailer_test.go
@@ -9,15 +9,15 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/suite"
 )
 
-var sortTs = zql.MustParseProc("sort ts")
+var sortTs = compiler.MustParseProc("sort ts")
 
 const expected = `#0:record[ts:time]
 0:[0;]
diff --git a/ztests/suite/zqd/alphazng/alphazng.yaml b/ppl/zqd/ztests/alphazng/alphazng.yaml
similarity index 100%
rename from ztests/suite/zqd/alphazng/alphazng.yaml
rename to ppl/zqd/ztests/alphazng/alphazng.yaml
diff --git a/ztests/suite/zqd/alphazng/zqd-v0.18.0-zqdroot.tgz b/ppl/zqd/ztests/alphazng/zqd-v0.18.0-zqdroot.tgz
similarity index 100%
rename from ztests/suite/zqd/alphazng/zqd-v0.18.0-zqdroot.tgz
rename to ppl/zqd/ztests/alphazng/zqd-v0.18.0-zqdroot.tgz
diff --git a/ztests/suite/zqd/archivestore/postpcap-stats.yaml b/ppl/zqd/ztests/archivestore/postpcap-stats.yaml
similarity index 86%
rename from ztests/suite/zqd/archivestore/postpcap-stats.yaml
rename to ppl/zqd/ztests/archivestore/postpcap-stats.yaml
index 2dea0b98b..e0846d107 100644
--- a/ztests/suite/zqd/archivestore/postpcap-stats.yaml
+++ b/ppl/zqd/ztests/archivestore/postpcap-stats.yaml
@@ -4,7 +4,7 @@ script: |
 
 inputs:
   - name: ng.pcap
-    source: ../../pcap/ng.pcap
+    source: ../../../../ztests/suite/pcap/ng.pcap
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/archivestore/postpcap-suricata.yaml b/ppl/zqd/ztests/archivestore/postpcap-suricata.yaml
similarity index 100%
rename from ztests/suite/zqd/archivestore/postpcap-suricata.yaml
rename to ppl/zqd/ztests/archivestore/postpcap-suricata.yaml
diff --git a/ztests/suite/zqd/archivestore/postpcap.yaml b/ppl/zqd/ztests/archivestore/postpcap.yaml
similarity index 97%
rename from ztests/suite/zqd/archivestore/postpcap.yaml
rename to ppl/zqd/ztests/archivestore/postpcap.yaml
index b564083c2..5f24382c2 100644
--- a/ztests/suite/zqd/archivestore/postpcap.yaml
+++ b/ppl/zqd/ztests/archivestore/postpcap.yaml
@@ -7,7 +7,7 @@ script: |
 
 inputs:
   - name: ng.pcap
-    source: ../../pcap/ng.pcap
+    source: ../../../../ztests/suite/pcap/ng.pcap
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/archivestore/subspace.yaml b/ppl/zqd/ztests/archivestore/subspace.yaml
similarity index 100%
rename from ztests/suite/zqd/archivestore/subspace.yaml
rename to ppl/zqd/ztests/archivestore/subspace.yaml
diff --git a/ztests/suite/zqd/common.sh b/ppl/zqd/ztests/common.sh
similarity index 100%
rename from ztests/suite/zqd/common.sh
rename to ppl/zqd/ztests/common.sh
diff --git a/ztests/suite/zqd/csv-fuse.yaml b/ppl/zqd/ztests/csv-fuse.yaml
similarity index 100%
rename from ztests/suite/zqd/csv-fuse.yaml
rename to ppl/zqd/ztests/csv-fuse.yaml
diff --git a/ztests/suite/zqd/csv.yaml b/ppl/zqd/ztests/csv.yaml
similarity index 100%
rename from ztests/suite/zqd/csv.yaml
rename to ppl/zqd/ztests/csv.yaml
diff --git a/ztests/suite/zqd/index-custom.yaml b/ppl/zqd/ztests/index-custom.yaml
similarity index 92%
rename from ztests/suite/zqd/index-custom.yaml
rename to ppl/zqd/ztests/index-custom.yaml
index ea7b035c2..c0a775d64 100644
--- a/ztests/suite/zqd/index-custom.yaml
+++ b/ppl/zqd/ztests/index-custom.yaml
@@ -9,7 +9,7 @@ inputs:
   - name: services.sh
     source: services.sh
   - name: babble.tzng
-    source: ../data/babble.tzng
+    source: ../../../ztests/suite/data/babble.tzng
 
 outputs:
   - name: stdout
diff --git a/ztests/suite/zqd/index-new-data.yaml b/ppl/zqd/ztests/index-new-data.yaml
similarity index 100%
rename from ztests/suite/zqd/index-new-data.yaml
rename to ppl/zqd/ztests/index-new-data.yaml
diff --git a/ztests/suite/zqd/index-type.yaml b/ppl/zqd/ztests/index-type.yaml
similarity index 100%
rename from ztests/suite/zqd/index-type.yaml
rename to ppl/zqd/ztests/index-type.yaml
diff --git a/ztests/suite/zqd/json.yaml b/ppl/zqd/ztests/json.yaml
similarity index 100%
rename from ztests/suite/zqd/json.yaml
rename to ppl/zqd/ztests/json.yaml
diff --git a/ztests/suite/zqd/mockbrim.sh b/ppl/zqd/ztests/mockbrim.sh
similarity index 100%
rename from ztests/suite/zqd/mockbrim.sh
rename to ppl/zqd/ztests/mockbrim.sh
diff --git a/ztests/suite/zqd/ndjson.yaml b/ppl/zqd/ztests/ndjson.yaml
similarity index 100%
rename from ztests/suite/zqd/ndjson.yaml
rename to ppl/zqd/ztests/ndjson.yaml
diff --git a/ztests/suite/zqd/orphaned-by-brim.yaml b/ppl/zqd/ztests/orphaned-by-brim.yaml
similarity index 100%
rename from ztests/suite/zqd/orphaned-by-brim.yaml
rename to ppl/zqd/ztests/orphaned-by-brim.yaml
diff --git a/ztests/suite/zqd/pcap/alerts.pcap b/ppl/zqd/ztests/pcap/alerts.pcap
similarity index 100%
rename from ztests/suite/zqd/pcap/alerts.pcap
rename to ppl/zqd/ztests/pcap/alerts.pcap
diff --git a/ztests/suite/zqd/pcap/emerging.rules.tar.gz b/ppl/zqd/ztests/pcap/emerging.rules.tar.gz
similarity index 100%
rename from ztests/suite/zqd/pcap/emerging.rules.tar.gz
rename to ppl/zqd/ztests/pcap/emerging.rules.tar.gz
diff --git a/ztests/suite/zqd/pcap/pcap-deleted.yaml b/ppl/zqd/ztests/pcap/pcap-deleted.yaml
similarity index 95%
rename from ztests/suite/zqd/pcap/pcap-deleted.yaml
rename to ppl/zqd/ztests/pcap/pcap-deleted.yaml
index 6a01f2239..eb319906d 100644
--- a/ztests/suite/zqd/pcap/pcap-deleted.yaml
+++ b/ppl/zqd/ztests/pcap/pcap-deleted.yaml
@@ -15,7 +15,7 @@ script: |
 
 inputs:
   - name: ng.pcap
-    source: ../../pcap/ng.pcap
+    source: ../../../../ztests/suite/pcap/ng.pcap
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/pcap/postpcap-suricata-update.yaml b/ppl/zqd/ztests/pcap/postpcap-suricata-update.yaml
similarity index 100%
rename from ztests/suite/zqd/pcap/postpcap-suricata-update.yaml
rename to ppl/zqd/ztests/pcap/postpcap-suricata-update.yaml
diff --git a/ztests/suite/zqd/pcap/postpcap-suricata.yaml b/ppl/zqd/ztests/pcap/postpcap-suricata.yaml
similarity index 100%
rename from ztests/suite/zqd/pcap/postpcap-suricata.yaml
rename to ppl/zqd/ztests/pcap/postpcap-suricata.yaml
diff --git a/ztests/suite/zqd/pcap/postpcap.yaml b/ppl/zqd/ztests/pcap/postpcap.yaml
similarity index 97%
rename from ztests/suite/zqd/pcap/postpcap.yaml
rename to ppl/zqd/ztests/pcap/postpcap.yaml
index abd5b1094..07ac873a6 100644
--- a/ztests/suite/zqd/pcap/postpcap.yaml
+++ b/ppl/zqd/ztests/pcap/postpcap.yaml
@@ -7,7 +7,7 @@ script: |
 
 inputs:
   - name: ng.pcap
-    source: ../../pcap/ng.pcap
+    source: ../../../../ztests/suite/pcap/ng.pcap
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/python.yaml b/ppl/zqd/ztests/python.yaml
similarity index 100%
rename from ztests/suite/zqd/python.yaml
rename to ppl/zqd/ztests/python.yaml
diff --git a/ztests/suite/zqd/recruiter-curl.sh b/ppl/zqd/ztests/recruiter-curl.sh
similarity index 100%
rename from ztests/suite/zqd/recruiter-curl.sh
rename to ppl/zqd/ztests/recruiter-curl.sh
diff --git a/ztests/suite/zqd/recruiter-curl.yaml b/ppl/zqd/ztests/recruiter-curl.yaml
similarity index 100%
rename from ztests/suite/zqd/recruiter-curl.yaml
rename to ppl/zqd/ztests/recruiter-curl.yaml
diff --git a/ztests/suite/zqd/recruiter-root.sh b/ppl/zqd/ztests/recruiter-root.sh
similarity index 100%
rename from ztests/suite/zqd/recruiter-root.sh
rename to ppl/zqd/ztests/recruiter-root.sh
diff --git a/ztests/suite/zqd/recruiter-root.yaml b/ppl/zqd/ztests/recruiter-root.yaml
similarity index 97%
rename from ztests/suite/zqd/recruiter-root.yaml
rename to ppl/zqd/ztests/recruiter-root.yaml
index 1f9e32973..053570f3b 100644
--- a/ztests/suite/zqd/recruiter-root.yaml
+++ b/ppl/zqd/ztests/recruiter-root.yaml
@@ -36,7 +36,7 @@ inputs:
   - name: recruiter-root.sh
     source: recruiter-root.sh
   - name: smtp.log.gz
-    source: ../data/smtp.log.gz
+    source: smtp.log.gz
 
 outputs:
   - name: stdout
diff --git a/ztests/suite/zqd/recruiter-worker.sh b/ppl/zqd/ztests/recruiter-worker.sh
similarity index 100%
rename from ztests/suite/zqd/recruiter-worker.sh
rename to ppl/zqd/ztests/recruiter-worker.sh
diff --git a/ztests/suite/zqd/recruiter-worker.yaml b/ppl/zqd/ztests/recruiter-worker.yaml
similarity index 100%
rename from ztests/suite/zqd/recruiter-worker.yaml
rename to ppl/zqd/ztests/recruiter-worker.yaml
diff --git a/ztests/suite/zqd/s3/info.yaml b/ppl/zqd/ztests/s3/info.yaml
similarity index 91%
rename from ztests/suite/zqd/s3/info.yaml
rename to ppl/zqd/ztests/s3/info.yaml
index f25fb53ab..01f6757ae 100644
--- a/ztests/suite/zqd/s3/info.yaml
+++ b/ppl/zqd/ztests/s3/info.yaml
@@ -7,7 +7,7 @@ script: |
 
 inputs:
   - name: babble.tzng
-    source: ../../data/babble.tzng
+    source: ../../../../ztests/suite/data/babble.tzng
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/s3/invalid.yaml b/ppl/zqd/ztests/s3/invalid.yaml
similarity index 85%
rename from ztests/suite/zqd/s3/invalid.yaml
rename to ppl/zqd/ztests/s3/invalid.yaml
index 10d632f2e..df349c01a 100644
--- a/ztests/suite/zqd/s3/invalid.yaml
+++ b/ppl/zqd/ztests/s3/invalid.yaml
@@ -8,9 +8,9 @@ inputs:
   - name: services.sh
     source: ../services.sh
   - name: babble.tzng
-    source: ../../data/babble.tzng
+    source: ../../../../ztests/suite/data/babble.tzng
   - name: ng.pcap
-    source: ../../pcap/ng.pcap
+    source: ../../../../ztests/suite/pcap/ng.pcap
 
 outputs:
   - name: stderr
diff --git a/ztests/suite/zqd/s3/ls.yaml b/ppl/zqd/ztests/s3/ls.yaml
similarity index 100%
rename from ztests/suite/zqd/s3/ls.yaml
rename to ppl/zqd/ztests/s3/ls.yaml
diff --git a/ztests/suite/zqd/s3/postpath-s3.yaml b/ppl/zqd/ztests/s3/postpath-s3.yaml
similarity index 87%
rename from ztests/suite/zqd/s3/postpath-s3.yaml
rename to ppl/zqd/ztests/s3/postpath-s3.yaml
index cfe086519..a50d471f4 100644
--- a/ztests/suite/zqd/s3/postpath-s3.yaml
+++ b/ppl/zqd/ztests/s3/postpath-s3.yaml
@@ -7,7 +7,7 @@ script: |
 
 inputs:
   - name: babble.tzng
-    source: ../../data/babble.tzng
+    source: ../../../../ztests/suite/data/babble.tzng
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/s3/rm.yaml b/ppl/zqd/ztests/s3/rm.yaml
similarity index 100%
rename from ztests/suite/zqd/s3/rm.yaml
rename to ppl/zqd/ztests/s3/rm.yaml
diff --git a/ztests/suite/zqd/s3/s3root.yaml b/ppl/zqd/ztests/s3/s3root.yaml
similarity index 93%
rename from ztests/suite/zqd/s3/s3root.yaml
rename to ppl/zqd/ztests/s3/s3root.yaml
index a3c97e39a..fc8bb3acd 100644
--- a/ztests/suite/zqd/s3/s3root.yaml
+++ b/ppl/zqd/ztests/s3/s3root.yaml
@@ -9,7 +9,7 @@ script: |
 
 inputs:
   - name: babble.tzng
-    source: ../../data/babble.tzng
+    source: ../../../../ztests/suite/data/babble.tzng
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/s3/search.yaml b/ppl/zqd/ztests/s3/search.yaml
similarity index 90%
rename from ztests/suite/zqd/s3/search.yaml
rename to ppl/zqd/ztests/s3/search.yaml
index 4971a9830..ce5ee5898 100644
--- a/ztests/suite/zqd/s3/search.yaml
+++ b/ppl/zqd/ztests/s3/search.yaml
@@ -7,7 +7,7 @@ script: |
 
 inputs:
   - name: babble.tzng
-    source: ../../data/babble.tzng
+    source: ../../../../ztests/suite/data/babble.tzng
   - name: services.sh
     source: ../services.sh
 
diff --git a/ztests/suite/zqd/services.sh b/ppl/zqd/ztests/services.sh
similarity index 100%
rename from ztests/suite/zqd/services.sh
rename to ppl/zqd/ztests/services.sh
diff --git a/ztests/suite/data/smtp.log.gz b/ppl/zqd/ztests/smtp.log.gz
similarity index 100%
rename from ztests/suite/data/smtp.log.gz
rename to ppl/zqd/ztests/smtp.log.gz
diff --git a/ztests/suite/zqd/version.yaml b/ppl/zqd/ztests/version.yaml
similarity index 100%
rename from ztests/suite/zqd/version.yaml
rename to ppl/zqd/ztests/version.yaml
diff --git a/ztests/suite/zqd/worker-dist.sh b/ppl/zqd/ztests/worker-dist.sh
similarity index 100%
rename from ztests/suite/zqd/worker-dist.sh
rename to ppl/zqd/ztests/worker-dist.sh
diff --git a/ztests/suite/zqd/worker-dist.yaml b/ppl/zqd/ztests/worker-dist.yaml
similarity index 98%
rename from ztests/suite/zqd/worker-dist.yaml
rename to ppl/zqd/ztests/worker-dist.yaml
index 1fa09dd23..35c2e06db 100644
--- a/ztests/suite/zqd/worker-dist.yaml
+++ b/ppl/zqd/ztests/worker-dist.yaml
@@ -49,7 +49,7 @@ inputs:
   - name: worker-dist.sh
     source: worker-dist.sh
   - name: smtp.log.gz
-    source: ../data/smtp.log.gz
+    source: smtp.log.gz
 
 outputs:
   - name: stdout
diff --git a/ztests/suite/zqd/zapi-post-shape.yaml b/ppl/zqd/ztests/zapi-post-shape.yaml
similarity index 100%
rename from ztests/suite/zqd/zapi-post-shape.yaml
rename to ppl/zqd/ztests/zapi-post-shape.yaml
diff --git a/ztests/suite/zqd/zapi-postpath-shape.yaml b/ppl/zqd/ztests/zapi-postpath-shape.yaml
similarity index 100%
rename from ztests/suite/zqd/zapi-postpath-shape.yaml
rename to ppl/zqd/ztests/zapi-postpath-shape.yaml
diff --git a/ztests/suite/zqd/zapi-postpath-warnings.yaml b/ppl/zqd/ztests/zapi-postpath-warnings.yaml
similarity index 100%
rename from ztests/suite/zqd/zapi-postpath-warnings.yaml
rename to ppl/zqd/ztests/zapi-postpath-warnings.yaml
diff --git a/proc/groupby/groupby_test.go b/proc/groupby/groupby_test.go
index 619c2d8dc..b8f74ecbf 100644
--- a/proc/groupby/groupby_test.go
+++ b/proc/groupby/groupby_test.go
@@ -23,7 +23,6 @@ import (
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -271,7 +270,7 @@ func TestGroupbySystem(t *testing.T) {
 }
 
 func compileGroupBy(code string) (*ast.GroupByProc, error) {
-	parsed, err := zql.ParseProc(code)
+	parsed, err := compiler.ParseProc(code)
 	if err != nil {
 		return nil, err
 	}
@@ -486,7 +485,7 @@ func TestGroupbyStreamingSpill(t *testing.T) {
 	}
 
 	runOne := func(inputSortKey string) []string {
-		proc, err := zql.ParseProc("every 1s count() by ip")
+		proc, err := compiler.ParseProc("every 1s count() by ip")
 		assert.NoError(t, err)
 
 		zctx := resolver.NewContext()
diff --git a/proc/proctest/utils.go b/proc/proctest/utils.go
index b16e0112f..b8a9fc610 100644
--- a/proc/proctest/utils.go
+++ b/proc/proctest/utils.go
@@ -19,7 +19,6 @@ import (
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/brimsec/zq/zql"
 	"github.com/stretchr/testify/require"
 )
 
@@ -49,7 +48,7 @@ func CompileTestProc(code string, pctx *proc.Context, parent proc.Interface) (pr
 	// a wildcard filter and then pull out just the proc we care
 	// about below.
 	prog := fmt.Sprintf("* | %s", code)
-	parsed, err := zql.ParseProc(prog)
+	parsed, err := compiler.ParseProc(prog)
 	if err != nil {
 		return nil, err
 	}
diff --git a/zql/zql_test.go b/zql/zql_test.go
index bcbf05cb7..fef385e05 100644
--- a/zql/zql_test.go
+++ b/zql/zql_test.go
@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"github.com/brimsec/zq/ast"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/zql"
 	"github.com/brimsec/zq/ztest"
@@ -48,7 +49,7 @@ func parsePEGjs(z string) ([]byte, error) {
 }
 
 func parseProc(z string) ([]byte, error) {
-	proc, err := zql.ParseProc(z)
+	proc, err := compiler.ParseProc(z)
 	if err != nil {
 		return nil, err
 	}
@@ -118,7 +119,7 @@ func TestInvalid(t *testing.T) {
 // string from inside the AST.
 func parseString(in string) (string, error) {
 	code := fmt.Sprintf("s = \"%s\"", in)
-	tree, err := zql.ParseProc(code)
+	tree, err := compiler.ParseProc(code)
 	if err != nil {
 		return "", err
 	}
diff --git a/ztest/ztest.go b/ztest/ztest.go
index bc25986b4..92a0ae682 100644
--- a/ztest/ztest.go
+++ b/ztest/ztest.go
@@ -128,12 +128,12 @@ import (
 	"unicode/utf8"
 
 	"github.com/brimsec/zq/cli/outputflags"
+	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zqe"
-	"github.com/brimsec/zq/zql"
 	"github.com/pmezard/go-difflib/difflib"
 	"gopkg.in/yaml.v3"
 )
@@ -581,7 +581,7 @@ func runzq(path, ZQL string, outputFlags []string, inputs ...string) (string, st
 		// tests.
 		return outbuf.String(), errbuf.String(), err
 	}
-	proc, err := zql.ParseProc(ZQL)
+	proc, err := compiler.ParseProc(ZQL)
 	if err != nil {
 		return "", "", err
 	}