Repository: brimsec/zq
Pull request #2448 (2021-03-27 18:04:20)

Update Zeek reference shaper to use a map
This PR simplifies the reference shaper for Zeek by leveraging the #2321 changes.

When I started this branch, I came to find that the previous shaper had stopped working due it containing words that became reserved as of the SQL changes in #2392. Therefore I added the necessary escapes in this PR to make it work again. This made me pine a bit for the proposed #2398 changes. I'll put some updates in that issue to remind us of the specifics of what failed here.

Since none of our automation had caught the shaper breakage, I've added a simple ztest here that would have caught it. Its goal is not to exhaustively test shaping for correctness, but just make sure it parses ok and can be minimally exercised.

To test that the generated output is equivalent for a large data set, here's side-by-side shaping of the [NDJSON zq-sample-data](https://github.com/brimsec/zq-sample-data/tree/main/zeek-ndjson) that give the same byte-for-byte output from both the older and new shaping configs, run through `zq` commit 7922fa6.

```
$ git branch
* main
  ref-shapers-use-maps

$ zq -Z -I shaper.zql '| sort ts,_path' ~/work/zq-sample-data/zeek-ndjson/* > /tmp/old.zson
```

```
$ git branch
  main
* ref-shapers-use-maps

$ zq -Z -I shaper.zed '| sort ts,_path' ~/work/zq-sample-data/zeek-ndjson/* > /tmp/new.zson
```

```
$ ls -l /tmp/*.zson
-rw-r--r--  1 phil  wheel  944433226 Mar 26 20:54 /tmp/new.zson
-rw-r--r--  1 phil  wheel  944433226 Mar 26 20:18 /tmp/old.zson

$ diff /tmp/old.zson /tmp/new.zson
[no output... files are the same]
```


Patch:
diff --git a/cli/outputflags/flags.go b/cli/outputflags/flags.go
index c34845284..0e025645a 100644
--- a/cli/outputflags/flags.go
+++ b/cli/outputflags/flags.go
@@ -7,11 +7,11 @@ import (
 	"os"
 
 	"github.com/brimsec/zq/emitter"
+	"github.com/brimsec/zq/pkg/terminal"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/zngio"
 	"github.com/brimsec/zq/zio/zstio"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 type Flags struct {
@@ -84,7 +84,7 @@ func (f *Flags) Init() error {
 	if f.outputFile == "-" {
 		f.outputFile = ""
 	}
-	if f.outputFile == "" && f.Format == "zng" && IsTerminal(os.Stdout) && !f.forceBinary {
+	if f.outputFile == "" && f.Format == "zng" && terminal.IsTerminalFile(os.Stdout) && !f.forceBinary {
 		return errors.New("writing binary zng data to terminal; override with -B or use -z for ZSON.")
 	}
 	return nil
@@ -94,7 +94,7 @@ func (f *Flags) InitWithFormat(format string) error {
 	if f.outputFile == "-" {
 		f.outputFile = ""
 	}
-	if f.outputFile == "" && f.Format == "zng" && IsTerminal(os.Stdout) && !f.forceBinary {
+	if f.outputFile == "" && f.Format == "zng" && terminal.IsTerminalFile(os.Stdout) && !f.forceBinary {
 		return errors.New("writing binary zng data to terminal; override with -B or use -z for ZSON.")
 	}
 	return nil
@@ -118,7 +118,3 @@ func (f *Flags) Open(ctx context.Context) (zbuf.WriteCloser, error) {
 	}
 	return w, nil
 }
-
-func IsTerminal(f *os.File) bool {
-	return terminal.IsTerminal(int(f.Fd()))
-}
diff --git a/cmd/ast/ast.go b/cmd/ast/ast.go
index 9ec34c89e..b2194ee6d 100644
--- a/cmd/ast/ast.go
+++ b/cmd/ast/ast.go
@@ -15,9 +15,9 @@ import (
 	"github.com/brimsec/zq/compiler/ast"
 	"github.com/brimsec/zq/compiler/parser"
 	"github.com/brimsec/zq/field"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zfmt"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 	"github.com/peterh/liner"
 )
 
diff --git a/cmd/microindex/convert/command.go b/cmd/microindex/convert/command.go
index d942770ec..1a9eabb86 100644
--- a/cmd/microindex/convert/command.go
+++ b/cmd/microindex/convert/command.go
@@ -8,11 +8,11 @@ import (
 	"github.com/brimsec/zq/cmd/microindex/root"
 	"github.com/brimsec/zq/field"
 	"github.com/brimsec/zq/microindex"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Convert = &charm.Spec{
diff --git a/cmd/microindex/create/command.go b/cmd/microindex/create/command.go
index 4ec71fb4e..499fe4237 100644
--- a/cmd/microindex/create/command.go
+++ b/cmd/microindex/create/command.go
@@ -9,11 +9,11 @@ import (
 	"github.com/brimsec/zq/expr"
 	"github.com/brimsec/zq/field"
 	"github.com/brimsec/zq/microindex"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Create = &charm.Spec{
diff --git a/cmd/microindex/lookup/command.go b/cmd/microindex/lookup/command.go
index 03c3418af..1d33ae813 100644
--- a/cmd/microindex/lookup/command.go
+++ b/cmd/microindex/lookup/command.go
@@ -9,10 +9,10 @@ import (
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/microindex/root"
 	"github.com/brimsec/zq/microindex"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Lookup = &charm.Spec{
diff --git a/cmd/microindex/root/command.go b/cmd/microindex/root/command.go
index 6adb7dbf2..5977f85ec 100644
--- a/cmd/microindex/root/command.go
+++ b/cmd/microindex/root/command.go
@@ -4,7 +4,7 @@ import (
 	"flag"
 
 	"github.com/brimsec/zq/cli"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var MicroIndex = &charm.Spec{
diff --git a/cmd/microindex/section/command.go b/cmd/microindex/section/command.go
index c185db764..daa3211ae 100644
--- a/cmd/microindex/section/command.go
+++ b/cmd/microindex/section/command.go
@@ -4,15 +4,13 @@ import (
 	"context"
 	"errors"
 	"flag"
-	"os"
 
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/microindex/root"
 	"github.com/brimsec/zq/microindex"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 var Section = &charm.Spec{
@@ -48,10 +46,6 @@ func newCommand(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 	return c, nil
 }
 
-func isTerminal(f *os.File) bool {
-	return terminal.IsTerminal(int(f.Fd()))
-}
-
 func (c *Command) Run(args []string) error {
 	defer c.Cleanup()
 	if err := c.Init(&c.outputFlags); err != nil {
diff --git a/cmd/microindex/seek/command.go b/cmd/microindex/seek/command.go
index 130308a12..b682ca7ef 100644
--- a/cmd/microindex/seek/command.go
+++ b/cmd/microindex/seek/command.go
@@ -10,11 +10,11 @@ import (
 	"github.com/brimsec/zq/expr"
 	"github.com/brimsec/zq/field"
 	"github.com/brimsec/zq/microindex"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/zio/zngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Seek = &charm.Spec{
diff --git a/cmd/pcap/cut/command.go b/cmd/pcap/cut/command.go
index e9b23bba3..a1fa71e11 100644
--- a/cmd/pcap/cut/command.go
+++ b/cmd/pcap/cut/command.go
@@ -12,8 +12,8 @@ import (
 
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap/pcapio"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
-	"github.com/mccanne/charm"
 )
 
 var Cut = &charm.Spec{
diff --git a/cmd/pcap/index/command.go b/cmd/pcap/index/command.go
index e9946b008..1d4615f06 100644
--- a/cmd/pcap/index/command.go
+++ b/cmd/pcap/index/command.go
@@ -10,8 +10,8 @@ import (
 
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
-	"github.com/mccanne/charm"
 )
 
 var Index = &charm.Spec{
diff --git a/cmd/pcap/info/command.go b/cmd/pcap/info/command.go
index 5d648417d..a0c349605 100644
--- a/cmd/pcap/info/command.go
+++ b/cmd/pcap/info/command.go
@@ -11,8 +11,8 @@ import (
 
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap/pcapio"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
-	"github.com/mccanne/charm"
 )
 
 var Info = &charm.Spec{
diff --git a/cmd/pcap/root/command.go b/cmd/pcap/root/command.go
index a00cc6086..1f9af8176 100644
--- a/cmd/pcap/root/command.go
+++ b/cmd/pcap/root/command.go
@@ -5,7 +5,7 @@ import (
 	"log"
 
 	"github.com/brimsec/zq/cli"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Pcap = &charm.Spec{
diff --git a/cmd/pcap/slice/command.go b/cmd/pcap/slice/command.go
index a6b7dd7a7..18d9232eb 100644
--- a/cmd/pcap/slice/command.go
+++ b/cmd/pcap/slice/command.go
@@ -12,9 +12,9 @@ import (
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap"
 	"github.com/brimsec/zq/pcap/pcapio"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/nano"
-	"github.com/mccanne/charm"
 )
 
 var Slice = &charm.Spec{
diff --git a/cmd/pcap/ts/command.go b/cmd/pcap/ts/command.go
index 0cef4ced1..59bff2e2a 100644
--- a/cmd/pcap/ts/command.go
+++ b/cmd/pcap/ts/command.go
@@ -9,8 +9,8 @@ import (
 
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap/pcapio"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
-	"github.com/mccanne/charm"
 )
 
 var Ts = &charm.Spec{
diff --git a/cmd/zapi/cmd/auth/command.go b/cmd/zapi/cmd/auth/command.go
index 28cd6f3c4..aacc63d47 100644
--- a/cmd/zapi/cmd/auth/command.go
+++ b/cmd/zapi/cmd/auth/command.go
@@ -4,7 +4,7 @@ import (
 	"flag"
 
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Auth = &charm.Spec{
diff --git a/cmd/zapi/cmd/auth/login.go b/cmd/zapi/cmd/auth/login.go
index 604b93ac4..8ed48019d 100644
--- a/cmd/zapi/cmd/auth/login.go
+++ b/cmd/zapi/cmd/auth/login.go
@@ -8,7 +8,7 @@ import (
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/cmd/zapi/cmd/auth/devauth"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/pkg/browser"
 )
 
diff --git a/cmd/zapi/cmd/auth/logout.go b/cmd/zapi/cmd/auth/logout.go
index b0c4bd663..5f7f9a58b 100644
--- a/cmd/zapi/cmd/auth/logout.go
+++ b/cmd/zapi/cmd/auth/logout.go
@@ -5,7 +5,7 @@ import (
 	"flag"
 	"fmt"
 
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Logout = &charm.Spec{
diff --git a/cmd/zapi/cmd/auth/method.go b/cmd/zapi/cmd/auth/method.go
index b43a21f0b..b9e522f66 100644
--- a/cmd/zapi/cmd/auth/method.go
+++ b/cmd/zapi/cmd/auth/method.go
@@ -6,7 +6,7 @@ import (
 	"flag"
 	"fmt"
 
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Method = &charm.Spec{
diff --git a/cmd/zapi/cmd/auth/store.go b/cmd/zapi/cmd/auth/store.go
index c2c41a59d..7840a792d 100644
--- a/cmd/zapi/cmd/auth/store.go
+++ b/cmd/zapi/cmd/auth/store.go
@@ -5,7 +5,7 @@ import (
 	"flag"
 	"fmt"
 
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Store = &charm.Spec{
diff --git a/cmd/zapi/cmd/auth/verify.go b/cmd/zapi/cmd/auth/verify.go
index 92bb06d58..bf3ac692b 100644
--- a/cmd/zapi/cmd/auth/verify.go
+++ b/cmd/zapi/cmd/auth/verify.go
@@ -6,7 +6,7 @@ import (
 	"flag"
 	"fmt"
 
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Verify = &charm.Spec{
diff --git a/cmd/zapi/cmd/cli.go b/cmd/zapi/cmd/cli.go
index 9d88ce28a..6b1f06abd 100644
--- a/cmd/zapi/cmd/cli.go
+++ b/cmd/zapi/cmd/cli.go
@@ -13,13 +13,13 @@ import (
 	"github.com/brimsec/zq/api/client"
 	"github.com/brimsec/zq/cli"
 	"github.com/brimsec/zq/cli/outputflags"
+	"github.com/brimsec/zq/pkg/charm"
+	"github.com/brimsec/zq/pkg/terminal"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zson"
 	"github.com/kballard/go-shellquote"
-	"github.com/mccanne/charm"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 var Get *charm.Spec
@@ -56,7 +56,7 @@ func New(f *flag.FlagSet) (charm.Command, error) {
 	}
 
 	// If not a terminal make nofancy on by default.
-	c.NoFancy = !terminal.IsTerminal(int(os.Stdout.Fd()))
+	c.NoFancy = !terminal.IsTerminalFile(os.Stdout)
 
 	defaultHost := "localhost:9867"
 	f.StringVar(&c.Host, "h", defaultHost, "<host[:port]>")
diff --git a/cmd/zapi/cmd/get/get.go b/cmd/zapi/cmd/get/get.go
index 7fdbbf1e2..761341da4 100644
--- a/cmd/zapi/cmd/get/get.go
+++ b/cmd/zapi/cmd/get/get.go
@@ -16,10 +16,10 @@ import (
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/compiler"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/nano"
 	"github.com/brimsec/zq/zbuf"
-	"github.com/mccanne/charm"
 )
 
 var Get = &charm.Spec{
diff --git a/cmd/zapi/cmd/index/create.go b/cmd/zapi/cmd/index/create.go
index 9cc1bfff7..5b6c25f58 100644
--- a/cmd/zapi/cmd/index/create.go
+++ b/cmd/zapi/cmd/index/create.go
@@ -8,7 +8,7 @@ import (
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/compiler"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Create = &charm.Spec{
diff --git a/cmd/zapi/cmd/index/find.go b/cmd/zapi/cmd/index/find.go
index 1933af52c..e304b964c 100644
--- a/cmd/zapi/cmd/index/find.go
+++ b/cmd/zapi/cmd/index/find.go
@@ -7,9 +7,9 @@ import (
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/emitter"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/zbuf"
-	"github.com/mccanne/charm"
 )
 
 var Find = &charm.Spec{
diff --git a/cmd/zapi/cmd/index/index.go b/cmd/zapi/cmd/index/index.go
index a41ec77f6..4689e669a 100644
--- a/cmd/zapi/cmd/index/index.go
+++ b/cmd/zapi/cmd/index/index.go
@@ -4,7 +4,7 @@ import (
 	"flag"
 
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Index = &charm.Spec{
diff --git a/cmd/zapi/cmd/info/info.go b/cmd/zapi/cmd/info/info.go
index 20dc81cab..bc9614c87 100644
--- a/cmd/zapi/cmd/info/info.go
+++ b/cmd/zapi/cmd/info/info.go
@@ -12,8 +12,8 @@ import (
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/cmd/zapi/format"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/nano"
-	"github.com/mccanne/charm"
 )
 
 var Info = &charm.Spec{
diff --git a/cmd/zapi/cmd/info/ls.go b/cmd/zapi/cmd/info/ls.go
index 99e7c4469..548c9cbb3 100644
--- a/cmd/zapi/cmd/info/ls.go
+++ b/cmd/zapi/cmd/info/ls.go
@@ -6,10 +6,10 @@ import (
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zson"
-	"github.com/mccanne/charm"
 )
 
 var Ls = &charm.Spec{
diff --git a/cmd/zapi/cmd/intake/command.go b/cmd/zapi/cmd/intake/command.go
index 1e1e4f966..5089f530f 100644
--- a/cmd/zapi/cmd/intake/command.go
+++ b/cmd/zapi/cmd/intake/command.go
@@ -5,11 +5,11 @@ import (
 
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zqe"
 	"github.com/brimsec/zq/zson"
-	"github.com/mccanne/charm"
 )
 
 var Intake = &charm.Spec{
diff --git a/cmd/zapi/cmd/intake/ls.go b/cmd/zapi/cmd/intake/ls.go
index 0fb9c356f..05228d812 100644
--- a/cmd/zapi/cmd/intake/ls.go
+++ b/cmd/zapi/cmd/intake/ls.go
@@ -5,7 +5,7 @@ import (
 
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Ls = &charm.Spec{
diff --git a/cmd/zapi/cmd/intake/new.go b/cmd/zapi/cmd/intake/new.go
index e11062dfd..b787233dc 100644
--- a/cmd/zapi/cmd/intake/new.go
+++ b/cmd/zapi/cmd/intake/new.go
@@ -7,7 +7,7 @@ import (
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var NewSpec = &charm.Spec{
diff --git a/cmd/zapi/cmd/intake/post.go b/cmd/zapi/cmd/intake/post.go
index 07b8f861d..4c05fb179 100644
--- a/cmd/zapi/cmd/intake/post.go
+++ b/cmd/zapi/cmd/intake/post.go
@@ -7,8 +7,8 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
-	"github.com/mccanne/charm"
 )
 
 var Post = &charm.Spec{
diff --git a/cmd/zapi/cmd/intake/rm.go b/cmd/zapi/cmd/intake/rm.go
index f44057fc8..728f6fadb 100644
--- a/cmd/zapi/cmd/intake/rm.go
+++ b/cmd/zapi/cmd/intake/rm.go
@@ -4,7 +4,7 @@ import (
 	"flag"
 	"fmt"
 
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Rm = &charm.Spec{
diff --git a/cmd/zapi/cmd/intake/update.go b/cmd/zapi/cmd/intake/update.go
index c5bc124c7..082cce249 100644
--- a/cmd/zapi/cmd/intake/update.go
+++ b/cmd/zapi/cmd/intake/update.go
@@ -7,7 +7,7 @@ import (
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Update = &charm.Spec{
diff --git a/cmd/zapi/cmd/new/new.go b/cmd/zapi/cmd/new/new.go
index 2bb189211..5cf95151f 100644
--- a/cmd/zapi/cmd/new/new.go
+++ b/cmd/zapi/cmd/new/new.go
@@ -6,7 +6,7 @@ import (
 	"fmt"
 
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var NewSpec = &charm.Spec{
diff --git a/cmd/zapi/cmd/post/path.go b/cmd/zapi/cmd/post/path.go
index 068016beb..a754d9ff3 100644
--- a/cmd/zapi/cmd/post/path.go
+++ b/cmd/zapi/cmd/post/path.go
@@ -13,9 +13,9 @@ import (
 	"github.com/brimsec/zq/api/client"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/cmd/zapi/format"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/display"
 	"github.com/brimsec/zq/pkg/iosrc"
-	"github.com/mccanne/charm"
 )
 
 var PostPath = &charm.Spec{
diff --git a/cmd/zapi/cmd/post/pcap.go b/cmd/zapi/cmd/post/pcap.go
index faff90e6b..b2ee3e961 100644
--- a/cmd/zapi/cmd/post/pcap.go
+++ b/cmd/zapi/cmd/post/pcap.go
@@ -14,8 +14,8 @@ import (
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/cmd/zapi/format"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/display"
-	"github.com/mccanne/charm"
 )
 
 var PostPcap = &charm.Spec{
diff --git a/cmd/zapi/cmd/post/post.go b/cmd/zapi/cmd/post/post.go
index ceed1d128..372fdbe8a 100644
--- a/cmd/zapi/cmd/post/post.go
+++ b/cmd/zapi/cmd/post/post.go
@@ -11,8 +11,8 @@ import (
 	"github.com/brimsec/zq/api/client"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
 	"github.com/brimsec/zq/cmd/zapi/format"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/display"
-	"github.com/mccanne/charm"
 )
 
 var Post = &charm.Spec{
diff --git a/cmd/zapi/cmd/rename/rename.go b/cmd/zapi/cmd/rename/rename.go
index 39e403638..012d87110 100644
--- a/cmd/zapi/cmd/rename/rename.go
+++ b/cmd/zapi/cmd/rename/rename.go
@@ -7,7 +7,7 @@ import (
 
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Rename = &charm.Spec{
diff --git a/cmd/zapi/cmd/repl/repl.go b/cmd/zapi/cmd/repl/repl.go
index 56a97972d..be8683c4b 100644
--- a/cmd/zapi/cmd/repl/repl.go
+++ b/cmd/zapi/cmd/repl/repl.go
@@ -7,9 +7,9 @@ import (
 
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cmd/zapi/cmd"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/repl"
 	"github.com/brimsec/zq/pkg/units"
-	"github.com/mccanne/charm"
 )
 
 var Repl = &charm.Spec{
diff --git a/cmd/zapi/cmd/rm/rm.go b/cmd/zapi/cmd/rm/rm.go
index 8e61a9742..2cf8d325b 100644
--- a/cmd/zapi/cmd/rm/rm.go
+++ b/cmd/zapi/cmd/rm/rm.go
@@ -6,7 +6,7 @@ import (
 	"fmt"
 
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Rm = &charm.Spec{
diff --git a/cmd/zapi/cmd/version/command.go b/cmd/zapi/cmd/version/command.go
index f7dd7ade6..7b81f0896 100644
--- a/cmd/zapi/cmd/version/command.go
+++ b/cmd/zapi/cmd/version/command.go
@@ -6,7 +6,7 @@ import (
 	"fmt"
 
 	"github.com/brimsec/zq/cmd/zapi/cmd"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Version = &charm.Spec{
diff --git a/cmd/zq/zq.go b/cmd/zq/zq.go
index be40814ff..7d0d04f6d 100644
--- a/cmd/zq/zq.go
+++ b/cmd/zq/zq.go
@@ -13,12 +13,12 @@ import (
 	"github.com/brimsec/zq/cli/procflags"
 	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/rlimit"
 	"github.com/brimsec/zq/pkg/s3io"
 	"github.com/brimsec/zq/pkg/signalctx"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Zq = &charm.Spec{
diff --git a/cmd/zsontypes/zsontypes.go b/cmd/zsontypes/zsontypes.go
index 57b1d16ca..ee54d47e8 100644
--- a/cmd/zsontypes/zsontypes.go
+++ b/cmd/zsontypes/zsontypes.go
@@ -5,11 +5,11 @@ import (
 	"fmt"
 
 	"github.com/brimsec/zq/cli/inputflags"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zio/ndjsonio/compat"
 	"github.com/brimsec/zq/zio/tzngio"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var ZsonTypes = &charm.Spec{
diff --git a/cmd/zst/create/command.go b/cmd/zst/create/command.go
index 75e4687f5..03a1307a5 100644
--- a/cmd/zst/create/command.go
+++ b/cmd/zst/create/command.go
@@ -8,9 +8,9 @@ import (
 	"github.com/brimsec/zq/cli/inputflags"
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zst/root"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Create = &charm.Spec{
diff --git a/cmd/zst/cut/command.go b/cmd/zst/cut/command.go
index f58ebe8c9..fc9533fa1 100644
--- a/cmd/zst/cut/command.go
+++ b/cmd/zst/cut/command.go
@@ -8,10 +8,10 @@ import (
 
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zst/root"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zst"
-	"github.com/mccanne/charm"
 )
 
 var Cut = &charm.Spec{
diff --git a/cmd/zst/inspect/command.go b/cmd/zst/inspect/command.go
index 8f534e4d3..d46ab01a6 100644
--- a/cmd/zst/inspect/command.go
+++ b/cmd/zst/inspect/command.go
@@ -7,10 +7,10 @@ import (
 
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zst/root"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zst"
-	"github.com/mccanne/charm"
 )
 
 var Inspect = &charm.Spec{
diff --git a/cmd/zst/read/command.go b/cmd/zst/read/command.go
index 7d6bef62e..4e1e53f5b 100644
--- a/cmd/zst/read/command.go
+++ b/cmd/zst/read/command.go
@@ -4,15 +4,13 @@ import (
 	"context"
 	"errors"
 	"flag"
-	"os"
 
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/cmd/zst/root"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
 	"github.com/brimsec/zq/zst"
-	"github.com/mccanne/charm"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 var Read = &charm.Spec{
@@ -45,10 +43,6 @@ func newCommand(parent charm.Command, f *flag.FlagSet) (charm.Command, error) {
 	return c, nil
 }
 
-func isTerminal(f *os.File) bool {
-	return terminal.IsTerminal(int(f.Fd()))
-}
-
 func (c *Command) Run(args []string) error {
 	defer c.Cleanup()
 	if err := c.Init(&c.outputFlags); err != nil {
diff --git a/cmd/zst/root/command.go b/cmd/zst/root/command.go
index a59e6d49b..7fb7d1211 100644
--- a/cmd/zst/root/command.go
+++ b/cmd/zst/root/command.go
@@ -4,7 +4,7 @@ import (
 	"flag"
 
 	"github.com/brimsec/zq/cli"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Zst = &charm.Spec{
diff --git a/emitter/file.go b/emitter/file.go
index c8d027fe7..9011c129e 100644
--- a/emitter/file.go
+++ b/emitter/file.go
@@ -7,11 +7,11 @@ import (
 
 	"github.com/brimsec/zq/pkg/bufwriter"
 	"github.com/brimsec/zq/pkg/iosrc"
+	"github.com/brimsec/zq/pkg/terminal"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 func NewFile(ctx context.Context, path string, opts zio.WriterOpts) (zbuf.WriteCloser, error) {
@@ -31,9 +31,7 @@ func NewFile(ctx context.Context, path string, opts zio.WriterOpts) (zbuf.WriteC
 
 func IsTerminal(w io.Writer) bool {
 	if f, ok := w.(*os.File); ok {
-		if terminal.IsTerminal(int(f.Fd())) {
-			return true
-		}
+		return terminal.IsTerminalFile(f)
 	}
 	return false
 }
diff --git a/go.mod b/go.mod
index db5ae456a..332f9ed3c 100644
--- a/go.mod
+++ b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/gosuri/uilive v0.0.4
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
-	github.com/mccanne/charm v0.0.3-0.20191224190439-b05e1b7b1be3
+	github.com/kr/text v0.2.0
 	github.com/mitchellh/mapstructure v1.3.3
 	github.com/pbnjay/memory v0.0.0-20190104145345-974d429e7ae4
 	github.com/peterh/liner v1.1.0
@@ -38,9 +38,9 @@ require (
 	go.temporal.io/server v1.6.3
 	go.uber.org/multierr v1.6.0
 	go.uber.org/zap v1.16.0
-	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
 	golang.org/x/sys v0.0.0-20210105210732-16f7687f5001
+	golang.org/x/term v0.0.0-20210317153231-de623e64d2a6
 	golang.org/x/text v0.3.4
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.0-20210106172901-c476de37821d
diff --git a/go.sum b/go.sum
index 419299e01..99163d864 100644
--- a/go.sum
+++ b/go.sum
@@ -385,8 +385,6 @@ github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOq
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mccanne/charm v0.0.3-0.20191224190439-b05e1b7b1be3 h1:Sj9zugMeJDXrKpOZgiQXhm0Z87vPF2QfUMWE41lwcRQ=
-github.com/mccanne/charm v0.0.3-0.20191224190439-b05e1b7b1be3/go.mod h1:R47PUgWIRA4UrwFWKI6vgAEWHXPYWUg3iAtyXB8yKNU=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -608,7 +606,6 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
@@ -746,7 +743,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -779,8 +775,9 @@ golang.org/x/sys v0.0.0-20201221093633-bc327ba9c2f0/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210105210732-16f7687f5001 h1:/dSxr6gT0FNI1MO5WLJo8mTmItROeOKTkDn+7OwWBos=
 golang.org/x/sys v0.0.0-20210105210732-16f7687f5001/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210317153231-de623e64d2a6 h1:EC6+IGYTjPpRfv9a2b/6Puw0W+hLtAhkV1tPsXhutqs=
+golang.org/x/term v0.0.0-20210317153231-de623e64d2a6/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/pkg/charm/charm.go b/pkg/charm/charm.go
new file mode 100644
index 000000000..680ef5aaf
--- /dev/null
+++ b/pkg/charm/charm.go
@@ -0,0 +1,91 @@
+// Package charm is minimilast CLI framework inspired by cobra and urfave/cli.
+package charm
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"strings"
+)
+
+var ErrNoRun = errors.New("no run method")
+
+type Constructor func(Command, *flag.FlagSet) (Command, error)
+
+type Command interface {
+	Run([]string) error
+}
+
+type Spec struct {
+	Name          string
+	Usage         string
+	Short         string
+	Long          string
+	New           Constructor
+	Hidden        bool
+	HiddenFlags   string
+	RedactedFlags string
+	Empty         *Spec
+	children      []*Spec
+	parent        *Spec
+}
+
+func (c *Spec) Add(child *Spec) {
+	c.children = append(c.children, child)
+	child.parent = c
+}
+
+func (c *Spec) lookupSub(name string) *Spec {
+	for _, child := range c.children {
+		if name == child.Name {
+			return child
+		}
+	}
+	return nil
+}
+
+func parseFlags(flags *flag.FlagSet, args []string) ([]string, error) {
+	var usage bool
+	flags.Usage = func() {
+		usage = true
+	}
+	if err := flags.Parse(args); err != nil {
+		if usage {
+			s := strings.Join(args, " ")
+			err = fmt.Errorf("unknown flag: \"%s\"", s)
+		}
+		return nil, err
+	}
+	return flags.Args(), nil
+}
+
+func (s *Spec) Exec(parent Command, args []string) error {
+	cmd, err := newInstance(parent, s)
+	if err != nil {
+		return err
+	}
+	return cmd.run(args)
+}
+
+// ExecRoot execute this command spec, which must be a root spec.
+// It returns the root command that was created.
+func (s *Spec) ExecRoot(args []string) (Command, error) {
+	cmd, err := newInstance(nil, s)
+	if err != nil {
+		return nil, err
+	}
+	return cmd.command, cmd.run(args)
+}
+
+//XXX
+func (c *Spec) Prefix() string {
+	return c.Name + ": "
+}
+
+func (c *Spec) Root() *Spec {
+	p := c
+	for p.parent != nil {
+		p = p.parent
+	}
+	return p
+}
diff --git a/pkg/charm/help.go b/pkg/charm/help.go
new file mode 100644
index 000000000..a01d103f3
--- /dev/null
+++ b/pkg/charm/help.go
@@ -0,0 +1,213 @@
+package charm
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/brimsec/zq/pkg/terminal"
+	"github.com/kr/text"
+)
+
+var Help = &Spec{
+	Name:  "help",
+	Usage: "help [command]",
+	Short: "display help for a command",
+	Long: `
+For help on the top-level command just type "help".
+For help on a subcommand, type "help command" where command is the name of
+the command.  For help on command nested further, type "help cmd1 cmd2" and
+so forth.`,
+	HiddenFlags: "v",
+	New: func(parent Command, f *flag.FlagSet) (Command, error) {
+		c := &HelpCommand{}
+		f.BoolVar(&c.vflag, "v", false, "show hidden commands and flags")
+		return c, nil
+	},
+}
+
+type HelpCommand struct {
+	vflag bool
+}
+
+// splitFlags is like strings.Split with a comma and also trims whitespace
+func splitFlags(flags string) []string {
+	var out []string
+	for _, flag := range strings.Split(flags, ",") {
+		out = append(out, strings.TrimSpace(flag))
+	}
+	return out
+}
+
+// flagMap creates a map that maps a name to a boolean based on the existence
+// of that name in the comma-separated string of flags.  Whitespace is removed
+// from each name in the flags list.  A map that contains no entries is returned
+// for an empty string.
+func flagMap(flags string) map[string]bool {
+	hidden := make(map[string]bool)
+	for _, flag := range splitFlags(flags) {
+		hidden[flag] = true
+	}
+	return hidden
+}
+
+func (c *HelpCommand) search(args []string) ([]*instance, error) {
+	//XXX parent of root can be custom...
+	parent, err := newInstance(nil, Help.Root())
+	if err != nil {
+		return nil, err
+	}
+	sequence := ""
+	var instances []*instance
+	instances = append(instances, parent)
+	for _, arg := range args {
+		sequence = sequence + " " + arg
+		subcmd := parent.spec.lookupSub(arg)
+		if subcmd == nil {
+			if len(args) == 1 {
+				return nil, fmt.Errorf("no such command: %s", arg)
+			} else {
+				return nil, fmt.Errorf("no such command:%s", sequence)
+			}
+		}
+		child, err := newInstance(parent.command, subcmd)
+		if err != nil {
+			return nil, err
+		}
+		instances = append(instances, child)
+		parent = child
+	}
+	return instances, nil
+}
+
+func (c *HelpCommand) Prepare(f *flag.FlagSet) {}
+
+func (c *HelpCommand) Run(args []string) error {
+	inst, err := c.search(args)
+	if err != nil {
+		return err
+	}
+	c.help(inst)
+	return nil
+}
+
+func formatParagraph(body, tab string, lineWidth int) string {
+	paragraphs := strings.Split(body, "\n\n")
+	var chunks []string
+	for _, paragraph := range paragraphs {
+		var chunk string
+		if len(paragraph) < lineWidth {
+			chunk = strings.TrimRight(paragraph, " \t\n")
+		} else {
+			paragraph = strings.TrimSpace(paragraph)
+			paragraph = text.Wrap(paragraph, lineWidth)
+			lines := strings.Split(paragraph, "\n")
+			chunk = strings.Join(lines, "\n"+tab)
+		}
+		chunks = append(chunks, chunk)
+	}
+	body = strings.Join(chunks, "\n\n"+tab)
+	body = strings.TrimRight(body, " \t\n")
+	return tab + body + "\n\n"
+}
+
+const tab = "    "
+
+func header(heading string) string {
+	boldOn := "\033[1m"
+	boldOff := "\033[0m"
+	return boldOn + heading + boldOff
+}
+
+func helpItem(heading, body string) {
+	hdr := header(heading)
+	body = hdr + "\n" + tab + body + "\n\n"
+	fmt.Fprint(os.Stderr, body)
+}
+
+func helpDesc(heading, body string) {
+	hdr := header(heading)
+	body = tab + body + "\n\n"
+	w := terminal.Width()
+	lineWidth := w - len(tab) - 5
+	if len(body) > lineWidth {
+		body = formatParagraph(body, tab, lineWidth)
+	}
+	fmt.Fprint(os.Stderr, hdr+"\n"+body)
+}
+
+func helpList(heading string, lines []string) {
+	hdr := header(heading)
+	body := strings.Join(lines, "\n"+tab)
+	body = hdr + "\n" + tab + body + "\n\n"
+	fmt.Fprint(os.Stderr, body)
+}
+
+func optionSection(body []string) []string {
+	if len(body) == 0 {
+		return []string{"no flags for this command"}
+	}
+	return body
+}
+
+func (c *HelpCommand) getCommands(target *Spec, vflag bool) []string {
+	var lines []string
+	for _, cmd := range target.children {
+		name := cmd.Name
+		if cmd.Hidden {
+			if vflag {
+				name = "[" + name + "]"
+			} else {
+				continue
+			}
+		}
+		line := name + " - " + cmd.Short
+		lines = append(lines, line)
+	}
+	return lines
+}
+
+func buildOptions(path []*instance, parentCmd string, vflag bool) []string {
+	n := len(path)
+	if n == 1 {
+		options := path[0].options(vflag)
+		if len(options) == 0 {
+			options = []string{"no flags for this command"}
+		}
+		return options
+	}
+	pathCmd := path[0].spec.Name
+	if parentCmd != "" {
+		pathCmd = parentCmd + " " + pathCmd
+	}
+	childOptions := buildOptions(path[1:], parentCmd, vflag)
+	options := path[0].options(vflag)
+	if len(options) == 0 {
+		return childOptions
+	}
+	// add a line separator then add the command path in parens as the header
+	// for the next set of flags
+	childOptions = append(childOptions, "")
+	childOptions = append(childOptions, "["+pathCmd+" flags]")
+	for _, option := range options {
+		childOptions = append(childOptions, option)
+	}
+	return childOptions
+}
+
+func optionsSection(path []*instance, vflag bool) []string {
+	return buildOptions(path, "", vflag)
+}
+
+func (c *HelpCommand) help(path []*instance) {
+	spec := path[len(path)-1].spec
+	name := spec.Name + " - " + spec.Short
+	helpItem("NAME", name)
+	helpDesc("USAGE", spec.Usage)
+	helpList("OPTIONS", optionsSection(path, c.vflag))
+	if len(spec.children) > 0 {
+		helpList("COMMANDS", c.getCommands(spec, c.vflag))
+	}
+	helpDesc("DESCRIPTION", spec.Long)
+}
diff --git a/pkg/charm/instance.go b/pkg/charm/instance.go
new file mode 100644
index 000000000..c4ffab961
--- /dev/null
+++ b/pkg/charm/instance.go
@@ -0,0 +1,88 @@
+package charm
+
+import (
+	"flag"
+	"fmt"
+)
+
+// instance represents a command that has been created but not run.
+// It's options and defaults may be queried with the options method and
+// the command can be run with the run method.
+type instance struct {
+	spec    *Spec
+	command Command
+	flags   *flag.FlagSet
+}
+
+func newInstance(parent Command, spec *Spec) (*instance, error) {
+	if spec.New == nil {
+		return nil, fmt.Errorf("command '%s': New function is nil", spec.Name)
+	}
+	flags := flag.NewFlagSet(spec.Name, flag.ContinueOnError)
+	cmd, err := spec.New(parent, flags)
+	if err != nil {
+		return nil, err
+	}
+	return &instance{spec, cmd, flags}, nil
+}
+
+// run runs this instance of the command.
+func (i *instance) run(args []string) error {
+	// set up the flags even if there aren't any args to parse
+	// since we need to establish default state in the command objects
+	rest, err := parseFlags(i.flags, args)
+	if err != nil {
+		return err
+	}
+	if len(rest) == 0 {
+		empty := i.spec.Empty
+		if empty == nil {
+			// call current command with no args
+			err = i.command.Run(rest)
+			if err == ErrNoRun {
+				err = fmt.Errorf("%s: no sub-command supplied", i.spec.Name)
+			}
+			return err
+		}
+		// Otherwise, this is an empty sub-command that has an empty spec,
+		// so invoke the spec with the current command as the parent.
+		return empty.Exec(i.command, rest)
+	}
+	// otherwise there are more stuff after the flags so
+	// look for a subcommand
+	child := i.spec.lookupSub(rest[0])
+	if child == nil {
+		// no command found so call the current command with the args
+		err = i.command.Run(rest)
+		if err == ErrNoRun {
+			err = fmt.Errorf("%s: no such sub-command: %s", i.spec.Name, rest[0])
+		}
+		return err
+	}
+	// we found a subcommand so execute it recursively and
+	// drop the cmd name from the args
+	return child.Exec(i.command, rest[1:])
+}
+
+// options returns a formatted slice of strings ready for printing as
+// help for this instance of a command.
+func (i *instance) options(vflag bool) []string {
+	hidden := flagMap(i.spec.HiddenFlags)
+	redacted := flagMap(i.spec.RedactedFlags)
+	var body []string
+	i.flags.VisitAll(func(f *flag.Flag) {
+		name := "-" + f.Name
+		if hidden[f.Name] {
+			if !vflag {
+				return
+			}
+			name = "[" + name + "]"
+		}
+		line := name + " " + f.Usage
+		if f.DefValue != "" && !redacted[f.Name] {
+			line = fmt.Sprintf("%s (default \"%s\")", line, f.DefValue)
+		}
+		body = append(body, line)
+	})
+	return body
+}
diff --git a/pkg/terminal/terminal.go b/pkg/terminal/terminal.go
new file mode 100644
index 000000000..4bb5aa47b
--- /dev/null
+++ b/pkg/terminal/terminal.go
@@ -0,0 +1,22 @@
+package terminal
+
+import (
+	"os"
+
+	"golang.org/x/term"
+)
+
+func IsTerminalFile(f *os.File) bool {
+	return term.IsTerminal(int(f.Fd()))
+}
+
+// Width returns the width in columns of the terminal that is the standard
+// input.  Width returns 80 if the standard input is not a terminal or its size
+// cannot be determined.
+func Width() int {
+	width, _, err := term.GetSize(int(os.Stdin.Fd()))
+	if err != nil {
+		return 80
+	}
+	return width
+}
diff --git a/ppl/cmd/gentoken/gentoken.go b/ppl/cmd/gentoken/gentoken.go
index 3f9f72fe0..211dd56f9 100644
--- a/ppl/cmd/gentoken/gentoken.go
+++ b/ppl/cmd/gentoken/gentoken.go
@@ -7,8 +7,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/zqd/auth"
-	"github.com/mccanne/charm"
 )
 
 var CLI = &charm.Spec{
diff --git a/ppl/cmd/pgctl/migrate/command.go b/ppl/cmd/pgctl/migrate/command.go
index b7fad50bf..2235e327f 100644
--- a/ppl/cmd/pgctl/migrate/command.go
+++ b/ppl/cmd/pgctl/migrate/command.go
@@ -5,13 +5,13 @@ import (
 	"flag"
 	"fmt"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
 	"github.com/brimsec/zq/ppl/cmd/pgctl/root"
 	"github.com/brimsec/zq/ppl/zqd/db/postgresdb"
 	"github.com/golang-migrate/migrate/v4"
 	_ "github.com/golang-migrate/migrate/v4/database/postgres"
 	_ "github.com/golang-migrate/migrate/v4/source/file"
-	"github.com/mccanne/charm"
 )
 
 var Migrate = &charm.Spec{
diff --git a/ppl/cmd/pgctl/root/command.go b/ppl/cmd/pgctl/root/command.go
index c8b6fa9cf..2dc298c9e 100644
--- a/ppl/cmd/pgctl/root/command.go
+++ b/ppl/cmd/pgctl/root/command.go
@@ -4,7 +4,7 @@ import (
 	"flag"
 
 	"github.com/brimsec/zq/cli"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var CLI = &charm.Spec{
diff --git a/ppl/cmd/zar/compact/command.go b/ppl/cmd/zar/compact/command.go
index b48aa53dc..d159045b4 100644
--- a/ppl/cmd/zar/compact/command.go
+++ b/ppl/cmd/zar/compact/command.go
@@ -5,9 +5,9 @@ import (
 	"flag"
 	"os"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
-	"github.com/mccanne/charm"
 )
 
 var Compact = &charm.Spec{
diff --git a/ppl/cmd/zar/find/command.go b/ppl/cmd/zar/find/command.go
index 66b66e4db..91c8dc478 100644
--- a/ppl/cmd/zar/find/command.go
+++ b/ppl/cmd/zar/find/command.go
@@ -8,12 +8,12 @@ import (
 
 	"github.com/brimsec/zq/cli/outputflags"
 	"github.com/brimsec/zq/emitter"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/ppl/lake/index"
 	"github.com/brimsec/zq/zng"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Find = &charm.Spec{
diff --git a/ppl/cmd/zar/import/command.go b/ppl/cmd/zar/import/command.go
index 8daa582ea..5a98a2222 100644
--- a/ppl/cmd/zar/import/command.go
+++ b/ppl/cmd/zar/import/command.go
@@ -7,6 +7,7 @@ import (
 
 	"github.com/brimsec/zq/cli/inputflags"
 	"github.com/brimsec/zq/cli/procflags"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/rlimit"
 	"github.com/brimsec/zq/pkg/signalctx"
 	"github.com/brimsec/zq/pkg/units"
@@ -14,7 +15,6 @@ import (
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Import = &charm.Spec{
diff --git a/ppl/cmd/zar/index/command.go b/ppl/cmd/zar/index/command.go
index b8489cb02..f620e850e 100644
--- a/ppl/cmd/zar/index/command.go
+++ b/ppl/cmd/zar/index/command.go
@@ -3,8 +3,8 @@ package index
 import (
 	"flag"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
-	"github.com/mccanne/charm"
 )
 
 var Index = &charm.Spec{
diff --git a/ppl/cmd/zar/index/create.go b/ppl/cmd/zar/index/create.go
index 9cb384763..92eaa1f56 100644
--- a/ppl/cmd/zar/index/create.go
+++ b/ppl/cmd/zar/index/create.go
@@ -10,12 +10,12 @@ import (
 
 	"github.com/brimsec/zq/cli/procflags"
 	"github.com/brimsec/zq/field"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/rlimit"
 	"github.com/brimsec/zq/pkg/signalctx"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/ppl/lake/index"
-	"github.com/mccanne/charm"
 )
 
 var Create = &charm.Spec{
diff --git a/ppl/cmd/zar/index/drop.go b/ppl/cmd/zar/index/drop.go
index 6afb6f9d9..44c5650a4 100644
--- a/ppl/cmd/zar/index/drop.go
+++ b/ppl/cmd/zar/index/drop.go
@@ -7,10 +7,10 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/ppl/lake/index"
-	"github.com/mccanne/charm"
 	"github.com/segmentio/ksuid"
 )
 
diff --git a/ppl/cmd/zar/index/ls.go b/ppl/cmd/zar/index/ls.go
index 7c00aefbd..8deb7c908 100644
--- a/ppl/cmd/zar/index/ls.go
+++ b/ppl/cmd/zar/index/ls.go
@@ -9,11 +9,11 @@ import (
 	"strings"
 
 	"github.com/brimsec/zq/cli/outputflags"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/ppl/lake/index"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 	"github.com/segmentio/ksuid"
 )
 
diff --git a/ppl/cmd/zar/ls/command.go b/ppl/cmd/zar/ls/command.go
index 1106ad692..eecaeaeda 100644
--- a/ppl/cmd/zar/ls/command.go
+++ b/ppl/cmd/zar/ls/command.go
@@ -8,12 +8,12 @@ import (
 	"os"
 	"strings"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/nano"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/ppl/lake/chunk"
 	"github.com/brimsec/zq/ppl/lake/index"
-	"github.com/mccanne/charm"
 )
 
 var Ls = &charm.Spec{
diff --git a/ppl/cmd/zar/map/command.go b/ppl/cmd/zar/map/command.go
index e93002ac0..afd246bb5 100644
--- a/ppl/cmd/zar/map/command.go
+++ b/ppl/cmd/zar/map/command.go
@@ -11,6 +11,7 @@ import (
 	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
 	"github.com/brimsec/zq/emitter"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
 	"github.com/brimsec/zq/pkg/rlimit"
 	"github.com/brimsec/zq/pkg/signalctx"
@@ -21,7 +22,6 @@ import (
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/detector"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Map = &charm.Spec{
diff --git a/ppl/cmd/zar/rm/command.go b/ppl/cmd/zar/rm/command.go
index 19ca0d95f..1185a77f7 100644
--- a/ppl/cmd/zar/rm/command.go
+++ b/ppl/cmd/zar/rm/command.go
@@ -9,12 +9,12 @@ import (
 	"path"
 	"strings"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/ppl/lake/chunk"
 	"github.com/brimsec/zq/zqe"
-	"github.com/mccanne/charm"
 )
 
 var Rm = &charm.Spec{
diff --git a/ppl/cmd/zar/rmdirs/command.go b/ppl/cmd/zar/rmdirs/command.go
index d1b29fc9a..c557ac293 100644
--- a/ppl/cmd/zar/rmdirs/command.go
+++ b/ppl/cmd/zar/rmdirs/command.go
@@ -5,9 +5,9 @@ import (
 	"flag"
 	"os"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
-	"github.com/mccanne/charm"
 )
 
 var RmDirs = &charm.Spec{
diff --git a/ppl/cmd/zar/root/command.go b/ppl/cmd/zar/root/command.go
index 6d50d2873..e41fe069f 100644
--- a/ppl/cmd/zar/root/command.go
+++ b/ppl/cmd/zar/root/command.go
@@ -4,7 +4,7 @@ import (
 	"flag"
 
 	"github.com/brimsec/zq/cli"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Zar = &charm.Spec{
diff --git a/ppl/cmd/zar/stat/command.go b/ppl/cmd/zar/stat/command.go
index 45b1c1f58..80b92f265 100644
--- a/ppl/cmd/zar/stat/command.go
+++ b/ppl/cmd/zar/stat/command.go
@@ -7,12 +7,12 @@ import (
 	"os"
 
 	"github.com/brimsec/zq/emitter"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Stat = &charm.Spec{
diff --git a/ppl/cmd/zar/zq/command.go b/ppl/cmd/zar/zq/command.go
index 329d2f13e..4dcdb5a84 100644
--- a/ppl/cmd/zar/zq/command.go
+++ b/ppl/cmd/zar/zq/command.go
@@ -11,12 +11,12 @@ import (
 	"github.com/brimsec/zq/cli/searchflags"
 	"github.com/brimsec/zq/compiler"
 	"github.com/brimsec/zq/driver"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/rlimit"
 	"github.com/brimsec/zq/pkg/signalctx"
 	"github.com/brimsec/zq/ppl/cmd/zar/root"
 	"github.com/brimsec/zq/ppl/lake"
 	"github.com/brimsec/zq/zng/resolver"
-	"github.com/mccanne/charm"
 )
 
 var Zq = &charm.Spec{
diff --git a/ppl/cmd/zqd/listen/command.go b/ppl/cmd/zqd/listen/command.go
index 09448c11b..4071c7c90 100644
--- a/ppl/cmd/zqd/listen/command.go
+++ b/ppl/cmd/zqd/listen/command.go
@@ -15,6 +15,7 @@ import (
 
 	"github.com/brimsec/zq/api"
 	"github.com/brimsec/zq/cli"
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/httpd"
 	"github.com/brimsec/zq/pkg/rlimit"
@@ -23,7 +24,6 @@ import (
 	"github.com/brimsec/zq/ppl/zqd"
 	"github.com/brimsec/zq/ppl/zqd/pcapanalyzer"
 	"github.com/brimsec/zq/proc/sort"
-	"github.com/mccanne/charm"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"gopkg.in/yaml.v3"
diff --git a/ppl/cmd/zqd/root/command.go b/ppl/cmd/zqd/root/command.go
index fa645f7a0..f4ed52a69 100644
--- a/ppl/cmd/zqd/root/command.go
+++ b/ppl/cmd/zqd/root/command.go
@@ -7,7 +7,7 @@ import (
 
 	"github.com/brimsec/zq/cli"
 	"github.com/brimsec/zq/cli/procflags"
-	"github.com/mccanne/charm"
+	"github.com/brimsec/zq/pkg/charm"
 )
 
 var Zqd = &charm.Spec{
diff --git a/ppl/cmd/zqd/winexec/command.go b/ppl/cmd/zqd/winexec/command.go
index 9245daf66..82d7b511f 100644
--- a/ppl/cmd/zqd/winexec/command.go
+++ b/ppl/cmd/zqd/winexec/command.go
@@ -3,8 +3,8 @@ package winexec
 import (
 	"flag"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/zqd/root"
-	"github.com/mccanne/charm"
 )
 
 func init() {
diff --git a/proc/apply.go b/proc/apply.go
index 6ec394fd3..980abb06a 100644
--- a/proc/apply.go
+++ b/proc/apply.go
@@ -17,6 +17,7 @@ type applier struct {
 	pctx     *Context
 	parent   Interface
 	function Function
+	warned   map[string]bool
 }
 
 func FromFunction(pctx *Context, parent Interface, f Function) *applier {
@@ -24,12 +25,7 @@ func FromFunction(pctx *Context, parent Interface, f Function) *applier {
 		pctx:     pctx,
 		parent:   parent,
 		function: f,
-	}
-}
-
-func (a *applier) warn() {
-	if s := a.function.Warning(); s != "" {
-		a.pctx.Warnings <- fmt.Sprintf("%s: %s", a.function, s)
+		warned:   map[string]bool{},
 	}
 }
 
@@ -37,18 +33,19 @@ func (a *applier) Pull() (zbuf.Batch, error) {
 	for {
 		batch, err := a.parent.Pull()
 		if EOS(batch, err) {
-			a.warn()
+			if s := a.function.Warning(); s != "" {
+				a.maybeWarn(s)
+			}
 			return nil, err
 		}
 		recs := make([]*zng.Record, 0, batch.Length())
 		for k := 0; k < batch.Length(); k++ {
 			in := batch.Index(k)
-
 			out, err := a.function.Apply(in)
 			if err != nil {
-				return nil, err
+				a.maybeWarn(err.Error())
+				continue
 			}
-
 			if out != nil {
 				recs = append(recs, out)
 			}
@@ -60,6 +57,13 @@ func (a *applier) Pull() (zbuf.Batch, error) {
 	}
 }
 
+func (a *applier) maybeWarn(s string) {
+	if !a.warned[s] {
+		a.pctx.Warnings <- fmt.Sprintf("%s: %s", a.function, s)
+		a.warned[s] = true
+	}
+}
+
 func (a *applier) Done() {
 	a.parent.Done()
 }
diff --git a/proc/rename/renamer.go b/proc/rename/renamer.go
index 2f0515860..66bd149db 100644
--- a/proc/rename/renamer.go
+++ b/proc/rename/renamer.go
@@ -5,8 +5,6 @@
 package rename
 
 import (
-	"fmt"
-
 	"github.com/brimsec/zq/field"
 	"github.com/brimsec/zq/proc"
 	"github.com/brimsec/zq/zng"
@@ -69,7 +67,7 @@ func (r *Function) Apply(in *zng.Record) (*zng.Record, error) {
 	if _, ok := r.typeMap[id]; !ok {
 		typ, err := r.computeType(zng.TypeRecordOf(in.Type))
 		if err != nil {
-			return nil, fmt.Errorf("%s: %w", r, err)
+			return nil, err
 		}
 		r.typeMap[id] = typ
 	}
diff --git a/zeek/shaper.zql b/zeek/shaper.zed
similarity index 78%
rename from zeek/shaper.zql
rename to zeek/shaper.zed
index 1f7f285b7..52558d06d 100644
--- a/zeek/shaper.zql
+++ b/zeek/shaper.zed
@@ -15,9 +15,9 @@ type dpd={_path:string,ts:time,uid:bstring,id:id,proto:zenum,analyzer:bstring,fa
 type files={_path:string,ts:time,fuid:bstring,tx_hosts:|[ip]|,rx_hosts:|[ip]|,conn_uids:|[bstring]|,source:bstring,depth:uint64,analyzers:|[bstring]|,mime_type:bstring,filename:bstring,duration:duration,local_orig:bool,is_orig:bool,seen_bytes:uint64,total_bytes:uint64,missing_bytes:uint64,overflow_bytes:uint64,timedout:bool,parent_fuid:bstring,md5:bstring,sha1:bstring,sha256:bstring,extracted:bstring,extracted_cutoff:bool,extracted_size:uint64,_write_ts:time};
 type ftp={_path:string,ts:time,uid:bstring,id:id,user:bstring,password:bstring,command:bstring,arg:bstring,mime_type:bstring,file_size:uint64,reply_code:uint64,reply_msg:bstring,data_channel:{passive:bool,orig_h:ip,resp_h:ip,resp_p:port=(uint16)},fuid:bstring,_write_ts:time};
 type http={_path:string,ts:time,uid:bstring,id:id,trans_depth:uint64,method:bstring,host:bstring,uri:bstring,referrer:bstring,version:bstring,user_agent:bstring,origin:bstring,request_body_len:uint64,response_body_len:uint64,status_code:uint64,status_msg:bstring,info_code:uint64,info_msg:bstring,tags:|[zenum]|,username:bstring,password:bstring,proxied:|[bstring]|,orig_fuids:[bstring],orig_filenames:[bstring],orig_mime_types:[bstring],resp_fuids:[bstring],resp_filenames:[bstring],resp_mime_types:[bstring],_write_ts:time};
-type intel={_path:string,ts:time,uid:bstring,id:id,seen:{indicator:bstring,indicator_type:zenum,where:zenum,node:bstring},matched:|[zenum]|,sources:|[bstring]|,fuid:bstring,file_mime_type:bstring,file_desc:bstring,_write_ts:time};
+type intel={_path:string,ts:time,uid:bstring,id:id,seen:{indicator:bstring,indicator_type:zenum,\where:zenum,node:bstring},matched:|[zenum]|,sources:|[bstring]|,fuid:bstring,file_mime_type:bstring,file_desc:bstring,_write_ts:time};
 type irc={_path:string,ts:time,uid:bstring,id:id,nick:bstring,user:bstring,command:bstring,value:bstring,addl:bstring,dcc_file_name:bstring,dcc_file_size:uint64,dcc_mime_type:bstring,fuid:bstring,_write_ts:time};
-type kerberos={_path:string,ts:time,uid:bstring,id:id,request_type:bstring,client:bstring,service:bstring,success:bool,error_msg:bstring,from:time,till:time,cipher:bstring,forwardable:bool,renewable:bool,client_cert_subject:bstring,client_cert_fuid:bstring,server_cert_subject:bstring,server_cert_fuid:bstring,_write_ts:time};
+type kerberos={_path:string,ts:time,uid:bstring,id:id,request_type:bstring,client:bstring,service:bstring,success:bool,error_msg:bstring,\from:time,till:time,cipher:bstring,forwardable:bool,renewable:bool,client_cert_subject:bstring,client_cert_fuid:bstring,server_cert_subject:bstring,server_cert_fuid:bstring,_write_ts:time};
 type known_certs={_path:string,ts:time,host:ip,port_num:port=(uint16),subject:bstring,issuer_subject:bstring,serial:bstring,_write_ts:time};
 type known_hosts={_path:string,ts:time,host:ip,_write_ts:time};
 type known_services={_path:string,ts:time,host:ip,port_num:port=(uint16),port_proto:zenum,service:|[bstring]|,_write_ts:time};
@@ -41,7 +41,7 @@ type signatures={_path:string,ts:time,uid:bstring,src_addr:ip,src_port:port=(uin
 type sip={_path:string,ts:time,uid:bstring,id:id,trans_depth:uint64,method:bstring,uri:bstring,date:bstring,request_from:bstring,request_to:bstring,response_from:bstring,response_to:bstring,reply_to:bstring,call_id:bstring,seq:bstring,subject:bstring,request_path:[bstring],response_path:[bstring],user_agent:bstring,status_code:uint64,status_msg:bstring,warning:bstring,request_body_len:uint64,response_body_len:uint64,content_type:bstring,_write_ts:time};
 type smb_files={_path:string,ts:time,uid:bstring,id:id,fuid:bstring,action:zenum,path:bstring,name:bstring,size:uint64,prev_name:bstring,times:{modified:time,accessed:time,created:time,changed:time},_write_ts:time};
 type smb_mapping={_path:string,ts:time,uid:bstring,id:id,path:bstring,service:bstring,native_file_system:bstring,share_type:bstring,_write_ts:time};
-type smtp={_path:string,ts:time,uid:bstring,id:id,trans_depth:uint64,helo:bstring,mailfrom:bstring,rcptto:|[bstring]|,date:bstring,from:bstring,to:|[bstring]|,cc:|[bstring]|,reply_to:bstring,msg_id:bstring,in_reply_to:bstring,subject:bstring,x_originating_ip:ip,first_received:bstring,second_received:bstring,last_reply:bstring,path:[ip],user_agent:bstring,tls:bool,fuids:[bstring],is_webmail:bool,_write_ts:time};
+type smtp={_path:string,ts:time,uid:bstring,id:id,trans_depth:uint64,helo:bstring,mailfrom:bstring,rcptto:|[bstring]|,date:bstring,\from:bstring,to:|[bstring]|,cc:|[bstring]|,reply_to:bstring,msg_id:bstring,in_reply_to:bstring,subject:bstring,x_originating_ip:ip,first_received:bstring,second_received:bstring,last_reply:bstring,path:[ip],user_agent:bstring,tls:bool,fuids:[bstring],is_webmail:bool,_write_ts:time};
 type snmp={_path:string,ts:time,uid:bstring,id:id,duration:duration,version:bstring,community:bstring,get_requests:uint64,get_bulk_requests:uint64,get_responses:uint64,set_requests:uint64,display_string:bstring,up_since:time,_write_ts:time};
 type socks={_path:string,ts:time,uid:bstring,id:id,version:uint64,user:bstring,password:bstring,status:bstring,request:{host:ip,name:bstring},request_p:port=(uint16),bound:{host:ip,name:bstring},bound_p:port=(uint16),_write_ts:time};
 type software={_path:string,ts:time,host:ip,host_p:port=(uint16),software_type:zenum,name:bstring,version:{major:uint64,minor:uint64,minor2:uint64,minor3:uint64,addl:bstring},unparsed_version:bstring,_write_ts:time};
@@ -53,57 +53,56 @@ type tunnel={_path:string,ts:time,uid:bstring,id:id,tunnel_type:zenum,action:zen
 type weird={_path:string,ts:time,uid:bstring,id:id,name:bstring,addl:bstring,notice:bool,peer:bstring,source:bstring,_write_ts:time};
 type x509={_path:string,ts:time,id:bstring,certificate:{version:uint64,serial:bstring,subject:bstring,issuer:bstring,not_valid_before:time,not_valid_after:time,key_alg:bstring,sig_alg:bstring,key_type:bstring,key_length:uint64,exponent:bstring,curve:bstring},san:{dns:[bstring],uri:[bstring],email:[bstring],ip:[ip]},basic_constraints:{ca:bool,path_len:uint64},_write_ts:time};
 
+const schemas = |{
+  "broker": broker,
+  "capture_loss": capture_loss,
+  "cluster": cluster,
+  "config": config,
+  "conn": conn,
+  "dce_rpc": dce_rpc,
+  "dhcp": dhcp,
+  "dnp3": dnp3,
+  "dns": dns,
+  "dpd": dpd,
+  "files": files,
+  "ftp": ftp,
+  "http": http,
+  "intel": intel,
+  "irc": irc,
+  "kerberos": kerberos,
+  "known_certs": known_certs,
+  "known_hosts": known_hosts,
+  "known_services": known_services,
+  "loaded_scripts": loaded_scripts,
+  "modbus": modbus, "mysql": mysql,
+  "netcontrol": netcontrol,
+  "netcontrol_drop": netcontrol_drop,
+  "netcontrol_shunt": netcontrol_shunt,
+  "notice": notice,
+  "notice_alarm": notice_alarm,
+  "ntlm": ntlm, "ntp": ntp,
+  "packet_filter": packet_filter,
+  "pe": pe,
+  "radius": radius,
+  "rdp": rdp,
+  "reporter": reporter,
+  "rfb": rfb,
+  "signatures": signatures,
+  "sip": sip,
+  "smb_files": smb_files,
+  "smb_mapping": smb_mapping,
+  "smtp": smtp,
+  "snmp": snmp,
+  "socks": socks,
+  "software": software,
+  "ssh": ssh,
+  "ssl": ssl,
+  "stats": stats,
+  "syslog": syslog,
+  "tunnel": tunnel,
+  "weird": weird,
+  "x509":
+  x509
+}|
 
-put .=unflatten(.)
-   | switch (
-    case _path=files => put . = shape(files)
-    case _path=http => put . = shape(http)
-    case _path=rdp => put . = shape(rdp)
-    case _path=ssl => put . = shape(ssl)
-    case _path=tunnel => put . = shape(tunnel)
-    case _path=dns => put . = shape(dns)
-    case _path=known_certs => put . = shape(known_certs)
-    case _path=dhcp => put . = shape(dhcp)
-    case _path=modbus => put . = shape(modbus)
-    case _path=ntp => put . = shape(ntp)
-    case _path=snmp => put . = shape(snmp)
-    case _path=ssh => put . = shape(ssh)
-    case _path=syslog => put . = shape(syslog)
-    case _path=x509 => put . = shape(x509)
-    case _path=capture_loss => put . = shape(capture_loss)
-    case _path=dce_rpc => put . = shape(dce_rpc)
-    case _path=ftp => put . = shape(ftp)
-    case _path=notice => put . = shape(notice)
-    case _path=packet_filter => put . = shape(packet_filter)
-    case _path=radius => put . = shape(radius)
-    case _path=smtp => put . = shape(smtp)
-    case _path=stats => put . = shape(stats)
-    case _path=weird => put . = shape(weird)
-    case _path=dpd => put . = shape(dpd)
-    case _path=intel => put . = shape(intel)
-    case _path=irc => put . = shape(irc)
-    case _path=kerberos => put . = shape(kerberos)
-    case _path=loaded_scripts => put . = shape(loaded_scripts)
-    case _path=mysql => put . = shape(mysql)
-    case _path=netcontrol => put . = shape(netcontrol)
-    case _path=netcontrol_shunt => put . = shape(netcontrol_shunt)
-    case _path=signatures => put . = shape(signatures)
-    case _path=socks => put . = shape(socks)
-    case _path=software => put . = shape(software)
-    case _path=known_services => put . = shape(known_services)
-    case _path=sip => put . = shape(sip)
-    case _path=smb_mapping => put . = shape(smb_mapping)
-    case _path=broker => put . = shape(broker)
-    case _path=config => put . = shape(config)
-    case _path=notice_alarm => put . = shape(notice_alarm)
-    case _path=pe => put . = shape(pe)
-    case _path=smb_files => put . = shape(smb_files)
-    case _path=cluster => put . = shape(cluster)
-    case _path=conn => put . = shape(conn)
-    case _path=dnp3 => put . = shape(dnp3)
-    case _path=known_hosts => put . = shape(known_hosts)
-    case _path=netcontrol_drop => put . = shape(netcontrol_drop)
-    case _path=ntlm => put . = shape(ntlm)
-    case _path=reporter => put . = shape(reporter)
-    case _path=rfb => put . = shape(rfb)
-)
+put .=unflatten(.) | put . = shape(schemas[_path])


Test patch:
diff --git a/expr/expr_test.go b/expr/expr_test.go
index 79259a383..9ef8b0da4 100644
--- a/expr/expr_test.go
+++ b/expr/expr_test.go
@@ -26,29 +26,32 @@ func testSuccessful(t *testing.T, e string, record string, expect zng.Value) {
 	formatter := zson.NewFormatter(0)
 	val, err := formatter.Format(rec.Value)
 	require.NoError(t, err)
-	zt := &ztest.ZTest{
+	runZTest(t, e, &ztest.ZTest{
 		Zed:    fmt.Sprintf("cut result = %s", e),
 		Input:  record,
 		Output: val + "\n",
-	}
-	t.Run(e, func(t *testing.T) {
-		t.Parallel()
-		if err := zt.RunInternal(""); err != nil {
-			t.Fatal(err)
-		}
 	})
 }
 
-func testError(t *testing.T, e string, record string, expectErr error, description string) {
-	if record == "" {
-		record = "{}"
-	}
-	zt := &ztest.ZTest{
+func testError(t *testing.T, e string, expectErr error, description string) {
+	runZTest(t, e, &ztest.ZTest{
 		Zed:     fmt.Sprintf("cut result = %s", e),
-		Input:   record,
-		Output:  "",
 		ErrorRE: expectErr.Error(),
+	})
+}
+
+func testWarning(t *testing.T, e string, record string, expectErr error, description string) {
+	if record == "" {
+		record = "{}"
 	}
+	runZTest(t, e, &ztest.ZTest{
+		Zed:      fmt.Sprintf("cut result = %s", e),
+		Input:    record,
+		Warnings: fmt.Sprintf("cut: %s\ncut: no record found with columns <not a field>\n", expectErr),
+	})
+}
+
+func runZTest(t *testing.T, e string, zt *ztest.ZTest) {
 	t.Run(e, func(t *testing.T) {
 		t.Parallel()
 		if err := zt.RunInternal(""); err != nil {
@@ -232,33 +235,33 @@ func TestCompareNumbers(t *testing.T) {
 #0:record[x:%s,s:string,bs:bstring,i:ip,n:net]
 0:[1;hello;world;10.1.1.1;10.1.0.0/16;]`, typ)
 
-		testError(t, "x = s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
-		testError(t, "x != s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
-		testError(t, "x < s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
-		testError(t, "x <= s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
-		testError(t, "x > s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
-		testError(t, "x >= s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
-
-		testError(t, "x = bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
-		testError(t, "x != bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
-		testError(t, "x < bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
-		testError(t, "x <= bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
-		testError(t, "x > bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
-		testError(t, "x >= bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
-
-		testError(t, "x = i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
-		testError(t, "x != i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
-		testError(t, "x < i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
-		testError(t, "x <= i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
-		testError(t, "x > i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
-		testError(t, "x >= i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
-
-		testError(t, "x = n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
-		testError(t, "x != n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
-		testError(t, "x < n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
-		testError(t, "x <= n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
-		testError(t, "x > n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
-		testError(t, "x >= n", record, expr.ErrIncompatibleTypes, "comparing integer and string")
+		testWarning(t, "x = s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
+		testWarning(t, "x != s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
+		testWarning(t, "x < s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
+		testWarning(t, "x <= s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
+		testWarning(t, "x > s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
+		testWarning(t, "x >= s", record, expr.ErrIncompatibleTypes, "comparing integer and string")
+
+		testWarning(t, "x = bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
+		testWarning(t, "x != bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
+		testWarning(t, "x < bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
+		testWarning(t, "x <= bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
+		testWarning(t, "x > bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
+		testWarning(t, "x >= bs", record, expr.ErrIncompatibleTypes, "comparing integer and bstring")
+
+		testWarning(t, "x = i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
+		testWarning(t, "x != i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
+		testWarning(t, "x < i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
+		testWarning(t, "x <= i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
+		testWarning(t, "x > i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
+		testWarning(t, "x >= i", record, expr.ErrIncompatibleTypes, "comparing integer and ip")
+
+		testWarning(t, "x = n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
+		testWarning(t, "x != n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
+		testWarning(t, "x < n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
+		testWarning(t, "x <= n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
+		testWarning(t, "x > n", record, expr.ErrIncompatibleTypes, "comparing integer and net")
+		testWarning(t, "x >= n", record, expr.ErrIncompatibleTypes, "comparing integer and string")
 	}
 
 	// Test comparison between signed and unsigned and also
@@ -359,7 +362,7 @@ func TestCompareNonNumbers(t *testing.T) {
 			for _, op := range allOperators {
 				exp := fmt.Sprintf("%s = %s", t1.field, t2.field)
 				desc := fmt.Sprintf("compare %s %s %s", t1.typ, op, t2.typ)
-				testError(t, exp, record, expr.ErrIncompatibleTypes, desc)
+				testWarning(t, exp, record, expr.ErrIncompatibleTypes, desc)
 			}
 		}
 	}
@@ -554,14 +557,14 @@ func TestArithmetic(t *testing.T) {
 	testSuccessful(t, `"hello" + " world"`, record, zstring("hello world"))
 
 	// Test string arithmetic other than + fails
-	testError(t, `"hello" - " world"`, record, expr.ErrIncompatibleTypes, "subtracting strings")
-	testError(t, `"hello" * " world"`, record, expr.ErrIncompatibleTypes, "multiplying strings")
-	testError(t, `"hello" / " world"`, record, expr.ErrIncompatibleTypes, "dividing strings")
+	testWarning(t, `"hello" - " world"`, record, expr.ErrIncompatibleTypes, "subtracting strings")
+	testWarning(t, `"hello" * " world"`, record, expr.ErrIncompatibleTypes, "multiplying strings")
+	testWarning(t, `"hello" / " world"`, record, expr.ErrIncompatibleTypes, "dividing strings")
 
 	// Test that addition fails on an unsupported type
-	testError(t, "10.1.1.1 + 1", record, expr.ErrIncompatibleTypes, "adding ip and integer")
-	testError(t, "10.1.1.1 + 3.14159", record, expr.ErrIncompatibleTypes, "adding ip and float")
-	testError(t, `10.1.1.1 + "foo"`, record, expr.ErrIncompatibleTypes, "adding ip and string")
+	testWarning(t, "10.1.1.1 + 1", record, expr.ErrIncompatibleTypes, "adding ip and integer")
+	testWarning(t, "10.1.1.1 + 3.14159", record, expr.ErrIncompatibleTypes, "adding ip and float")
+	testWarning(t, `10.1.1.1 + "foo"`, record, expr.ErrIncompatibleTypes, "adding ip and string")
 }
 
 func TestArrayIndex(t *testing.T) {
@@ -594,7 +597,7 @@ func TestConditional(t *testing.T) {
 
 	testSuccessful(t, `x = 0 ? "zero" : "not zero"`, record, zstring("not zero"))
 	testSuccessful(t, `x = 1 ? "one" : "not one"`, record, zstring("one"))
-	testError(t, `x ? "x" : "not x"`, record, expr.ErrIncompatibleTypes, "conditional with non-boolean condition")
+	testWarning(t, `x ? "x" : "not x"`, record, expr.ErrIncompatibleTypes, "conditional with non-boolean condition")
 
 	// Ensure that the unevaluated clause doesn't generate errors
 	// (field y doesn't exist but it shouldn't be evaluated)
@@ -605,47 +608,47 @@ func TestConditional(t *testing.T) {
 func TestCasts(t *testing.T) {
 	// Test casts to byte
 	testSuccessful(t, "uint8(10)", "", zng.Value{zng.TypeUint8, zng.EncodeUint(10)})
-	testError(t, "uint8(-1)", "", expr.ErrBadCast, "out of range cast to uint8")
-	testError(t, "uint8(300)", "", expr.ErrBadCast, "out of range cast to uint8")
-	testError(t, `uint8("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint8")
+	testWarning(t, "uint8(-1)", "", expr.ErrBadCast, "out of range cast to uint8")
+	testWarning(t, "uint8(300)", "", expr.ErrBadCast, "out of range cast to uint8")
+	testWarning(t, `uint8("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint8")
 
 	// Test casts to int16
 	testSuccessful(t, "int16(10)", "", zng.Value{zng.TypeInt16, zng.EncodeInt(10)})
-	testError(t, "int16(-33000)", "", expr.ErrBadCast, "out of range cast to int16")
-	testError(t, "int16(33000)", "", expr.ErrBadCast, "out of range cast to int16")
-	testError(t, `int16("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to int16")
+	testWarning(t, "int16(-33000)", "", expr.ErrBadCast, "out of range cast to int16")
+	testWarning(t, "int16(33000)", "", expr.ErrBadCast, "out of range cast to int16")
+	testWarning(t, `int16("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to int16")
 
 	// Test casts to uint16
 	testSuccessful(t, "uint16(10)", "", zng.Value{zng.TypeUint16, zng.EncodeUint(10)})
-	testError(t, "uint16(-1)", "", expr.ErrBadCast, "out of range cast to uint16")
-	testError(t, "uint16(66000)", "", expr.ErrBadCast, "out of range cast to uint16")
-	testError(t, `uint16("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint16")
+	testWarning(t, "uint16(-1)", "", expr.ErrBadCast, "out of range cast to uint16")
+	testWarning(t, "uint16(66000)", "", expr.ErrBadCast, "out of range cast to uint16")
+	testWarning(t, `uint16("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint16")
 
 	// Test casts to int32
 	testSuccessful(t, "int32(10)", "", zng.Value{zng.TypeInt32, zng.EncodeInt(10)})
-	testError(t, "int32(-2200000000)", "", expr.ErrBadCast, "out of range cast to int32")
-	testError(t, "int32(2200000000)", "", expr.ErrBadCast, "out of range cast to int32")
-	testError(t, `int32("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to int32")
+	testWarning(t, "int32(-2200000000)", "", expr.ErrBadCast, "out of range cast to int32")
+	testWarning(t, "int32(2200000000)", "", expr.ErrBadCast, "out of range cast to int32")
+	testWarning(t, `int32("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to int32")
 
 	// Test casts to uint32
 	testSuccessful(t, "uint32(10)", "", zng.Value{zng.TypeUint32, zng.EncodeUint(10)})
-	testError(t, "uint32(-1)", "", expr.ErrBadCast, "out of range cast to uint32")
-	testError(t, "uint8(4300000000)", "", expr.ErrBadCast, "out of range cast to uint32")
-	testError(t, `uint32("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint32")
+	testWarning(t, "uint32(-1)", "", expr.ErrBadCast, "out of range cast to uint32")
+	testWarning(t, "uint8(4300000000)", "", expr.ErrBadCast, "out of range cast to uint32")
+	testWarning(t, `uint32("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint32")
 
 	// Test casts to uint64
 	testSuccessful(t, "uint64(10)", "", zuint64(10))
-	testError(t, "uint64(-1)", "", expr.ErrBadCast, "out of range cast to uint64")
-	testError(t, `uint64("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint64")
+	testWarning(t, "uint64(-1)", "", expr.ErrBadCast, "out of range cast to uint64")
+	testWarning(t, `uint64("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to uint64")
 
 	// Test casts to float64
 	testSuccessful(t, "float64(10)", "", zfloat64(10))
-	testError(t, `float64("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to float64")
+	testWarning(t, `float64("foo")`, "", expr.ErrBadCast, "cannot cast incompatible type to float64")
 
 	// Test casts to ip
 	testSuccessful(t, `ip("1.2.3.4")`, "", zip(t, "1.2.3.4"))
-	testError(t, "ip(1234)", "", expr.ErrBadCast, "cast of invalid ip address fails")
-	testError(t, `ip("not an address")`, "", expr.ErrBadCast, "cast of invalid ip address fails")
+	testWarning(t, "ip(1234)", "", expr.ErrBadCast, "cast of invalid ip address fails")
+	testWarning(t, `ip("not an address")`, "", expr.ErrBadCast, "cast of invalid ip address fails")
 
 	// Test casts to time
 	ts := zng.Value{zng.TypeTime, zng.EncodeTime(nano.Ts(1589126400_000_000_000))}
@@ -661,8 +664,8 @@ func TestCasts(t *testing.T) {
 	testSuccessful(t, `float64("5.5")`, "", zfloat64(5.5))
 	testSuccessful(t, `ip("1.2.3.4")`, "", zaddr("1.2.3.4"))
 
-	testError(t, "ip(1)", "", expr.ErrBadCast, "ip cast non-ip arg")
-	testError(t, `int64("abc")`, "", expr.ErrBadCast, "int64 cast with non-parseable string")
-	testError(t, `float64("abc")`, "", expr.ErrBadCast, "float64 cast with non-parseable string")
-	testError(t, `ip("abc")`, "", expr.ErrBadCast, "ip cast with non-parseable string")
+	testWarning(t, "ip(1)", "", expr.ErrBadCast, "ip cast non-ip arg")
+	testWarning(t, `int64("abc")`, "", expr.ErrBadCast, "int64 cast with non-parseable string")
+	testWarning(t, `float64("abc")`, "", expr.ErrBadCast, "float64 cast with non-parseable string")
+	testWarning(t, `ip("abc")`, "", expr.ErrBadCast, "ip cast with non-parseable string")
 }
diff --git a/expr/functions_test.go b/expr/functions_test.go
index 29afee926..17bb20d4e 100644
--- a/expr/functions_test.go
+++ b/expr/functions_test.go
@@ -10,13 +10,17 @@ import (
 	"github.com/brimsec/zq/zng"
 )
 
+func namedErrBadArgument(name string) error {
+	return fmt.Errorf("%s: %w", name, function.ErrBadArgument)
+}
+
 func zaddr(addr string) zng.Value {
 	parsed := net.ParseIP(addr)
 	return zng.Value{zng.TypeIP, zng.EncodeIP(parsed)}
 }
 
 func TestBadFunction(t *testing.T) {
-	testError(t, "notafunction()", "", function.ErrNoSuchFunction, "calling nonexistent function")
+	testError(t, "notafunction()", function.ErrNoSuchFunction, "calling nonexistent function")
 }
 
 func TestAbs(t *testing.T) {
@@ -30,9 +34,9 @@ func TestAbs(t *testing.T) {
 	testSuccessful(t, "abs(3.2)", record, zfloat64(3.2))
 	testSuccessful(t, "abs(u)", record, zuint64(50))
 
-	testError(t, "abs()", record, function.ErrTooFewArgs, "abs with no args")
-	testError(t, "abs(1, 2)", record, function.ErrTooManyArgs, "abs with too many args")
-	testError(t, `abs("hello")`, record, function.ErrBadArgument, "abs with non-number")
+	testError(t, "abs()", function.ErrTooFewArgs, "abs with no args")
+	testError(t, "abs(1, 2)", function.ErrTooManyArgs, "abs with too many args")
+	testWarning(t, `abs("hello")`, record, namedErrBadArgument("abs"), "abs with non-number")
 }
 
 func TestSqrt(t *testing.T) {
@@ -44,9 +48,9 @@ func TestSqrt(t *testing.T) {
 	testSuccessful(t, "sqrt(f)", record, zfloat64(2.5))
 	testSuccessful(t, "sqrt(i)", record, zfloat64(3.0))
 
-	testError(t, "sqrt()", record, function.ErrTooFewArgs, "sqrt with no args")
-	testError(t, "sqrt(1, 2)", record, function.ErrTooManyArgs, "sqrt with too many args")
-	testError(t, "sqrt(-1)", record, function.ErrBadArgument, "sqrt of negative")
+	testError(t, "sqrt()", function.ErrTooFewArgs, "sqrt with no args")
+	testError(t, "sqrt(1, 2)", function.ErrTooManyArgs, "sqrt with too many args")
+	testWarning(t, "sqrt(-1)", record, namedErrBadArgument("sqrt"), "sqrt of negative")
 }
 
 func TestMinMax(t *testing.T) {
@@ -63,8 +67,8 @@ func TestMinMax(t *testing.T) {
 	testSuccessful(t, "max(3, 2, 1)", record, zint64(3))
 
 	// Fails with no arguments
-	testError(t, "min()", record, function.ErrTooFewArgs, "min with no args")
-	testError(t, "max()", record, function.ErrTooFewArgs, "max with no args")
+	testError(t, "min()", function.ErrTooFewArgs, "min with no args")
+	testError(t, "max()", function.ErrTooFewArgs, "max with no args")
 
 	// Mixed types work
 	testSuccessful(t, "min(i, 2, 3)", record, zuint64(1))
@@ -77,10 +81,10 @@ func TestMinMax(t *testing.T) {
 	testSuccessful(t, "max(2.0, -1)", record, zfloat64(2))
 
 	// Fails on invalid types
-	testError(t, `min("hello", 2)`, record, function.ErrBadArgument, "min() on string")
-	testError(t, `max("hello", 2)`, record, function.ErrBadArgument, "max() on string")
-	testError(t, `min(1.2.3.4, 2)`, record, function.ErrBadArgument, "min() on ip")
-	testError(t, `max(1.2.3.4, 2)`, record, function.ErrBadArgument, "max() on ip")
+	testWarning(t, `min("hello", 2)`, record, function.ErrBadArgument, "min() on string")
+	testWarning(t, `max("hello", 2)`, record, function.ErrBadArgument, "max() on string")
+	testWarning(t, `min(1.2.3.4, 2)`, record, function.ErrBadArgument, "min() on ip")
+	testWarning(t, `max(1.2.3.4, 2)`, record, function.ErrBadArgument, "max() on ip")
 
 }
 
@@ -93,12 +97,12 @@ func TestCeilFloorRound(t *testing.T) {
 	testSuccessful(t, "floor(5)", "", zint64(5))
 	testSuccessful(t, "round(5)", "", zint64(5))
 
-	testError(t, "ceil()", "", function.ErrTooFewArgs, "ceil() with no args")
-	testError(t, "ceil(1, 2)", "", function.ErrTooManyArgs, "ceil() with too many args")
-	testError(t, "floor()", "", function.ErrTooFewArgs, "floor() with no args")
-	testError(t, "floor(1, 2)", "", function.ErrTooManyArgs, "floor() with too many args")
-	testError(t, "round()", "", function.ErrTooFewArgs, "round() with no args")
-	testError(t, "round(1, 2)", "", function.ErrTooManyArgs, "round() with too many args")
+	testError(t, "ceil()", function.ErrTooFewArgs, "ceil() with no args")
+	testError(t, "ceil(1, 2)", function.ErrTooManyArgs, "ceil() with too many args")
+	testError(t, "floor()", function.ErrTooFewArgs, "floor() with no args")
+	testError(t, "floor(1, 2)", function.ErrTooManyArgs, "floor() with too many args")
+	testError(t, "round()", function.ErrTooFewArgs, "round() with no args")
+	testError(t, "round(1, 2)", function.ErrTooManyArgs, "round() with too many args")
 }
 
 func TestLogPow(t *testing.T) {
@@ -111,14 +115,14 @@ func TestLogPow(t *testing.T) {
 	testSuccessful(t, "pow(10, 2)", "", zfloat64(100))
 	testSuccessful(t, "pow(4.0, 1.5)", "", zfloat64(8))
 
-	testError(t, "log()", "", function.ErrTooFewArgs, "log() with no args")
-	testError(t, "log(2, 3)", "", function.ErrTooManyArgs, "log() with too many args")
-	testError(t, "log(0)", "", function.ErrBadArgument, "log() of 0")
-	testError(t, "log(-1)", "", function.ErrBadArgument, "log() of negative number")
+	testError(t, "log()", function.ErrTooFewArgs, "log() with no args")
+	testError(t, "log(2, 3)", function.ErrTooManyArgs, "log() with too many args")
+	testWarning(t, "log(0)", "", namedErrBadArgument("log"), "log() of 0")
+	testWarning(t, "log(-1)", "", namedErrBadArgument("log"), "log() of negative number")
 
-	testError(t, "pow()", "", function.ErrTooFewArgs, "pow() with no args")
-	testError(t, "pow(2, 3, r)", "", function.ErrTooManyArgs, "pow() with too many args")
-	testError(t, "pow(-1, 0.5)", "", function.ErrBadArgument, "pow() with invalid arguments")
+	testError(t, "pow()", function.ErrTooFewArgs, "pow() with no args")
+	testError(t, "pow(2, 3, r)", function.ErrTooManyArgs, "pow() with too many args")
+	testWarning(t, "pow(-1, 0.5)", "", namedErrBadArgument("pow"), "pow() with invalid arguments")
 }
 
 func TestMod(t *testing.T) {
@@ -131,28 +135,28 @@ func TestMod(t *testing.T) {
 	testSuccessful(t, "mod(8, 5)", record, zint64(3))
 	testSuccessful(t, "mod(8, u)", record, zint64(3))
 
-	testError(t, "mod()", record, function.ErrTooFewArgs, "mod() with no args")
-	testError(t, "mod(1, 2, 3)", record, function.ErrTooManyArgs, "mod() with too many args")
-	testError(t, "mod(3.2, 2)", record, function.ErrBadArgument, "mod() with float64 arg")
+	testError(t, "mod()", function.ErrTooFewArgs, "mod() with no args")
+	testError(t, "mod(1, 2, 3)", function.ErrTooManyArgs, "mod() with too many args")
+	testWarning(t, "mod(3.2, 2)", record, namedErrBadArgument("mod"), "mod() with float64 arg")
 }
 
 func TestOtherStrFuncs(t *testing.T) {
 	testSuccessful(t, `replace("bann", "n", "na")`, "", zstring("banana"))
-	testError(t, `replace("foo", "bar")`, "", function.ErrTooFewArgs, "replace() with too few args")
-	testError(t, `replace("foo", "bar", "baz", "blort")`, "", function.ErrTooManyArgs, "replace() with too many args")
-	testError(t, `replace("foo", "o", 5)`, "", function.ErrBadArgument, "replace() with non-string arg")
+	testError(t, `replace("foo", "bar")`, function.ErrTooFewArgs, "replace() with too few args")
+	testError(t, `replace("foo", "bar", "baz", "blort")`, function.ErrTooManyArgs, "replace() with too many args")
+	testWarning(t, `replace("foo", "o", 5)`, "", namedErrBadArgument("replace"), "replace() with non-string arg")
 
 	testSuccessful(t, `to_lower("BOO")`, "", zstring("boo"))
-	testError(t, `to_lower()`, "", function.ErrTooFewArgs, "toLower() with no args")
-	testError(t, `to_lower("BOO", "HOO")`, "", function.ErrTooManyArgs, "toLower() with too many args")
+	testError(t, `to_lower()`, function.ErrTooFewArgs, "toLower() with no args")
+	testError(t, `to_lower("BOO", "HOO")`, function.ErrTooManyArgs, "toLower() with too many args")
 
 	testSuccessful(t, `to_upper("boo")`, "", zstring("BOO"))
-	testError(t, `to_upper()`, "", function.ErrTooFewArgs, "toUpper() with no args")
-	testError(t, `to_upper("boo", "hoo")`, "", function.ErrTooManyArgs, "toUpper() with too many args")
+	testError(t, `to_upper()`, function.ErrTooFewArgs, "toUpper() with no args")
+	testError(t, `to_upper("boo", "hoo")`, function.ErrTooManyArgs, "toUpper() with too many args")
 
 	testSuccessful(t, `trim("  hi  there   ")`, "", zstring("hi  there"))
-	testError(t, `trim()`, "", function.ErrTooFewArgs, "trim() with no args")
-	testError(t, `trim("  hi  ", "  there  ")`, "", function.ErrTooManyArgs, "trim() with too many args")
+	testError(t, `trim()`, function.ErrTooFewArgs, "trim() with no args")
+	testError(t, `trim("  hi  ", "  there  ")`, function.ErrTooManyArgs, "trim() with too many args")
 }
 
 func TestLen(t *testing.T) {
@@ -163,9 +167,9 @@ func TestLen(t *testing.T) {
 	testSuccessful(t, "len(s)", record, zint64(3))
 	testSuccessful(t, "len(a)", record, zint64(3))
 
-	testError(t, "len()", record, function.ErrTooFewArgs, "len() with no args")
-	testError(t, `len("foo", "bar")`, record, function.ErrTooManyArgs, "len() with too many args")
-	testError(t, "len(5)", record, function.ErrBadArgument, "len() with non-container arg")
+	testError(t, "len()", function.ErrTooFewArgs, "len() with no args")
+	testError(t, `len("foo", "bar")`, function.ErrTooManyArgs, "len() with too many args")
+	testWarning(t, "len(5)", record, namedErrBadArgument("len"), "len() with non-container arg")
 
 	record = `
 #0:record[s:string,bs:bstring,bs2:bstring]
@@ -194,6 +198,6 @@ func TestTime(t *testing.T) {
 
 	testSuccessful(t, "trunc(1590506867.967, 1)", "", zng.Value{zng.TypeTime, zng.EncodeTime(nano.Ts(1590506867 * 1_000_000_000))})
 
-	testError(t, "iso()", "", function.ErrTooFewArgs, "iso() with no args")
-	testError(t, `iso("abc", "def")`, "", function.ErrTooManyArgs, "iso() with too many args")
+	testError(t, "iso()", function.ErrTooFewArgs, "iso() with no args")
+	testError(t, `iso("abc", "def")`, function.ErrTooManyArgs, "iso() with too many args")
 }
diff --git a/expr/ztests/cut-to-root-error-non-record.yaml b/expr/ztests/cut-to-root-error-non-record.yaml
index fb3f1e9fd..0bb9e13ca 100644
--- a/expr/ztests/cut-to-root-error-non-record.yaml
+++ b/expr/ztests/cut-to-root-error-non-record.yaml
@@ -4,4 +4,6 @@ input: |
   #0:record[a:int32]
   0:[1;]
 
-errorRE: cannot cut a non-record to \.
+warnings: |
+  cut: cannot cut a non-record to .
+  cut: no record found with columns a
diff --git a/ppl/cmd/pgctl/rmtestdb/command.go b/ppl/cmd/pgctl/rmtestdb/command.go
index b733c37b0..f2e146c93 100644
--- a/ppl/cmd/pgctl/rmtestdb/command.go
+++ b/ppl/cmd/pgctl/rmtestdb/command.go
@@ -7,10 +7,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/ppl/cmd/pgctl/root"
 	"github.com/brimsec/zq/ppl/zqd/db/postgresdb"
 	"github.com/go-pg/pg/v10"
-	"github.com/mccanne/charm"
 )
 
 var RmTestDB = &charm.Spec{
diff --git a/ppl/cmd/pgctl/testdb/command.go b/ppl/cmd/pgctl/testdb/command.go
index f2c12e078..e6a736a21 100644
--- a/ppl/cmd/pgctl/testdb/command.go
+++ b/ppl/cmd/pgctl/testdb/command.go
@@ -6,6 +6,7 @@ import (
 	"flag"
 	"fmt"
 
+	"github.com/brimsec/zq/pkg/charm"
 	"github.com/brimsec/zq/pkg/iosrc"
 	"github.com/brimsec/zq/ppl/cmd/pgctl/root"
 	"github.com/brimsec/zq/ppl/zqd/db/postgresdb"
@@ -13,7 +14,6 @@ import (
 	"github.com/golang-migrate/migrate/v4"
 	_ "github.com/golang-migrate/migrate/v4/database/postgres"
 	_ "github.com/golang-migrate/migrate/v4/source/file"
-	"github.com/mccanne/charm"
 	"github.com/segmentio/ksuid"
 )
 
diff --git a/zeek/ztests/shape-zeek-ndjson.yaml b/zeek/ztests/shape-zeek-ndjson.yaml
new file mode 100644
index 000000000..0a6cabd41
--- /dev/null
+++ b/zeek/ztests/shape-zeek-ndjson.yaml
@@ -0,0 +1,30 @@
+script: |
+  zq -Z -I shaper.zed -
+
+inputs:
+  - name: shaper.zed
+    source: ../shaper.zed
+  - name: stdin
+    data: |
+      {"_path":"weird","_write_ts":"2018-03-24T17:15:20.600843Z","ts":"2018-03-24T17:15:20.600843Z","uid":"C1zOivgBT6dBmknqk","id.orig_h":"10.47.1.152","id.orig_p":49562,"id.resp_h":"23.217.103.245","id.resp_p":80,"name":"TCP_ack_underflow_or_misorder","notice":false,"peer":"zeek"}
+
+outputs:
+  - name: stdout
+    data: |
+      {
+          _path: "weird",
+          ts: 2018-03-24T17:15:20.600843Z,
+          uid: "C1zOivgBT6dBmknqk" (bstring),
+          id: {
+              orig_h: 10.47.1.152,
+              orig_p: 49562 (port=(uint16)),
+              resp_h: 23.217.103.245,
+              resp_p: 80 (port)
+          } (=0),
+          name: "TCP_ack_underflow_or_misorder" (bstring),
+          addl: null (bstring),
+          notice: false,
+          peer: "zeek" (bstring),
+          source: null (bstring),
+          _write_ts: 2018-03-24T17:15:20.600843Z
+      } (=1)
diff --git a/ztests/suite/rename/rename-error-dupe.yaml b/ztests/suite/rename/rename-error-dupe.yaml
index bfa6cc38e..33b6d8908 100644
--- a/ztests/suite/rename/rename-error-dupe.yaml
+++ b/ztests/suite/rename/rename-error-dupe.yaml
@@ -4,4 +4,5 @@ input: |
   #0:record[foo:string,goo:string]
   0:[foo1;goo1;]
 
-errorRE: "rename: duplicate field foo"
+warnings: |
+  rename: duplicate field foo