func (s *storageCeph) getRBDSize() (string, error) {
	sz, err := shared.ParseByteSizeString(s.volume.Config["size"])
	if err != nil {
		return "", err
	}

	// Safety net: Set to default value.
	if sz == 0 {
		sz, _ = shared.ParseByteSizeString("10GB")
	}

	return fmt.Sprintf("%dB", sz), nil
}

func
(
s
*
storageCeph
)
getRBDSize
(
)
(
string
,
error
)
{
sz
,
err
:=
shared
.
ParseByteSizeString
(
s
.
volume
.
Config
[
"
"
]
)
if
err
!=
{
return
"
"
,
err
}
// Safety net: Set to default value.
if
sz
==
0
{
sz
,
_
=
shared
.
ParseByteSizeString
(
"
"
)
}
return
fmt
.
Sprintf
(
"
"
,
sz
)
,
}

// getRBDSize returns the size the RBD storage volume is supposed to be created
// with

getRBDSize returns the size the RBD storage volume is supposed to be created with


func (s *storageCeph) getRBDFilesystem() string {
	if s.volume.Config["block.filesystem"] != "" {
		return s.volume.Config["block.filesystem"]
	}

	if s.pool.Config["volume.block.filesystem"] != "" {
		return s.pool.Config["volume.block.filesystem"]
	}

	return "ext4"
}

func ( s * storageCeph ) getRBDFilesystem ( ) string { if s . volume . Config [ " " ] != " " { return s . volume . Config [ " " ] } if s . pool . Config [ " " ] != " " { return s . pool . Config [ " " ] } return " " }

// getRBDFilesystem returns the filesystem the RBD storage volume is supposed to
// be created with

getRBDFilesystem returns the filesystem the RBD storage volume is supposed to be created with


func (s *storageCeph) copyWithoutSnapshotsFull(target container,
	source container) error {
	logger.Debugf(`Creating non-sparse copy of RBD storage volume for container "%s" to "%s" without snapshots`, source.Name(), target.Name())

	sourceIsSnapshot := source.IsSnapshot()
	sourceContainerName := projectPrefix(source.Project(), source.Name())
	targetContainerName := projectPrefix(target.Project(), target.Name())
	oldVolumeName := fmt.Sprintf("%s/container_%s", s.OSDPoolName,
		sourceContainerName)
	newVolumeName := fmt.Sprintf("%s/container_%s", s.OSDPoolName,
		targetContainerName)
	if sourceIsSnapshot {
		sourceContainerOnlyName, sourceSnapshotOnlyName, _ :=
			containerGetParentAndSnapshotName(sourceContainerName)
		oldVolumeName = fmt.Sprintf("%s/container_%s@snapshot_%s",
			s.OSDPoolName, sourceContainerOnlyName,
			sourceSnapshotOnlyName)
	}

	err := cephRBDVolumeCopy(s.ClusterName, oldVolumeName, newVolumeName,
		s.UserName)
	if err != nil {
		logger.Debugf(`Failed to create full RBD copy "%s" to "%s": %s`, source.Name(), target.Name(), err)
		return err
	}

	_, err = cephRBDVolumeMap(s.ClusterName, s.OSDPoolName, targetContainerName,
		storagePoolVolumeTypeNameContainer, s.UserName)
	if err != nil {
		logger.Errorf(`Failed to map RBD storage volume for image "%s" on storage pool "%s": %s`, targetContainerName, s.pool.Name, err)
		return err
	}

	targetContainerMountPoint := getContainerMountPoint(target.Project(), s.pool.Name, target.Name())
	err = createContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged())
	if err != nil {
		return err
	}

	ourMount, err := target.StorageStart()
	if err != nil {
		return err
	}
	if ourMount {
		defer target.StorageStop()
	}

	err = target.TemplateApply("copy")
	if err != nil {
		logger.Errorf(`Failed to apply copy template for container "%s": %s`, target.Name(), err)
		return err
	}
	logger.Debugf(`Applied copy template for container "%s"`, target.Name())

	logger.Debugf(`Created non-sparse copy of RBD storage volume for container "%s" to "%s" without snapshots`, source.Name(),
		target.Name())
	return nil
}

// copyWithoutSnapshotsFull creates a non-sparse copy of a container
// This does not introduce a dependency relation between the source RBD storage
// volume and the target RBD storage volume.

copyWithoutSnapshotsFull creates a non - sparse copy of a container This does not introduce a dependency relation between the source RBD storage volume and the target RBD storage volume .


func (s *storageCeph) copyWithoutSnapshotsSparse(target container,
	source container) error {
	logger.Debugf(`Creating sparse copy of RBD storage volume for container "%s" to "%s" without snapshots`, source.Name(),
		target.Name())

	sourceIsSnapshot := source.IsSnapshot()
	sourceContainerName := projectPrefix(source.Project(), source.Name())
	targetContainerName := projectPrefix(target.Project(), target.Name())
	sourceContainerOnlyName := sourceContainerName
	sourceSnapshotOnlyName := ""
	snapshotName := fmt.Sprintf("zombie_snapshot_%s",
		uuid.NewRandom().String())
	if sourceIsSnapshot {
		sourceContainerOnlyName, sourceSnapshotOnlyName, _ =
			containerGetParentAndSnapshotName(sourceContainerName)
		snapshotName = fmt.Sprintf("snapshot_%s", sourceSnapshotOnlyName)
	} else {
		// create snapshot
		err := cephRBDSnapshotCreate(s.ClusterName, s.OSDPoolName,
			sourceContainerName, storagePoolVolumeTypeNameContainer,
			snapshotName, s.UserName)
		if err != nil {
			logger.Errorf(`Failed to create snapshot for RBD storage volume for image "%s" on storage pool "%s": %s`, targetContainerName, s.pool.Name, err)
			return err
		}
	}

	// protect volume so we can create clones of it
	err := cephRBDSnapshotProtect(s.ClusterName, s.OSDPoolName,
		sourceContainerOnlyName, storagePoolVolumeTypeNameContainer,
		snapshotName, s.UserName)
	if err != nil {
		logger.Errorf(`Failed to protect snapshot for RBD storage volume for image "%s" on storage pool "%s": %s`, snapshotName, s.pool.Name, err)
		return err
	}

	err = cephRBDCloneCreate(s.ClusterName, s.OSDPoolName,
		sourceContainerOnlyName, storagePoolVolumeTypeNameContainer,
		snapshotName, s.OSDPoolName, targetContainerName,
		storagePoolVolumeTypeNameContainer, s.UserName)
	if err != nil {
		logger.Errorf(`Failed to clone new RBD storage volume for container "%s": %s`, targetContainerName, err)
		return err
	}

	// Re-generate the UUID
	err = s.cephRBDGenerateUUID(projectPrefix(target.Project(), target.Name()), storagePoolVolumeTypeNameContainer)
	if err != nil {
		return err
	}

	// Create mountpoint
	targetContainerMountPoint := getContainerMountPoint(target.Project(), s.pool.Name, target.Name())
	err = createContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged())
	if err != nil {
		return err
	}

	ourMount, err := target.StorageStart()
	if err != nil {
		return err
	}
	if ourMount {
		defer target.StorageStop()
	}

	err = target.TemplateApply("copy")
	if err != nil {
		logger.Errorf(`Failed to apply copy template for container "%s": %s`, target.Name(), err)
		return err
	}
	logger.Debugf(`Applied copy template for container "%s"`, target.Name())

	logger.Debugf(`Created sparse copy of RBD storage volume for container "%s" to "%s" without snapshots`, source.Name(),
		target.Name())
	return nil
}

// copyWithoutSnapshotsFull creates a sparse copy of a container
// This introduces a dependency relation between the source RBD storage volume
// and the target RBD storage volume.

copyWithoutSnapshotsFull creates a sparse copy of a container This introduces a dependency relation between the source RBD storage volume and the target RBD storage volume .


func GetConfigCmd(noPortForwarding *bool) *cobra.Command {
	var format string
	getConfig := &cobra.Command{
		Short: "Retrieve Pachyderm's current auth configuration",
		Long:  "Retrieve Pachyderm's current auth configuration",
		Run: cmdutil.RunFixedArgs(0, func(args []string) error {
			c, err := client.NewOnUserMachine(true, !*noPortForwarding, "user")
			if err != nil {
				return fmt.Errorf("could not connect: %v", err)
			}
			defer c.Close()
			resp, err := c.GetConfiguration(c.Ctx(), &auth.GetConfigurationRequest{})
			if err != nil {
				return grpcutil.ScrubGRPC(err)
			}
			if resp.Configuration == nil {
				fmt.Println("no auth config set")
				return nil
			}
			output, err := json.MarshalIndent(resp.Configuration, "", "  ")
			if err != nil {
				return fmt.Errorf("could not marshal response:\n%v\ndue to: %v", resp.Configuration, err)
			}
			switch format {
			case "json":
				// already done
			case "yaml":
				output, err = yaml.JSONToYAML(output)
				if err != nil {
					return fmt.Errorf("could not convert json to yaml: %v", err)
				}
			default:
				return fmt.Errorf("invalid output format: %v", format)
			}
			fmt.Println(string(output))
			return nil
		}),
	}
	getConfig.Flags().StringVarP(&format, "output-format", "o", "json", "output "+
		"format (\"json\" or \"yaml\")")
	return cmdutil.CreateAlias(getConfig, "auth get-config")
}

func
GetConfigCmd
(
noPortForwarding
*
bool
)
*
cobra
.
Command
{
var
format
string
getConfig
:=
&
cobra
.
Command
{
Short
:
"
"
,
Long
:
"
"
,
Run
:
cmdutil
.
RunFixedArgs
(
0
,
func
(
args
[
]
string
)
error
{
c
,
err
:=
client
.
NewOnUserMachine
(
true
,
!
*
noPortForwarding
,
"
"
)
if
err
!=
{
return
fmt
.
Errorf
(
"
"
,
err
)
}
defer
c
.
Close
(
)
resp
,
err
:=
c
.
GetConfiguration
(
c
.
Ctx
(
)
,
&
auth
.
GetConfigurationRequest
{
}
)
if
err
!=
{
return
grpcutil
.
ScrubGRPC
(
err
)
}
if
resp
.
Configuration
==
{
fmt
.
Println
(
"
"
)
return
}
output
,
err
:=
json
.
MarshalIndent
(
resp
.
Configuration
,
"
"
,
"
"
)
if
err
!=
{
return
fmt
.
Errorf
(
"
\n
\n
"
,
resp
.
Configuration
,
err
)
}
switch
format
{
case
"
"
:
// already done
case
"
"
:
output
,
err
=
yaml
.
JSONToYAML
(
output
)
if
err
!=
{
return
fmt
.
Errorf
(
"
"
,
err
)
}
default
:
return
fmt
.
Errorf
(
"
"
,
format
)
}
fmt
.
Println
(
string
(
output
)
)
return
}
)
,
}
getConfig
.
Flags
(
)
.
StringVarP
(
&
format
,
"
"
,
"
"
,
"
"
,
"
"
+
"
\"
\"
\"
\"
"
)
return
cmdutil
.
CreateAlias
(
getConfig
,
"
"
)
}

// GetConfigCmd returns a cobra command that lets the caller see the configured
// auth backends in Pachyderm

GetConfigCmd returns a cobra command that lets the caller see the configured auth backends in Pachyderm


func SetConfigCmd(noPortForwarding *bool) *cobra.Command {
	var file string
	setConfig := &cobra.Command{
		Short: "Set Pachyderm's current auth configuration",
		Long:  "Set Pachyderm's current auth configuration",
		Run: cmdutil.RunFixedArgs(0, func(args []string) error {
			c, err := client.NewOnUserMachine(true, !*noPortForwarding, "user")
			if err != nil {
				return fmt.Errorf("could not connect: %v", err)
			}
			defer c.Close()
			var configBytes []byte
			if file == "-" {
				var err error
				configBytes, err = ioutil.ReadAll(os.Stdin)
				if err != nil {
					return fmt.Errorf("could not read config from stdin: %v", err)
				}
			} else if file != "" {
				var err error
				configBytes, err = ioutil.ReadFile(file)
				if err != nil {
					return fmt.Errorf("could not read config from %q: %v", file, err)
				}
			} else {
				return errors.New("must set input file (use \"-\" to read from stdin)")
			}

			// Try to parse config as YAML (JSON is a subset of YAML)
			var config auth.AuthConfig
			if err := yaml.Unmarshal(configBytes, &config); err != nil {
				return fmt.Errorf("could not parse config: %v", err)
			}
			// TODO(msteffen): try to handle empty config?
			_, err = c.SetConfiguration(c.Ctx(), &auth.SetConfigurationRequest{
				Configuration: &config,
			})
			return grpcutil.ScrubGRPC(err)
		}),
	}
	setConfig.Flags().StringVarP(&file, "file", "f", "-", "input file (to use "+
		"as the new config")
	return cmdutil.CreateAlias(setConfig, "auth set-config")
}

// SetConfigCmd returns a cobra command that lets the caller configure auth
// backends in Pachyderm

SetConfigCmd returns a cobra command that lets the caller configure auth backends in Pachyderm


func NewSharder(discoveryClient discovery.Client, numShards uint64, namespace string) Sharder {
	return newSharder(discoveryClient, numShards, namespace)
}

func NewSharder ( discoveryClient discovery . Client , numShards uint64 , namespace string ) Sharder { return newSharder ( discoveryClient , numShards , namespace ) }

// NewSharder creates a Sharder using a discovery client.

NewSharder creates a Sharder using a discovery client .


func NewRouter(
	sharder Sharder,
	dialer grpcutil.Dialer,
	localAddress string,
) Router {
	return newRouter(
		sharder,
		dialer,
		localAddress,
	)
}

func NewRouter ( sharder Sharder , dialer grpcutil . Dialer , localAddress string , ) Router { return newRouter ( sharder , dialer , localAddress , ) }


func renewUserCredentials(ctx context.Context, pachdAddress string, adminToken string, userToken string, ttl time.Duration) error {
	// Setup a single use client w the given admin token / address
	client, err := pclient.NewFromAddress(pachdAddress)
	if err != nil {
		return err
	}
	defer client.Close() // avoid leaking connections

	client = client.WithCtx(ctx)
	client.SetAuthToken(adminToken)

	_, err = client.AuthAPIClient.ExtendAuthToken(client.Ctx(), &auth.ExtendAuthTokenRequest{
		Token: userToken,
		TTL:   int64(ttl.Seconds()),
	})

	if err != nil {
		return err
	}

	return nil
}

// renewUserCredentials extends the TTL of the Pachyderm authentication token
// 'userToken', using the vault plugin's Admin credentials. 'userToken' belongs
// to the user who is calling vault, and would like to extend their Pachyderm
// session.

renewUserCredentials extends the TTL of the Pachyderm authentication token userToken using the vault plugin s Admin credentials . userToken belongs to the user who is calling vault and would like to extend their Pachyderm session .


func NewLocalClient(root string) (Client, error) {
	if err := os.MkdirAll(root, 0755); err != nil {
		return nil, err
	}
	return &localClient{root}, nil
}

func NewLocalClient ( root string ) ( Client , error ) { if err := os . MkdirAll ( root , 0755 ) ; err != nil { return nil , err } return & localClient { root } , nil }

// NewLocalClient returns a Client that stores data on the local file system


func AddSpanToAnyExisting(ctx context.Context, operation string, kvs ...interface{}) (opentracing.Span, context.Context) {
	if parentSpan := opentracing.SpanFromContext(ctx); parentSpan != nil {
		span := opentracing.StartSpan(operation, opentracing.ChildOf(parentSpan.Context()))
		tagSpan(span, kvs)
		return span, opentracing.ContextWithSpan(ctx, span)
	}
	return nil, ctx
}

func AddSpanToAnyExisting ( ctx context . Context , operation string , kvs ... interface { } ) ( opentracing . Span , context . Context ) { if parentSpan := opentracing . SpanFromContext ( ctx ) ; parentSpan != nil { span := opentracing . StartSpan ( operation , opentracing . ChildOf ( parentSpan . Context ( ) ) ) tagSpan ( span , kvs ) return span , opentracing . ContextWithSpan ( ctx , span ) } return nil , ctx }

// AddSpanToAnyExisting checks 'ctx' for Jaeger tracing information, and if
// tracing metadata is present, it generates a new span for 'operation', marks
// it as a child of the existing span, and returns it.

AddSpanToAnyExisting checks ctx for Jaeger tracing information and if tracing metadata is present it generates a new span for operation marks it as a child of the existing span and returns it .


func InstallJaegerTracerFromEnv() {
	jaegerOnce.Do(func() {
		jaegerEndpoint, onUserMachine := os.LookupEnv(jaegerEndpointEnvVar)
		if !onUserMachine {
			if host, ok := os.LookupEnv("JAEGER_COLLECTOR_SERVICE_HOST"); ok {
				port := os.Getenv("JAEGER_COLLECTOR_SERVICE_PORT_JAEGER_COLLECTOR_HTTP")
				jaegerEndpoint = fmt.Sprintf("%s:%s", host, port)
			}
		}
		if jaegerEndpoint == "" {
			return // break early -- not using Jaeger
		}

		// canonicalize jaegerEndpoint as http://<hostport>/api/traces
		jaegerEndpoint = strings.TrimPrefix(jaegerEndpoint, "http://")
		jaegerEndpoint = strings.TrimSuffix(jaegerEndpoint, "/api/traces")
		jaegerEndpoint = fmt.Sprintf("http://%s/api/traces", jaegerEndpoint)
		cfg := jaegercfg.Configuration{
			// Configure Jaeger to sample every call, but use the SpanInclusionFunc
			// addTraceIfTracingEnabled (defined below) to skip sampling every RPC
			// unless the PACH_ENABLE_TRACING environment variable is set
			Sampler: &jaegercfg.SamplerConfig{
				Type:  "const",
				Param: 1,
			},
			Reporter: &jaegercfg.ReporterConfig{
				LogSpans:            true,
				BufferFlushInterval: 1 * time.Second,
				CollectorEndpoint:   jaegerEndpoint,
			},
		}

		// configure jaeger logger
		logger := jaeger.Logger(jaeger.NullLogger)
		if !onUserMachine {
			logger = jaeger.StdLogger
		}

		// Hack: ignore second argument (io.Closer) because the Jaeger
		// implementation of opentracing.Tracer also implements io.Closer (i.e. the
		// first and second return values from cfg.New(), here, are two interfaces
		// that wrap the same underlying type). Instead of storing the second return
		// value here, just cast the tracer to io.Closer in CloseAndReportTraces()
		// (below) and call 'Close()' on it there.
		tracer, _, err := cfg.New(JaegerServiceName, jaegercfg.Logger(logger))
		if err != nil {
			panic(fmt.Sprintf("could not install Jaeger tracer: %v", err))
		}
		opentracing.SetGlobalTracer(tracer)
	})
}

// InstallJaegerTracerFromEnv installs a Jaeger client as then opentracing
// global tracer, relying on environment variables to configure the client. It
// returns the address used to initialize the global tracer, if any
// initialization occurred

InstallJaegerTracerFromEnv installs a Jaeger client as then opentracing global tracer relying on environment variables to configure the client . It returns the address used to initialize the global tracer if any initialization occurred


func UnaryClientInterceptor() grpc.UnaryClientInterceptor {
	return otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer(),
		otgrpc.IncludingSpans(addTraceIfTracingEnabled))
}

func UnaryClientInterceptor ( ) grpc . UnaryClientInterceptor { return otgrpc . OpenTracingClientInterceptor ( opentracing . GlobalTracer ( ) , otgrpc . IncludingSpans ( addTraceIfTracingEnabled ) ) }

// UnaryClientInterceptor returns a GRPC interceptor for non-streaming GRPC RPCs

UnaryClientInterceptor returns a GRPC interceptor for non - streaming GRPC RPCs


func StreamClientInterceptor() grpc.StreamClientInterceptor {
	return otgrpc.OpenTracingStreamClientInterceptor(opentracing.GlobalTracer(),
		otgrpc.IncludingSpans(addTraceIfTracingEnabled))
}

func StreamClientInterceptor ( ) grpc . StreamClientInterceptor { return otgrpc . OpenTracingStreamClientInterceptor ( opentracing . GlobalTracer ( ) , otgrpc . IncludingSpans ( addTraceIfTracingEnabled ) ) }

// StreamClientInterceptor returns a GRPC interceptor for non-streaming GRPC RPCs

StreamClientInterceptor returns a GRPC interceptor for non - streaming GRPC RPCs


func UnaryServerInterceptor() grpc.UnaryServerInterceptor {
	return otgrpc.OpenTracingServerInterceptor(opentracing.GlobalTracer(),
		otgrpc.IncludingSpans(addTraceIfTracingEnabled))
}

func UnaryServerInterceptor ( ) grpc . UnaryServerInterceptor { return otgrpc . OpenTracingServerInterceptor ( opentracing . GlobalTracer ( ) , otgrpc . IncludingSpans ( addTraceIfTracingEnabled ) ) }

// UnaryServerInterceptor returns a GRPC interceptor for non-streaming GRPC RPCs

UnaryServerInterceptor returns a GRPC interceptor for non - streaming GRPC RPCs


func StreamServerInterceptor() grpc.StreamServerInterceptor {
	return otgrpc.OpenTracingStreamServerInterceptor(opentracing.GlobalTracer(),
		otgrpc.IncludingSpans(addTraceIfTracingEnabled))
}

func StreamServerInterceptor ( ) grpc . StreamServerInterceptor { return otgrpc . OpenTracingStreamServerInterceptor ( opentracing . GlobalTracer ( ) , otgrpc . IncludingSpans ( addTraceIfTracingEnabled ) ) }

// StreamServerInterceptor returns a GRPC interceptor for non-streaming GRPC RPCs

StreamServerInterceptor returns a GRPC interceptor for non - streaming GRPC RPCs


func CloseAndReportTraces() {
	if c, ok := opentracing.GlobalTracer().(io.Closer); ok {
		c.Close()
	}
}

func CloseAndReportTraces ( ) { if c , ok := opentracing . GlobalTracer ( ) . ( io . Closer ) ; ok { c . Close ( ) } }

// CloseAndReportTraces tries to close the global tracer, which, in the case of
// the Jaeger tracer, causes it to send any unreported traces to the collector

CloseAndReportTraces tries to close the global tracer which in the case of the Jaeger tracer causes it to send any unreported traces to the collector


func newWriter(ctx context.Context, objC obj.Client, prefix string) *Writer {
	// Initialize buzhash64 with WindowSize window.
	hash := buzhash64.New()
	hash.Write(make([]byte, WindowSize))
	return &Writer{
		ctx:       ctx,
		objC:      objC,
		prefix:    prefix,
		cbs:       []func([]*DataRef) error{},
		buf:       &bytes.Buffer{},
		hash:      hash,
		splitMask: (1 << uint64(AverageBits)) - 1,
	}
}


func (b *ConstantBackOff) For(maxElapsed time.Duration) *ConstantBackOff {
	b.MaxElapsedTime = maxElapsed
	return b
}

func ( b * ConstantBackOff ) For ( maxElapsed time . Duration ) * ConstantBackOff { b . MaxElapsedTime = maxElapsed return b }

// For sets b.MaxElapsedTime to 'maxElapsed' and returns b

For sets b . MaxElapsedTime to maxElapsed and returns b


func (l *logger) Log(request interface{}, response interface{}, err error, duration time.Duration) {
	if err != nil {
		l.LogAtLevelFromDepth(request, response, err, duration, logrus.ErrorLevel, 4)
	} else {
		l.LogAtLevelFromDepth(request, response, err, duration, logrus.InfoLevel, 4)
	}
	// We have to grab the method's name here before we
	// enter the goro's stack
	go l.ReportMetric(getMethodName(), duration, err)
}

// Helper function used to log requests and responses from our GRPC method
// implementations

Helper function used to log requests and responses from our GRPC method implementations


func (f FormatterFunc) Format(entry *logrus.Entry) ([]byte, error) {
	return f(entry)
}

func ( f FormatterFunc ) Format ( entry * logrus . Entry ) ( [ ] byte , error ) { return f ( entry ) }

// Format proxies the closure in order to satisfy `logrus.Formatter`'s
// interface.

Format proxies the closure in order to satisfy logrus . Formatter s interface .


func NewGRPCLogWriter(logger *logrus.Logger, source string) *GRPCLogWriter {
	return &GRPCLogWriter{
		logger: logger,
		source: source,
	}
}

func NewGRPCLogWriter ( logger * logrus . Logger , source string ) * GRPCLogWriter { return & GRPCLogWriter { logger : logger , source : source , } }

// NewGRPCLogWriter creates a new GRPC log writer. `logger` specifies the
// underlying logger, and `source` specifies where these logs are coming from;
// it is added as a entry field for all log messages.

NewGRPCLogWriter creates a new GRPC log writer . logger specifies the underlying logger and source specifies where these logs are coming from ; it is added as a entry field for all log messages .


func Read() (*Config, error) {
	var c *Config

	// Read json file
	p := configPath()
	if raw, err := ioutil.ReadFile(p); err == nil {
		err = json.Unmarshal(raw, &c)
		if err != nil {
			return nil, err
		}
	} else if os.IsNotExist(err) {
		// File doesn't exist, so create a new config
		fmt.Println("no config detected at %q. Generating new config...", p)
		c = &Config{}
	} else {
		return nil, fmt.Errorf("fatal: could not read config at %q: %v", p, err)
	}
	if c.UserID == "" {
		fmt.Printf("No UserID present in config. Generating new UserID and "+
			"updating config at %s\n", p)
		uuid, err := uuid.NewV4()
		if err != nil {
			return nil, err
		}
		c.UserID = uuid.String()
		if err := c.Write(); err != nil {
			return nil, err
		}
	}
	return c, nil
}

// Read loads the Pachyderm config on this machine.
// If an existing configuration cannot be found, it sets up the defaults. Read
// returns a nil Config if and only if it returns a non-nil error.

Read loads the Pachyderm config on this machine . If an existing configuration cannot be found it sets up the defaults . Read returns a nil Config if and only if it returns a non - nil error .


func (c *Config) Write() error {
	rawConfig, err := json.MarshalIndent(c, "", "  ")
	if err != nil {
		return err
	}

	// If we're not using a custom config path, create the default config path
	p := configPath()
	if _, ok := os.LookupEnv(configEnvVar); ok {
		// using overridden config path -- just make sure the parent dir exists
		d := filepath.Dir(p)
		if _, err := os.Stat(d); err != nil {
			return fmt.Errorf("cannot use config at %s: could not stat parent directory (%v)", p, err)
		}
	} else {
		// using the default config path, create the config directory
		err = os.MkdirAll(defaultConfigDir, 0755)
		if err != nil {
			return err
		}
	}
	return ioutil.WriteFile(p, rawConfig, 0644)
}

// Write writes the configuration in 'c' to this machine's Pachyderm config
// file.

Write writes the configuration in c to this machine s Pachyderm config file .


func (r *readWriter) Read(val proto.Message) error {
	buf, err := r.ReadBytes()
	if err != nil {
		return err
	}
	return proto.Unmarshal(buf, val)
}

func ( r * readWriter ) Read ( val proto . Message ) error { buf , err := r . ReadBytes ( ) if err != nil { return err } return proto . Unmarshal ( buf , val ) }


func (r *readWriter) Write(val proto.Message) (int64, error) {
	bytes, err := proto.Marshal(val)
	if err != nil {
		return 0, err
	}
	return r.WriteBytes(bytes)
}

func ( r * readWriter ) Write ( val proto . Message ) ( int64 , error ) { bytes , err := proto . Marshal ( val ) if err != nil { return 0 , err } return r . WriteBytes ( bytes ) }


func NewReadWriter(rw io.ReadWriter) ReadWriter {
	return &readWriter{r: rw, w: rw}
}

func NewReadWriter ( rw io . ReadWriter ) ReadWriter { return & readWriter { r : rw , w : rw } }

// NewReadWriter returns a new ReadWriter with rw as both its source and its sink.

NewReadWriter returns a new ReadWriter with rw as both its source and its sink .


func RunGitHookServer(address string, etcdAddress string, etcdPrefix string) error {
	c, err := client.NewFromAddress(address)
	if err != nil {
		return err
	}
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{etcdAddress},
		DialOptions: client.DefaultDialOptions(),
	})
	if err != nil {
		return err
	}
	hook, err := github.New()
	if err != nil {
		return err
	}
	s := &gitHookServer{
		hook,
		c,
		etcdClient,
		ppsdb.Pipelines(etcdClient, etcdPrefix),
	}
	return http.ListenAndServe(fmt.Sprintf(":%d", GitHookPort), s)
}

func RunGitHookServer ( address string , etcdAddress string , etcdPrefix string ) error { c , err := client . NewFromAddress ( address ) if err != nil { return err } etcdClient , err := etcd . New ( etcd . Config { Endpoints : [ ] string { etcdAddress } , DialOptions : client . DefaultDialOptions ( ) , } ) if err != nil { return err } hook , err := github . New ( ) if err != nil { return err } s := & gitHookServer { hook , c , etcdClient , ppsdb . Pipelines ( etcdClient , etcdPrefix ) , } return http . ListenAndServe ( fmt . Sprintf ( " " , GitHookPort ) , s ) }

// RunGitHookServer starts the webhook server


func newLoggingPipe() *loggingPipe {
	p := &loggingPipe{}
	p.clientReader, p.clientWriter = io.Pipe()
	p.clientReader = io.TeeReader(p.clientReader, &p.ServerToClientBuf)
	p.serverReader, p.serverWriter = io.Pipe()
	p.serverReader = io.TeeReader(p.serverReader, &p.ClientToServerBuf)
	return p
}

func newLoggingPipe ( ) * loggingPipe { p := & loggingPipe { } p . clientReader , p . clientWriter = io . Pipe ( ) p . clientReader = io . TeeReader ( p . clientReader , & p . ServerToClientBuf ) p . serverReader , p . serverWriter = io . Pipe ( ) p . serverReader = io . TeeReader ( p . serverReader , & p . ClientToServerBuf ) return p }

// newLoggingPipe initializes a loggingPipe


func (l *loggingConn) Read(b []byte) (n int, err error) {
	return l.r.Read(b)
}

func ( l * loggingConn ) Read ( b [ ] byte ) ( n int , err error ) { return l . r . Read ( b ) }

// Read implements the corresponding method of net.Conn

Read implements the corresponding method of net . Conn


func (l *loggingConn) Write(b []byte) (n int, err error) {
	return l.w.Write(b)
}

func ( l * loggingConn ) Write ( b [ ] byte ) ( n int , err error ) { return l . w . Write ( b ) }

// Write implements the corresponding method of net.Conn

Write implements the corresponding method of net . Conn


func (l *TestListener) Accept() (net.Conn, error) {
	conn := <-l.connCh
	if conn == nil {
		return nil, errors.New("Accept() has already been called on this TestListener")
	}
	return conn, nil
}

func ( l * TestListener ) Accept ( ) ( net . Conn , error ) { conn := <- l . connCh if conn == nil { return nil , errors . New ( " " ) } return conn , nil }

// Accept implements the corresponding method of net.Listener for
// TestListener

Accept implements the corresponding method of net . Listener for TestListener


func (l *TestListener) Close() error {
	l.connMu.Lock()
	defer l.connMu.Unlock()
	c := <-l.connCh
	if c != nil {
		close(l.connCh)
	}
	return nil
}

func ( l * TestListener ) Close ( ) error { l . connMu . Lock ( ) defer l . connMu . Unlock ( ) c := <- l . connCh if c != nil { close ( l . connCh ) } return nil }

// Close implements the corresponding method of net.Listener for
// TestListener. Any blocked Accept operations will be unblocked and return
// errors.

Close implements the corresponding method of net . Listener for TestListener . Any blocked Accept operations will be unblocked and return errors .


func errorf(c ErrCode, fmtStr string, args ...interface{}) error {
	return &hashTreeError{
		code: c,
		s:    fmt.Sprintf(fmtStr, args...),
	}
}

func errorf ( c ErrCode , fmtStr string , args ... interface { } ) error { return & hashTreeError { code : c , s : fmt . Sprintf ( fmtStr , args ... ) , } }

// errorf is analogous to fmt.Errorf, but generates hashTreeErrors instead of
// errorStrings.

errorf is analogous to fmt . Errorf but generates hashTreeErrors instead of errorStrings .


func InitWithKube(config *Configuration) *ServiceEnv {
	env := InitServiceEnv(config)
	env.kubeEg.Go(env.initKubeClient)
	return env // env is not ready yet
}

func
InitWithKube
(
config
*
Configuration
)
*
ServiceEnv
{
env
:=
InitServiceEnv
(
config
)
env
.
kubeEg
.
Go
(
env
.
initKubeClient
)
return
env
// env is not ready yet
}

// InitWithKube is like InitServiceEnv, but also assumes that it's run inside
// a kubernetes cluster and tries to connect to the kubernetes API server.

InitWithKube is like InitServiceEnv but also assumes that it s run inside a kubernetes cluster and tries to connect to the kubernetes API server .


func (env *ServiceEnv) GetEtcdClient() *etcd.Client {
	if err := env.etcdEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	if env.etcdClient == nil {
		panic("service env never connected to etcd")
	}
	return env.etcdClient
}

// GetEtcdClient returns the already connected etcd client without modification.

GetEtcdClient returns the already connected etcd client without modification .


func (env *ServiceEnv) GetKubeClient() *kube.Clientset {
	if err := env.kubeEg.Wait(); err != nil {
		panic(err) // If env can't connect, there's no sensible way to recover
	}
	if env.kubeClient == nil {
		panic("service env never connected to kubernetes")
	}
	return env.kubeClient
}

// GetKubeClient returns the already connected Kubernetes API client without
// modification.

GetKubeClient returns the already connected Kubernetes API client without modification .


func NewHasher(jobModulus uint64, pipelineModulus uint64) *Hasher {
	return &Hasher{
		JobModulus:      jobModulus,
		PipelineModulus: pipelineModulus,
	}
}

func NewHasher ( jobModulus uint64 , pipelineModulus uint64 ) * Hasher { return & Hasher { JobModulus : jobModulus , PipelineModulus : pipelineModulus , } }


func (s *Hasher) HashJob(jobID string) uint64 {
	return uint64(adler32.Checksum([]byte(jobID))) % s.JobModulus
}

func ( s * Hasher ) HashJob ( jobID string ) uint64 { return uint64 ( adler32 . Checksum ( [ ] byte ( jobID ) ) ) % s . JobModulus }

// HashJob computes and returns the hash of a job.

HashJob computes and returns the hash of a job .


func (s *Hasher) HashPipeline(pipelineName string) uint64 {
	return uint64(adler32.Checksum([]byte(pipelineName))) % s.PipelineModulus
}

func ( s * Hasher ) HashPipeline ( pipelineName string ) uint64 { return uint64 ( adler32 . Checksum ( [ ] byte ( pipelineName ) ) ) % s . PipelineModulus }

// HashPipeline computes and returns the hash of a pipeline.

HashPipeline computes and returns the hash of a pipeline .


func Status(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]*pps.WorkerStatus, error) {
	workerClients, err := Clients(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return nil, err
	}
	var result []*pps.WorkerStatus
	for _, workerClient := range workerClients {
		status, err := workerClient.Status(ctx, &types.Empty{})
		if err != nil {
			return nil, err
		}
		result = append(result, status)
	}
	return result, nil
}

func Status ( ctx context . Context , pipelineRcName string , etcdClient * etcd . Client , etcdPrefix string , workerGrpcPort uint16 ) ( [ ] * pps . WorkerStatus , error ) { workerClients , err := Clients ( ctx , pipelineRcName , etcdClient , etcdPrefix , workerGrpcPort ) if err != nil { return nil , err } var result [ ] * pps . WorkerStatus for _ , workerClient := range workerClients { status , err := workerClient . Status ( ctx , & types . Empty { } ) if err != nil { return nil , err } result = append ( result , status ) } return result , nil }

// Status returns the statuses of workers referenced by pipelineRcName.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.

Status returns the statuses of workers referenced by pipelineRcName . pipelineRcName is the name of the pipeline s RC and can be gotten with ppsutil . PipelineRcName . You can also pass for pipelineRcName to get all clients for all workers .


func Cancel(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client,
	etcdPrefix string, workerGrpcPort uint16, jobID string, dataFilter []string) error {
	workerClients, err := Clients(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return err
	}
	success := false
	for _, workerClient := range workerClients {
		resp, err := workerClient.Cancel(ctx, &CancelRequest{
			JobID:       jobID,
			DataFilters: dataFilter,
		})
		if err != nil {
			return err
		}
		if resp.Success {
			success = true
		}
	}
	if !success {
		return fmt.Errorf("datum matching filter %+v could not be found for jobID %s", dataFilter, jobID)
	}
	return nil
}

func Cancel ( ctx context . Context , pipelineRcName string , etcdClient * etcd . Client , etcdPrefix string , workerGrpcPort uint16 , jobID string , dataFilter [ ] string ) error { workerClients , err := Clients ( ctx , pipelineRcName , etcdClient , etcdPrefix , workerGrpcPort ) if err != nil { return err } success := false for _ , workerClient := range workerClients { resp , err := workerClient . Cancel ( ctx , & CancelRequest { JobID : jobID , DataFilters : dataFilter , } ) if err != nil { return err } if resp . Success { success = true } } if ! success { return fmt . Errorf ( " " , dataFilter , jobID ) } return nil }

// Cancel cancels a set of datums running on workers.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName.

Cancel cancels a set of datums running on workers . pipelineRcName is the name of the pipeline s RC and can be gotten with ppsutil . PipelineRcName .


func Conns(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]*grpc.ClientConn, error) {
	resp, err := etcdClient.Get(ctx, path.Join(etcdPrefix, WorkerEtcdPrefix, pipelineRcName), etcd.WithPrefix())
	if err != nil {
		return nil, err
	}
	var result []*grpc.ClientConn
	for _, kv := range resp.Kvs {
		conn, err := grpc.Dial(fmt.Sprintf("%s:%d", path.Base(string(kv.Key)), workerGrpcPort),
			append(client.DefaultDialOptions(), grpc.WithInsecure())...)
		if err != nil {
			return nil, err
		}
		result = append(result, conn)
	}
	return result, nil
}

func Conns ( ctx context . Context , pipelineRcName string , etcdClient * etcd . Client , etcdPrefix string , workerGrpcPort uint16 ) ( [ ] * grpc . ClientConn , error ) { resp , err := etcdClient . Get ( ctx , path . Join ( etcdPrefix , WorkerEtcdPrefix , pipelineRcName ) , etcd . WithPrefix ( ) ) if err != nil { return nil , err } var result [ ] * grpc . ClientConn for _ , kv := range resp . Kvs { conn , err := grpc . Dial ( fmt . Sprintf ( " " , path . Base ( string ( kv . Key ) ) , workerGrpcPort ) , append ( client . DefaultDialOptions ( ) , grpc . WithInsecure ( ) ) ... ) if err != nil { return nil , err } result = append ( result , conn ) } return result , nil }

// Conns returns a slice of connections to worker servers.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.

Conns returns a slice of connections to worker servers . pipelineRcName is the name of the pipeline s RC and can be gotten with ppsutil . PipelineRcName . You can also pass for pipelineRcName to get all clients for all workers .


func Clients(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) ([]Client, error) {
	conns, err := Conns(ctx, pipelineRcName, etcdClient, etcdPrefix, workerGrpcPort)
	if err != nil {
		return nil, err
	}
	var result []Client
	for _, conn := range conns {
		result = append(result, newClient(conn))
	}
	return result, nil
}

func Clients ( ctx context . Context , pipelineRcName string , etcdClient * etcd . Client , etcdPrefix string , workerGrpcPort uint16 ) ( [ ] Client , error ) { conns , err := Conns ( ctx , pipelineRcName , etcdClient , etcdPrefix , workerGrpcPort ) if err != nil { return nil , err } var result [ ] Client for _ , conn := range conns { result = append ( result , newClient ( conn ) ) } return result , nil }

// Clients returns a slice of worker clients for a pipeline.
// pipelineRcName is the name of the pipeline's RC and can be gotten with
// ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all
// clients for all workers.

Clients returns a slice of worker clients for a pipeline . pipelineRcName is the name of the pipeline s RC and can be gotten with ppsutil . PipelineRcName . You can also pass for pipelineRcName to get all clients for all workers .


func NewClient(address string) (Client, error) {
	port, err := strconv.Atoi(os.Getenv(client.PPSWorkerPortEnv))
	if err != nil {
		return Client{}, err
	}
	conn, err := grpc.Dial(fmt.Sprintf("%s:%d", address, port),
		append(client.DefaultDialOptions(), grpc.WithInsecure())...)
	if err != nil {
		return Client{}, err
	}
	return newClient(conn), nil
}

func NewClient ( address string ) ( Client , error ) { port , err := strconv . Atoi ( os . Getenv ( client . PPSWorkerPortEnv ) ) if err != nil { return Client { } , err } conn , err := grpc . Dial ( fmt . Sprintf ( " " , address , port ) , append ( client . DefaultDialOptions ( ) , grpc . WithInsecure ( ) ) ... ) if err != nil { return Client { } , err } return newClient ( conn ) , nil }

// NewClient returns a worker client for the worker at the IP address passed in.

NewClient returns a worker client for the worker at the IP address passed in .


func RunFixedArgs(numArgs int, run func([]string) error) func(*cobra.Command, []string) {
	return func(cmd *cobra.Command, args []string) {
		if len(args) != numArgs {
			fmt.Printf("expected %d arguments, got %d\n\n", numArgs, len(args))
			cmd.Usage()
		} else {
			if err := run(args); err != nil {
				ErrorAndExit("%v", err)
			}
		}
	}
}

func RunFixedArgs ( numArgs int , run func ( [ ] string ) error ) func ( * cobra . Command , [ ] string ) { return func ( cmd * cobra . Command , args [ ] string ) { if len ( args ) != numArgs { fmt . Printf ( " \n \n " , numArgs , len ( args ) ) cmd . Usage ( ) } else { if err := run ( args ) ; err != nil { ErrorAndExit ( " " , err ) } } } }

// RunFixedArgs wraps a function in a function
// that checks its exact argument count.

RunFixedArgs wraps a function in a function that checks its exact argument count .


func RunBoundedArgs(min int, max int, run func([]string) error) func(*cobra.Command, []string) {
	return func(cmd *cobra.Command, args []string) {
		if len(args) < min || len(args) > max {
			fmt.Printf("expected %d to %d arguments, got %d\n\n", min, max, len(args))
			cmd.Usage()
		} else {
			if err := run(args); err != nil {
				ErrorAndExit("%v", err)
			}
		}
	}
}

func RunBoundedArgs ( min int , max int , run func ( [ ] string ) error ) func ( * cobra . Command , [ ] string ) { return func ( cmd * cobra . Command , args [ ] string ) { if len ( args ) < min || len ( args ) > max { fmt . Printf ( " \n \n " , min , max , len ( args ) ) cmd . Usage ( ) } else { if err := run ( args ) ; err != nil { ErrorAndExit ( " " , err ) } } } }

// RunBoundedArgs wraps a function in a function
// that checks its argument count is within a range.

RunBoundedArgs wraps a function in a function that checks its argument count is within a range .


func Run(run func(args []string) error) func(*cobra.Command, []string) {
	return func(_ *cobra.Command, args []string) {
		if err := run(args); err != nil {
			ErrorAndExit(err.Error())
		}
	}
}

func Run ( run func ( args [ ] string ) error ) func ( * cobra . Command , [ ] string ) { return func ( _ * cobra . Command , args [ ] string ) { if err := run ( args ) ; err != nil { ErrorAndExit ( err . Error ( ) ) } } }

// Run makes a new cobra run function that wraps the given function.

Run makes a new cobra run function that wraps the given function .


func ErrorAndExit(format string, args ...interface{}) {
	if errString := strings.TrimSpace(fmt.Sprintf(format, args...)); errString != "" {
		fmt.Fprintf(os.Stderr, "%s\n", errString)
	}
	os.Exit(1)
}

func ErrorAndExit ( format string , args ... interface { } ) { if errString := strings . TrimSpace ( fmt . Sprintf ( format , args ... ) ) ; errString != " " { fmt . Fprintf ( os . Stderr , " \n " , errString ) } os . Exit ( 1 ) }

// ErrorAndExit errors with the given format and args, and then exits.

ErrorAndExit errors with the given format and args and then exits .


func ParseCommit(arg string) (*pfs.Commit, error) {
	parts := strings.SplitN(arg, "@", 2)
	if parts[0] == "" {
		return nil, fmt.Errorf("invalid format \"%s\": repo cannot be empty", arg)
	}
	commit := &pfs.Commit{
		Repo: &pfs.Repo{
			Name: parts[0],
		},
		ID: "",
	}
	if len(parts) == 2 {
		commit.ID = parts[1]
	}
	return commit, nil
}

func ParseCommit ( arg string ) ( * pfs . Commit , error ) { parts := strings . SplitN ( arg , " " , 2 ) if parts [ 0 ] == " " { return nil , fmt . Errorf ( " \" \" " , arg ) } commit := & pfs . Commit { Repo : & pfs . Repo { Name : parts [ 0 ] , } , ID : " " , } if len ( parts ) == 2 { commit . ID = parts [ 1 ] } return commit , nil }

// ParseCommit takes an argument of the form "repo[@branch-or-commit]" and
// returns the corresponding *pfs.Commit.

ParseCommit takes an argument of the form repo [


func ParseBranch(arg string) (*pfs.Branch, error) {
	commit, err := ParseCommit(arg)
	if err != nil {
		return nil, err
	}
	return &pfs.Branch{Repo: commit.Repo, Name: commit.ID}, nil
}

func ParseBranch ( arg string ) ( * pfs . Branch , error ) { commit , err := ParseCommit ( arg ) if err != nil { return nil , err } return & pfs . Branch { Repo : commit . Repo , Name : commit . ID } , nil }

// ParseBranch takes an argument of the form "repo[@branch]" and
// returns the corresponding *pfs.Branch.  This uses ParseCommit under the hood
// because a branch name is usually interchangeable with a commit-id.

ParseBranch takes an argument of the form repo [


func ParseFile(arg string) (*pfs.File, error) {
	repoAndRest := strings.SplitN(arg, "@", 2)
	if repoAndRest[0] == "" {
		return nil, fmt.Errorf("invalid format \"%s\": repo cannot be empty", arg)
	}
	file := &pfs.File{
		Commit: &pfs.Commit{
			Repo: &pfs.Repo{
				Name: repoAndRest[0],
			},
			ID: "",
		},
		Path: "",
	}
	if len(repoAndRest) > 1 {
		commitAndPath := strings.SplitN(repoAndRest[1], ":", 2)
		if commitAndPath[0] == "" {
			return nil, fmt.Errorf("invalid format \"%s\": commit cannot be empty", arg)
		}
		file.Commit.ID = commitAndPath[0]
		if len(commitAndPath) > 1 {
			file.Path = commitAndPath[1]
		}
	}
	return file, nil
}

func ParseFile ( arg string ) ( * pfs . File , error ) { repoAndRest := strings . SplitN ( arg , " " , 2 ) if repoAndRest [ 0 ] == " " { return nil , fmt . Errorf ( " \" \" " , arg ) } file := & pfs . File { Commit : & pfs . Commit { Repo : & pfs . Repo { Name : repoAndRest [ 0 ] , } , ID : " " , } , Path : " " , } if len ( repoAndRest ) > 1 { commitAndPath := strings . SplitN ( repoAndRest [ 1 ] , " " , 2 ) if commitAndPath [ 0 ] == " " { return nil , fmt . Errorf ( " \" \" " , arg ) } file . Commit . ID = commitAndPath [ 0 ] if len ( commitAndPath ) > 1 { file . Path = commitAndPath [ 1 ] } } return file , nil }

// ParseFile takes an argument of the form "repo[@branch-or-commit[:path]]", and
// returns the corresponding *pfs.File.

ParseFile takes an argument of the form repo [


func (r *RepeatedStringArg) Set(s string) error {
	*r = append(*r, s)
	return nil
}

func ( r * RepeatedStringArg ) Set ( s string ) error { * r = append ( * r , s ) return nil }


func SetDocsUsage(command *cobra.Command) {
	command.SetHelpTemplate(`{{or .Long .Short}}

{{.UsageString}}
`)

	command.SetUsageFunc(func(cmd *cobra.Command) error {
		rootCmd := cmd.Root()

		// Walk the command tree, finding commands with the documented word
		var associated []*cobra.Command
		var walk func(*cobra.Command)
		walk = func(cursor *cobra.Command) {
			if cursor.Name() == cmd.Name() && cursor.CommandPath() != cmd.CommandPath() {
				associated = append(associated, cursor)
			}
			for _, subcmd := range cursor.Commands() {
				walk(subcmd)
			}
		}
		walk(rootCmd)

		var maxCommandPath int
		for _, x := range associated {
			commandPathLen := len(x.CommandPath())
			if commandPathLen > maxCommandPath {
				maxCommandPath = commandPathLen
			}
		}

		templateFuncs := template.FuncMap{
			"pad": func(s string) string {
				format := fmt.Sprintf("%%-%ds", maxCommandPath+1)
				return fmt.Sprintf(format, s)
			},
			"associated": func() []*cobra.Command {
				return associated
			},
		}

		text := `Associated Commands:{{range associated}}{{if .IsAvailableCommand}}
  {{pad .CommandPath}} {{.Short}}{{end}}{{end}}`

		t := template.New("top")
		t.Funcs(templateFuncs)
		template.Must(t.Parse(text))
		return t.Execute(cmd.Out(), cmd)
	})
}

`{{or .Long .Short}}

{{.UsageString}}
`

// SetDocsUsage sets the usage string for a docs-style command.  Docs commands
// have no functionality except to output some docs and related commands, and
// should not specify a 'Run' attribute.

SetDocsUsage sets the usage string for a docs - style command . Docs commands have no functionality except to output some docs and related commands and should not specify a Run attribute .


func (a *apiServer) makeCronCommits(pachClient *client.APIClient, in *pps.Input) error {
	schedule, err := cron.ParseStandard(in.Cron.Spec)
	if err != nil {
		return err // Shouldn't happen, as the input is validated in CreatePipeline
	}
	// make sure there isn't an unfinished commit on the branch
	commitInfo, err := pachClient.InspectCommit(in.Cron.Repo, "master")
	if err != nil && !isNilBranchErr(err) {
		return err
	} else if commitInfo != nil && commitInfo.Finished == nil {
		// and if there is, delete it
		if err = pachClient.DeleteCommit(in.Cron.Repo, "master"); err != nil {
			return err
		}
	}

	var latestTime time.Time
	files, err := pachClient.ListFile(in.Cron.Repo, "master", "")
	if err != nil && !isNilBranchErr(err) {
		return err
	} else if err != nil || len(files) == 0 {
		// File not found, this happens the first time the pipeline is run
		latestTime, err = types.TimestampFromProto(in.Cron.Start)
		if err != nil {
			return err
		}
	} else {
		// Take the name of the most recent file as the latest timestamp
		// ListFile returns the files in lexicographical order, and the RFC3339 format goes
		// from largest unit of time to smallest, so the most recent file will be the last one
		latestTime, err = time.Parse(time.RFC3339, path.Base(files[len(files)-1].File.Path))
		if err != nil {
			return err
		}
	}

	for {
		// get the time of the next time from the latest time using the cron schedule
		next := schedule.Next(latestTime)
		// and wait until then to make the next commit
		time.Sleep(time.Until(next))
		if err != nil {
			return err
		}

		// We need the DeleteFile and the PutFile to happen in the same commit
		_, err = pachClient.StartCommit(in.Cron.Repo, "master")
		if err != nil {
			return err
		}
		if in.Cron.Overwrite {
			// If we want to "overwrite" the file, we need to delete the file with the previous time
			err := pachClient.DeleteFile(in.Cron.Repo, "master", latestTime.Format(time.RFC3339))
			if err != nil && !isNotFoundErr(err) && !isNilBranchErr(err) {
				return fmt.Errorf("delete error %v", err)
			}
		}

		// Put in an empty file named by the timestamp
		_, err = pachClient.PutFile(in.Cron.Repo, "master", next.Format(time.RFC3339), strings.NewReader(""))
		if err != nil {
			return fmt.Errorf("put error %v", err)
		}

		err = pachClient.FinishCommit(in.Cron.Repo, "master")
		if err != nil {
			return err
		}

		// set latestTime to the next time
		latestTime = next
	}
}

// makeCronCommits makes commits to a single cron input's repo. It's
// a helper function called by monitorPipeline.

makeCronCommits makes commits to a single cron input s repo . It s a helper function called by monitorPipeline .


func (o *tracingObjClient) Writer(ctx context.Context, name string) (io.WriteCloser, error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx, o.provider+".Writer", "name", name)
	if span != nil {
		defer span.Finish()
	}
	return o.Client.Writer(ctx, name)
}

func ( o * tracingObjClient ) Writer ( ctx context . Context , name string ) ( io . WriteCloser , error ) { span , ctx := tracing . AddSpanToAnyExisting ( ctx , o . provider + " " , " " , name ) if span != nil { defer span . Finish ( ) } return o . Client . Writer ( ctx , name ) }

// Writer implements the corresponding method in the Client interface


func (o *tracingObjClient) Reader(ctx context.Context, name string, offset uint64, size uint64) (io.ReadCloser, error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx, o.provider+".Reader",
		"name", name,
		"offset", fmt.Sprintf("%d", offset),
		"size", fmt.Sprintf("%d", size))
	defer tracing.FinishAnySpan(span)
	return o.Client.Reader(ctx, name, offset, size)
}

func ( o * tracingObjClient ) Reader ( ctx context . Context , name string , offset uint64 , size uint64 ) ( io . ReadCloser , error ) { span , ctx := tracing . AddSpanToAnyExisting ( ctx , o . provider + " " , " " , name , " " , fmt . Sprintf ( " " , offset ) , " " , fmt . Sprintf ( " " , size ) ) defer tracing . FinishAnySpan ( span ) return o . Client . Reader ( ctx , name , offset , size ) }

// Reader implements the corresponding method in the Client interface


func (o *tracingObjClient) Delete(ctx context.Context, name string) error {
	span, ctx := tracing.AddSpanToAnyExisting(ctx, o.provider+".Delete",
		"name", name)
	defer tracing.FinishAnySpan(span)
	return o.Client.Delete(ctx, name)
}

func ( o * tracingObjClient ) Delete ( ctx context . Context , name string ) error { span , ctx := tracing . AddSpanToAnyExisting ( ctx , o . provider + " " , " " , name ) defer tracing . FinishAnySpan ( span ) return o . Client . Delete ( ctx , name ) }

// Delete implements the corresponding method in the Client interface


func (o *tracingObjClient) Walk(ctx context.Context, prefix string, fn func(name string) error) error {
	span, ctx := tracing.AddSpanToAnyExisting(ctx, o.provider+".Walk",
		"prefix", prefix)
	defer tracing.FinishAnySpan(span)
	return o.Client.Walk(ctx, prefix, fn)
}

func ( o * tracingObjClient ) Walk ( ctx context . Context , prefix string , fn func ( name string ) error ) error { span , ctx := tracing . AddSpanToAnyExisting ( ctx , o . provider + " " , " " , prefix ) defer tracing . FinishAnySpan ( span ) return o . Client . Walk ( ctx , prefix , fn ) }

// Walk implements the corresponding method in the Client interface


func (o *tracingObjClient) Exists(ctx context.Context, name string) bool {
	span, ctx := tracing.AddSpanToAnyExisting(ctx, o.provider+".Exists",
		"name", name)
	defer tracing.FinishAnySpan(span)
	return o.Client.Exists(ctx, name)
}

func ( o * tracingObjClient ) Exists ( ctx context . Context , name string ) bool { span , ctx := tracing . AddSpanToAnyExisting ( ctx , o . provider + " " , " " , name ) defer tracing . FinishAnySpan ( span ) return o . Client . Exists ( ctx , name ) }

// Exists implements the corresponding method in the Client interface


func GetBlock(hash hash.Hash) *Block {
	return &Block{
		Hash: base64.URLEncoding.EncodeToString(hash.Sum(nil)),
	}
}

func GetBlock ( hash hash . Hash ) * Block { return & Block { Hash : base64 . URLEncoding . EncodeToString ( hash . Sum ( nil ) ) , } }

// GetBlock encodes a hash into a readable format in the form of a Block.

GetBlock encodes a hash into a readable format in the form of a Block .


func (h *healthServer) Health(context.Context, *types.Empty) (*types.Empty, error) {
	if !h.ready {
		return nil, fmt.Errorf("server not ready")
	}
	return &types.Empty{}, nil
}

func ( h * healthServer ) Health ( context . Context , * types . Empty ) ( * types . Empty , error ) { if ! h . ready { return nil , fmt . Errorf ( " " ) } return & types . Empty { } , nil }

// Health implements the Health method for healthServer.

Health implements the Health method for healthServer .


func split(p string) (string, string) {
	return clean(path.Dir(p)), base(p)
}

func split ( p string ) ( string , string ) { return clean ( path . Dir ( p ) ) , base ( p ) }

// split is like path.Split, but uses this library's defaults for canonical
// paths

split is like path . Split but uses this library s defaults for canonical paths


func ValidatePath(path string) error {
	path = clean(path)
	match, _ := regexp.MatchString("^[ -~]+$", path)

	if !match {
		return fmt.Errorf("path (%v) invalid: only printable ASCII characters allowed", path)
	}

	if IsGlob(path) {
		return fmt.Errorf("path (%v) invalid: globbing character (%v) not allowed in path", path, globRegex.FindString(path))
	}

	return nil
}

func ValidatePath ( path string ) error { path = clean ( path ) match , _ := regexp . MatchString ( " " , path ) if ! match { return fmt . Errorf ( " " , path ) } if IsGlob ( path ) { return fmt . Errorf ( " " , path , globRegex . FindString ( path ) ) } return nil }

// ValidatePath checks if a file path is legal


func MatchDatum(filter []string, data []*pps.InputFile) bool {
	// All paths in request.DataFilters must appear somewhere in the log
	// line's inputs, or it's filtered
	matchesData := true
dataFilters:
	for _, dataFilter := range filter {
		for _, datum := range data {
			if dataFilter == datum.Path ||
				dataFilter == base64.StdEncoding.EncodeToString(datum.Hash) ||
				dataFilter == hex.EncodeToString(datum.Hash) {
				continue dataFilters // Found, move to next filter
			}
		}
		matchesData = false
		break
	}
	return matchesData
}

// MatchDatum checks if a datum matches a filter.  To match each string in
// filter must correspond match at least 1 datum's Path or Hash. Order of
// filter and data is irrelevant.

MatchDatum checks if a datum matches a filter . To match each string in filter must correspond match at least 1 datum s Path or Hash . Order of filter and data is irrelevant .


func NewCacheServer(router shard.Router, shards uint64) CacheServer {
	server := &groupCacheServer{
		Logger:      log.NewLogger("CacheServer"),
		router:      router,
		localShards: make(map[uint64]bool),
		shards:      shards,
	}
	groupcache.RegisterPeerPicker(func() groupcache.PeerPicker { return server })
	return server
}

func NewCacheServer ( router shard . Router , shards uint64 ) CacheServer { server := & groupCacheServer { Logger : log . NewLogger ( " " ) , router : router , localShards : make ( map [ uint64 ] bool ) , shards : shards , } groupcache . RegisterPeerPicker ( func ( ) groupcache . PeerPicker { return server } ) return server }

// NewCacheServer creates a new CacheServer.

NewCacheServer creates a new CacheServer .


func (a *apiServer) authorizePipelineOp(pachClient *client.APIClient, operation pipelineOperation, input *pps.Input, output string) error {
	ctx := pachClient.Ctx()
	me, err := pachClient.WhoAmI(ctx, &auth.WhoAmIRequest{})
	if auth.IsErrNotActivated(err) {
		return nil // Auth isn't activated, skip authorization completely
	} else if err != nil {
		return err
	}

	if input != nil {
		// Check that the user is authorized to read all input repos, and write to the
		// output repo (which the pipeline needs to be able to do on the user's
		// behalf)
		var eg errgroup.Group
		done := make(map[string]struct{}) // don't double-authorize repos
		pps.VisitInput(input, func(in *pps.Input) {
			var repo string

			if in.Pfs != nil {
				repo = in.Pfs.Repo
			} else {
				return
			}

			if _, ok := done[repo]; ok {
				return
			}
			done[repo] = struct{}{}
			eg.Go(func() error {
				resp, err := pachClient.Authorize(ctx, &auth.AuthorizeRequest{
					Repo:  repo,
					Scope: auth.Scope_READER,
				})
				if err != nil {
					return err
				}
				if !resp.Authorized {
					return &auth.ErrNotAuthorized{
						Subject:  me.Username,
						Repo:     repo,
						Required: auth.Scope_READER,
					}
				}
				return nil
			})
		})
		if err := eg.Wait(); err != nil {
			return err
		}
	}

	// Check that the user is authorized to write to the output repo.
	// Note: authorizePipelineOp is called before CreateRepo creates a
	// PipelineInfo proto in etcd, so PipelineManager won't have created an output
	// repo yet, and it's possible to check that the output repo doesn't exist
	// (if it did exist, we'd have to check that the user has permission to write
	// to it, and this is simpler)
	var required auth.Scope
	switch operation {
	case pipelineOpCreate:
		if _, err := pachClient.InspectRepo(output); err == nil {
			return fmt.Errorf("cannot overwrite repo \"%s\" with new output repo", output)
		} else if !isNotFoundErr(err) {
			return err
		}
	case pipelineOpListDatum, pipelineOpGetLogs:
		required = auth.Scope_READER
	case pipelineOpUpdate:
		required = auth.Scope_WRITER
	case pipelineOpDelete:
		required = auth.Scope_OWNER
	default:
		return fmt.Errorf("internal error, unrecognized operation %v", operation)
	}
	if required != auth.Scope_NONE {
		resp, err := pachClient.Authorize(ctx, &auth.AuthorizeRequest{
			Repo:  output,
			Scope: required,
		})
		if err != nil {
			return err
		}
		if !resp.Authorized {
			return &auth.ErrNotAuthorized{
				Subject:  me.Username,
				Repo:     output,
				Required: required,
			}
		}
	}
	return nil
}

// authorizePipelineOp checks if the user indicated by 'ctx' is authorized
// to perform 'operation' on the pipeline in 'info'

authorizePipelineOp checks if the user indicated by ctx is authorized to perform operation on the pipeline in info


func (a *apiServer) sudo(pachClient *client.APIClient, f func(*client.APIClient) error) error {
	// Get PPS auth token
	superUserTokenOnce.Do(func() {
		b := backoff.NewExponentialBackOff()
		b.MaxElapsedTime = 60 * time.Second
		b.MaxInterval = 5 * time.Second
		if err := backoff.Retry(func() error {
			superUserTokenCol := col.NewCollection(a.env.GetEtcdClient(), ppsconsts.PPSTokenKey, nil, &types.StringValue{}, nil, nil).ReadOnly(pachClient.Ctx())
			var result types.StringValue
			if err := superUserTokenCol.Get("", &result); err != nil {
				return err
			}
			superUserToken = result.Value
			return nil
		}, b); err != nil {
			panic(fmt.Sprintf("couldn't get PPS superuser token: %v", err))
		}
	})

	// Copy pach client, but keep ctx (to propagate cancellation). Replace token
	// with superUserToken
	superUserClient := pachClient.WithCtx(pachClient.Ctx())
	superUserClient.SetAuthToken(superUserToken)
	return f(superUserClient)
}

// sudo is a helper function that copies 'pachClient' grants it PPS's superuser
// token, and calls 'f' with the superuser client. This helps isolate PPS's use
// of its superuser token so that it's not widely copied and is unlikely to
// leak authority to parts of the code that aren't supposed to have it.
//
// Note that because the argument to 'f' is a superuser client, it should not
// be used to make any calls with unvalidated user input. Any such use could be
// exploited to make PPS a confused deputy

sudo is a helper function that copies pachClient grants it PPS s superuser token and calls f with the superuser client . This helps isolate PPS s use of its superuser token so that it s not widely copied and is unlikely to leak authority to parts of the code that aren t supposed to have it . Note that because the argument to f is a superuser client it should not be used to make any calls with unvalidated user input . Any such use could be exploited to make PPS a confused deputy


func setPipelineDefaults(pipelineInfo *pps.PipelineInfo) {
	now := time.Now()
	if pipelineInfo.Transform.Image == "" {
		pipelineInfo.Transform.Image = DefaultUserImage
	}
	pps.VisitInput(pipelineInfo.Input, func(input *pps.Input) {
		if input.Pfs != nil {
			if input.Pfs.Branch == "" {
				input.Pfs.Branch = "master"
			}
			if input.Pfs.Name == "" {
				input.Pfs.Name = input.Pfs.Repo
			}
		}
		if input.Cron != nil {
			if input.Cron.Start == nil {
				start, _ := types.TimestampProto(now)
				input.Cron.Start = start
			}
			if input.Cron.Repo == "" {
				input.Cron.Repo = fmt.Sprintf("%s_%s", pipelineInfo.Pipeline.Name, input.Cron.Name)
			}
		}
		if input.Git != nil {
			if input.Git.Branch == "" {
				input.Git.Branch = "master"
			}
			if input.Git.Name == "" {
				// We know URL looks like:
				// "https://github.com/sjezewski/testgithook.git",
				tokens := strings.Split(path.Base(input.Git.URL), ".")
				input.Git.Name = tokens[0]
			}
		}
	})
	if pipelineInfo.OutputBranch == "" {
		// Output branches default to master
		pipelineInfo.OutputBranch = "master"
	}
	if pipelineInfo.CacheSize == "" {
		pipelineInfo.CacheSize = "64M"
	}
	if pipelineInfo.ResourceRequests == nil && pipelineInfo.CacheSize != "" {
		pipelineInfo.ResourceRequests = &pps.ResourceSpec{
			Memory: pipelineInfo.CacheSize,
		}
	}
	if pipelineInfo.MaxQueueSize < 1 {
		pipelineInfo.MaxQueueSize = 1
	}
	if pipelineInfo.DatumTries == 0 {
		pipelineInfo.DatumTries = DefaultDatumTries
	}
}

// setPipelineDefaults sets the default values for a pipeline info


func (a *apiServer) incrementGCGeneration(ctx context.Context) error {
	resp, err := a.env.GetEtcdClient().Get(ctx, client.GCGenerationKey)
	if err != nil {
		return err
	}

	if resp.Count == 0 {
		// If the generation number does not exist, create it.
		// It's important that the new generation is 1, as the first
		// generation is assumed to be 0.
		if _, err := a.env.GetEtcdClient().Put(ctx, client.GCGenerationKey, "1"); err != nil {
			return err
		}
	} else {
		oldGen, err := strconv.Atoi(string(resp.Kvs[0].Value))
		if err != nil {
			return err
		}
		newGen := oldGen + 1
		if _, err := a.env.GetEtcdClient().Put(ctx, client.GCGenerationKey, strconv.Itoa(newGen)); err != nil {
			return err
		}
	}
	return nil
}

// incrementGCGeneration increments the GC generation number in etcd


func NewDebugServer(name string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16) debug.DebugServer {
	return &debugServer{
		name:           name,
		etcdClient:     etcdClient,
		etcdPrefix:     etcdPrefix,
		workerGrpcPort: workerGrpcPort,
	}
}

func NewDebugServer ( name string , etcdClient * etcd . Client , etcdPrefix string , workerGrpcPort uint16 ) debug . DebugServer { return & debugServer { name : name , etcdClient : etcdClient , etcdPrefix : etcdPrefix , workerGrpcPort : workerGrpcPort , } }

// NewDebugServer creates a new server that serves the debug api over GRPC


func (c APIClient) Health() error {
	_, err := c.healthClient.Health(c.Ctx(), &types.Empty{})
	return grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) Health ( ) error { _ , err := c . healthClient . Health ( c . Ctx ( ) , & types . Empty { } ) return grpcutil . ScrubGRPC ( err ) }

// Health health checks pachd, it returns an error if pachd isn't healthy.

Health health checks pachd it returns an error if pachd isn t healthy .


func newObjBlockAPIServer(dir string, cacheBytes int64, etcdAddress string, objClient obj.Client, test bool) (*objBlockAPIServer, error) {
	// defensive measure to make sure storage is working and error early if it's not
	// this is where we'll find out if the credentials have been misconfigured
	if err := obj.TestStorage(context.Background(), objClient); err != nil {
		return nil, err
	}
	oneCacheShare := cacheBytes / (objectCacheShares + tagCacheShares + objectInfoCacheShares + blockCacheShares)
	s := &objBlockAPIServer{
		Logger:           log.NewLogger("pfs.BlockAPI.Obj"),
		dir:              dir,
		objClient:        objClient,
		objectIndexes:    make(map[string]*pfsclient.ObjectIndex),
		objectCacheBytes: oneCacheShare * objectCacheShares,
	}

	objectGroupName := "object"
	tagGroupName := "tag"
	objectInfoGroupName := "objectInfo"
	blockGroupName := "block"

	if test {
		uuid := uuid.New()
		objectGroupName += uuid
		tagGroupName += uuid
		objectInfoGroupName += uuid
		blockGroupName += uuid
	}

	s.objectCache = groupcache.NewGroup(objectGroupName, oneCacheShare*objectCacheShares, groupcache.GetterFunc(s.objectGetter))
	s.tagCache = groupcache.NewGroup(tagGroupName, oneCacheShare*tagCacheShares, groupcache.GetterFunc(s.tagGetter))
	s.objectInfoCache = groupcache.NewGroup(objectInfoGroupName, oneCacheShare*objectInfoCacheShares, groupcache.GetterFunc(s.objectInfoGetter))
	s.blockCache = groupcache.NewGroup(blockGroupName, oneCacheShare*blockCacheShares, groupcache.GetterFunc(s.blockGetter))

	if !test {
		RegisterCacheStats("tag", &s.tagCache.Stats)
		RegisterCacheStats("object", &s.objectCache.Stats)
		RegisterCacheStats("object_info", &s.objectInfoCache.Stats)
	}

	go s.watchGC(etcdAddress)
	return s, nil
}

// In test mode, we use unique names for cache groups, since we might want
// to run multiple block servers locally, which would conflict if groups
// had the same name. We also do not report stats to prometheus

In test mode we use unique names for cache groups since we might want to run multiple block servers locally which would conflict if groups had the same name . We also do not report stats to prometheus


func (s *objBlockAPIServer) watchGC(etcdAddress string) {
	b := backoff.NewInfiniteBackOff()
	backoff.RetryNotify(func() error {
		etcdClient, err := etcd.New(etcd.Config{
			Endpoints:   []string{etcdAddress},
			DialOptions: client.DefaultDialOptions(),
		})
		if err != nil {
			return fmt.Errorf("error instantiating etcd client: %v", err)
		}

		watcher, err := watch.NewWatcher(context.Background(), etcdClient, "", client.GCGenerationKey, nil)
		if err != nil {
			return fmt.Errorf("error instantiating watch stream from generation number: %v", err)
		}
		defer watcher.Close()

		for {
			ev, ok := <-watcher.Watch()
			if ev.Err != nil {
				return fmt.Errorf("error from generation number watch: %v", ev.Err)
			}
			if !ok {
				return fmt.Errorf("generation number watch stream closed unexpectedly")
			}
			newGen, err := strconv.Atoi(string(ev.Value))
			if err != nil {
				return fmt.Errorf("error converting the generation number: %v", err)
			}
			s.setGeneration(newGen)
		}
	}, b, func(err error, d time.Duration) error {
		logrus.Errorf("error running GC watcher in block server: %v; retrying in %s", err, d)
		return nil
	})
}

func ( s * objBlockAPIServer ) watchGC ( etcdAddress string ) { b := backoff . NewInfiniteBackOff ( ) backoff . RetryNotify ( func ( ) error { etcdClient , err := etcd . New ( etcd . Config { Endpoints : [ ] string { etcdAddress } , DialOptions : client . DefaultDialOptions ( ) , } ) if err != nil { return fmt . Errorf ( " " , err ) } watcher , err := watch . NewWatcher ( context . Background ( ) , etcdClient , " " , client . GCGenerationKey , nil ) if err != nil { return fmt . Errorf ( " " , err ) } defer watcher . Close ( ) for { ev , ok := <- watcher . Watch ( ) if ev . Err != nil { return fmt . Errorf ( " " , ev . Err ) } if ! ok { return fmt . Errorf ( " " ) } newGen , err := strconv . Atoi ( string ( ev . Value ) ) if err != nil { return fmt . Errorf ( " " , err ) } s . setGeneration ( newGen ) } } , b , func ( err error , d time . Duration ) error { logrus . Errorf ( " " , err , d ) return nil } ) }

// watchGC watches for GC runs and invalidate all cache when GC happens.

watchGC watches for GC runs and invalidate all cache when GC happens .


func (s *objBlockAPIServer) splitKey(key string) string {
	gen := s.getGeneration()
	if len(key) < prefixLength {
		return fmt.Sprintf("%s.%d", key, gen)
	}
	return fmt.Sprintf("%s.%s.%d", key[:prefixLength], key[prefixLength:], gen)
}

func ( s * objBlockAPIServer ) splitKey ( key string ) string { gen := s . getGeneration ( ) if len ( key ) < prefixLength { return fmt . Sprintf ( " " , key , gen ) } return fmt . Sprintf ( " " , key [ : prefixLength ] , key [ prefixLength : ] , gen ) }

// splitKey splits a key into the format we want, and also postpends
// the generation number

splitKey splits a key into the format we want and also postpends the generation number


func NewWriter(w io.Writer, header string) *Writer {
	if header[len(header)-1] != '\n' {
		panic("header must end in a new line")
	}
	tabwriter := ansiterm.NewTabWriter(w, 0, 1, 1, ' ', 0)
	tabwriter.Write([]byte(header))
	return &Writer{
		w:      tabwriter,
		lines:  1, // 1 because we just printed the header
		header: []byte(header),
	}
}

func
NewWriter
(
w
io
.
Writer
,
header
string
)
*
Writer
{
if
header
[
len
(
header
)
1
]
!=
'\n'
{
panic
(
"
"
)
}
tabwriter
:=
ansiterm
.
NewTabWriter
(
w
,
0
,
1
,
1
,
' '
,
0
)
tabwriter
.
Write
(
[
]
byte
(
header
)
)
return
&
Writer
{
w
:
tabwriter
,
lines
:
1
,
// 1 because we just printed the header
header
:
[
]
byte
(
header
)
,
}
}

// NewWriter returns a new Writer, it will flush when
// it gets termHeight many lines, including the header line.
// The header line will be reprinted termHeight many lines have been written.
// NewStreamingWriter will panic if it's given a header that doesn't end in \n.

NewWriter returns a new Writer it will flush when it gets termHeight many lines including the header line . The header line will be reprinted termHeight many lines have been written . NewStreamingWriter will panic if it s given a header that doesn t end in \ n .


func (w *Writer) Write(buf []byte) (int, error) {
	if w.lines >= termHeight {
		if err := w.Flush(); err != nil {
			return 0, err
		}
		if _, err := w.w.Write(w.header); err != nil {
			return 0, err
		}
		w.lines++
	}
	w.lines += bytes.Count(buf, []byte{'\n'})
	return w.w.Write(buf)
}

func ( w * Writer ) Write ( buf [ ] byte ) ( int , error ) { if w . lines >= termHeight { if err := w . Flush ( ) ; err != nil { return 0 , err } if _ , err := w . w . Write ( w . header ) ; err != nil { return 0 , err } w . lines ++ } w . lines += bytes . Count ( buf , [ ] byte { '\n' } ) return w . w . Write ( buf ) }

// Write writes a line to the tabwriter.


func PrintRepoHeader(w io.Writer, printAuth bool) {
	if printAuth {
		fmt.Fprint(w, RepoAuthHeader)
		return
	}
	fmt.Fprint(w, RepoHeader)
}

func PrintRepoHeader ( w io . Writer , printAuth bool ) { if printAuth { fmt . Fprint ( w , RepoAuthHeader ) return } fmt . Fprint ( w , RepoHeader ) }

// PrintRepoHeader prints a repo header.


func PrintRepoInfo(w io.Writer, repoInfo *pfs.RepoInfo, fullTimestamps bool) {
	fmt.Fprintf(w, "%s\t", repoInfo.Repo.Name)
	if fullTimestamps {
		fmt.Fprintf(w, "%s\t", repoInfo.Created.String())
	} else {
		fmt.Fprintf(w, "%s\t", pretty.Ago(repoInfo.Created))
	}
	fmt.Fprintf(w, "%s\t", units.BytesSize(float64(repoInfo.SizeBytes)))
	if repoInfo.AuthInfo != nil {
		fmt.Fprintf(w, "%s\t", repoInfo.AuthInfo.AccessLevel.String())
	}
	fmt.Fprintln(w)
}

func PrintRepoInfo ( w io . Writer , repoInfo * pfs . RepoInfo , fullTimestamps bool ) { fmt . Fprintf ( w , " \t " , repoInfo . Repo . Name ) if fullTimestamps { fmt . Fprintf ( w , " \t " , repoInfo . Created . String ( ) ) } else { fmt . Fprintf ( w , " \t " , pretty . Ago ( repoInfo . Created ) ) } fmt . Fprintf ( w , " \t " , units . BytesSize ( float64 ( repoInfo . SizeBytes ) ) ) if repoInfo . AuthInfo != nil { fmt . Fprintf ( w , " \t " , repoInfo . AuthInfo . AccessLevel . String ( ) ) } fmt . Fprintln ( w ) }

// PrintRepoInfo pretty-prints repo info.

PrintRepoInfo pretty - prints repo info .


func PrintDetailedRepoInfo(repoInfo *PrintableRepoInfo) error {
	template, err := template.New("RepoInfo").Funcs(funcMap).Parse(
		`Name: {{.Repo.Name}}{{if .Description}}
Description: {{.Description}}{{end}}{{if .FullTimestamps}}
Created: {{.Created}}{{else}}
Created: {{prettyAgo .Created}}{{end}}
Size of HEAD on master: {{prettySize .SizeBytes}}{{if .AuthInfo}}
Access level: {{ .AuthInfo.AccessLevel.String }}{{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, repoInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedRepoInfo pretty-prints detailed repo info.

PrintDetailedRepoInfo pretty - prints detailed repo info .


func PrintBranch(w io.Writer, branchInfo *pfs.BranchInfo) {
	fmt.Fprintf(w, "%s\t", branchInfo.Branch.Name)
	if branchInfo.Head != nil {
		fmt.Fprintf(w, "%s\t\n", branchInfo.Head.ID)
	} else {
		fmt.Fprintf(w, "-\t\n")
	}
}

func PrintBranch ( w io . Writer , branchInfo * pfs . BranchInfo ) { fmt . Fprintf ( w , " \t " , branchInfo . Branch . Name ) if branchInfo . Head != nil { fmt . Fprintf ( w , " \t \n " , branchInfo . Head . ID ) } else { fmt . Fprintf ( w , " \t \n " ) } }


func PrintCommitInfo(w io.Writer, commitInfo *pfs.CommitInfo, fullTimestamps bool) {
	fmt.Fprintf(w, "%s\t", commitInfo.Commit.Repo.Name)
	fmt.Fprintf(w, "%s\t", commitInfo.Branch.Name)
	fmt.Fprintf(w, "%s\t", commitInfo.Commit.ID)
	if commitInfo.ParentCommit != nil {
		fmt.Fprintf(w, "%s\t", commitInfo.ParentCommit.ID)
	} else {
		fmt.Fprint(w, "<none>\t")
	}
	if fullTimestamps {
		fmt.Fprintf(w, "%s\t", commitInfo.Started.String())
	} else {
		fmt.Fprintf(w, "%s\t", pretty.Ago(commitInfo.Started))
	}
	if commitInfo.Finished != nil {
		fmt.Fprintf(w, fmt.Sprintf("%s\t", pretty.TimeDifference(commitInfo.Started, commitInfo.Finished)))
		fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(commitInfo.SizeBytes)))
	} else {
		fmt.Fprintf(w, "-\t")
		// Open commits don't have meaningful size information
		fmt.Fprintf(w, "-\t\n")
	}
}

// PrintCommitInfo pretty-prints commit info.

PrintCommitInfo pretty - prints commit info .


func PrintDetailedCommitInfo(commitInfo *PrintableCommitInfo) error {
	template, err := template.New("CommitInfo").Funcs(funcMap).Parse(
		`Commit: {{.Commit.Repo.Name}}@{{.Commit.ID}}{{if .Branch}}
Original Branch: {{.Branch.Name}}{{end}}{{if .Description}}
Description: {{.Description}}{{end}}{{if .ParentCommit}}
Parent: {{.ParentCommit.ID}}{{end}}{{if .FullTimestamps}}
Started: {{.Started}}{{else}}
Started: {{prettyAgo .Started}}{{end}}{{if .Finished}}{{if .FullTimestamps}}
Finished: {{.Finished}}{{else}}
Finished: {{prettyAgo .Finished}}{{end}}{{end}}
Size: {{prettySize .SizeBytes}}{{if .Provenance}}
Provenance: {{range .Provenance}} {{.Commit.Repo.Name}}@{{.Commit.ID}} ({{.Branch.Name}}) {{end}} {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, commitInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedCommitInfo pretty-prints detailed commit info.

PrintDetailedCommitInfo pretty - prints detailed commit info .


func PrintFileInfo(w io.Writer, fileInfo *pfs.FileInfo, fullTimestamps bool) {
	fmt.Fprintf(w, "%s\t", fileInfo.File.Commit.ID)
	fmt.Fprintf(w, "%s\t", fileInfo.File.Path)
	if fileInfo.FileType == pfs.FileType_FILE {
		fmt.Fprint(w, "file\t")
	} else {
		fmt.Fprint(w, "dir\t")
	}
	if fileInfo.Committed == nil {
		fmt.Fprintf(w, "-\t")
	} else if fullTimestamps {
		fmt.Fprintf(w, "%s\t", fileInfo.Committed.String())
	} else {
		fmt.Fprintf(w, "%s\t", pretty.Ago(fileInfo.Committed))
	}
	fmt.Fprintf(w, "%s\t\n", units.BytesSize(float64(fileInfo.SizeBytes)))
}

func PrintFileInfo ( w io . Writer , fileInfo * pfs . FileInfo , fullTimestamps bool ) { fmt . Fprintf ( w , " \t " , fileInfo . File . Commit . ID ) fmt . Fprintf ( w , " \t " , fileInfo . File . Path ) if fileInfo . FileType == pfs . FileType_FILE { fmt . Fprint ( w , " \t " ) } else { fmt . Fprint ( w , " \t " ) } if fileInfo . Committed == nil { fmt . Fprintf ( w , " \t " ) } else if fullTimestamps { fmt . Fprintf ( w , " \t " , fileInfo . Committed . String ( ) ) } else { fmt . Fprintf ( w , " \t " , pretty . Ago ( fileInfo . Committed ) ) } fmt . Fprintf ( w , " \t \n " , units . BytesSize ( float64 ( fileInfo . SizeBytes ) ) ) }

// PrintFileInfo pretty-prints file info.
// If recurse is false and directory size is 0, display "-" instead
// If fast is true and file size is 0, display "-" instead

PrintFileInfo pretty - prints file info . If recurse is false and directory size is 0 display - instead If fast is true and file size is 0 display - instead


func PrintDetailedFileInfo(fileInfo *pfs.FileInfo) error {
	template, err := template.New("FileInfo").Funcs(funcMap).Parse(
		`Path: {{.File.Path}}
Type: {{fileType .FileType}}
Size: {{prettySize .SizeBytes}}
Children: {{range .Children}} {{.}} {{end}}
`)
	if err != nil {
		return err
	}
	return template.Execute(os.Stdout, fileInfo)
}

// PrintDetailedFileInfo pretty-prints detailed file info.

PrintDetailedFileInfo pretty - prints detailed file info .


func Add(s string, ancestors int) string {
	return fmt.Sprintf("%s~%d", s, ancestors)
}

func Add ( s string , ancestors int ) string { return fmt . Sprintf ( " " , s , ancestors ) }

// Add adds an ancestry reference to the given string.

Add adds an ancestry reference to the given string .


func RetryNotify(operation Operation, b BackOff, notify Notify) error {
	var err error
	var next time.Duration

	b.Reset()
	for {
		if err = operation(); err == nil {
			return nil
		}

		if next = b.NextBackOff(); next == Stop {
			return err
		}

		if notify != nil {
			if err := notify(err, next); err != nil {
				return err
			}
		}

		time.Sleep(next)
	}
}

func RetryNotify ( operation Operation , b BackOff , notify Notify ) error { var err error var next time . Duration b . Reset ( ) for { if err = operation ( ) ; err == nil { return nil } if next = b . NextBackOff ( ) ; next == Stop { return err } if notify != nil { if err := notify ( err , next ) ; err != nil { return err } } time . Sleep ( next ) } }

// RetryNotify calls notify function with the error and wait duration
// for each failed attempt before sleep.

RetryNotify calls notify function with the error and wait duration for each failed attempt before sleep .


func (c *MergeCache) Get(id int64, w io.Writer, filter Filter) (retErr error) {
	r, err := c.Cache.Get(fmt.Sprint(id))
	if err != nil {
		return err
	}
	defer func() {
		if err := r.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	return NewWriter(w).Copy(NewReader(r, filter))
}

func ( c * MergeCache ) Get ( id int64 , w io . Writer , filter Filter ) ( retErr error ) { r , err := c . Cache . Get ( fmt . Sprint ( id ) ) if err != nil { return err } defer func ( ) { if err := r . Close ( ) ; err != nil && retErr == nil { retErr = err } } ( ) return NewWriter ( w ) . Copy ( NewReader ( r , filter ) ) }

// Get does a filtered write of id's hashtree to the passed in io.Writer.

Get does a filtered write of id s hashtree to the passed in io . Writer .


func (c *MergeCache) Delete(id int64) error {
	return c.Cache.Delete(fmt.Sprint(id))
}

func ( c * MergeCache ) Delete ( id int64 ) error { return c . Cache . Delete ( fmt . Sprint ( id ) ) }

// Delete deletes a hashtree from the cache.

Delete deletes a hashtree from the cache .


func PrintJobInfo(w io.Writer, jobInfo *ppsclient.JobInfo, fullTimestamps bool) {
	fmt.Fprintf(w, "%s\t", jobInfo.Job.ID)
	fmt.Fprintf(w, "%s\t", jobInfo.Pipeline.Name)
	if fullTimestamps {
		fmt.Fprintf(w, "%s\t", jobInfo.Started.String())
	} else {
		fmt.Fprintf(w, "%s\t", pretty.Ago(jobInfo.Started))
	}
	if jobInfo.Finished != nil {
		fmt.Fprintf(w, "%s\t", pretty.TimeDifference(jobInfo.Started, jobInfo.Finished))
	} else {
		fmt.Fprintf(w, "-\t")
	}
	fmt.Fprintf(w, "%d\t", jobInfo.Restart)
	if jobInfo.DataRecovered != 0 {
		fmt.Fprintf(w, "%d + %d + %d / %d\t", jobInfo.DataProcessed, jobInfo.DataSkipped, jobInfo.DataRecovered, jobInfo.DataTotal)
	} else {
		fmt.Fprintf(w, "%d + %d / %d\t", jobInfo.DataProcessed, jobInfo.DataSkipped, jobInfo.DataTotal)
	}
	fmt.Fprintf(w, "%s\t", pretty.Size(jobInfo.Stats.DownloadBytes))
	fmt.Fprintf(w, "%s\t", pretty.Size(jobInfo.Stats.UploadBytes))
	if jobInfo.State == ppsclient.JobState_JOB_FAILURE {
		fmt.Fprintf(w, "%s: %s\t\n", jobState(jobInfo.State), safeTrim(jobInfo.Reason, jobReasonLen))
	} else {
		fmt.Fprintf(w, "%s\t\n", jobState(jobInfo.State))
	}
}

func PrintJobInfo ( w io . Writer , jobInfo * ppsclient . JobInfo , fullTimestamps bool ) { fmt . Fprintf ( w , " \t " , jobInfo . Job . ID ) fmt . Fprintf ( w , " \t " , jobInfo . Pipeline . Name ) if fullTimestamps { fmt . Fprintf ( w , " \t " , jobInfo . Started . String ( ) ) } else { fmt . Fprintf ( w , " \t " , pretty . Ago ( jobInfo . Started ) ) } if jobInfo . Finished != nil { fmt . Fprintf ( w , " \t " , pretty . TimeDifference ( jobInfo . Started , jobInfo . Finished ) ) } else { fmt . Fprintf ( w , " \t " ) } fmt . Fprintf ( w , " \t " , jobInfo . Restart ) if jobInfo . DataRecovered != 0 { fmt . Fprintf ( w , " \t " , jobInfo . DataProcessed , jobInfo . DataSkipped , jobInfo . DataRecovered , jobInfo . DataTotal ) } else { fmt . Fprintf ( w , " \t " , jobInfo . DataProcessed , jobInfo . DataSkipped , jobInfo . DataTotal ) } fmt . Fprintf ( w , " \t " , pretty . Size ( jobInfo . Stats . DownloadBytes ) ) fmt . Fprintf ( w , " \t " , pretty . Size ( jobInfo . Stats . UploadBytes ) ) if jobInfo . State == ppsclient . JobState_JOB_FAILURE { fmt . Fprintf ( w , " \t \n " , jobState ( jobInfo . State ) , safeTrim ( jobInfo . Reason , jobReasonLen ) ) } else { fmt . Fprintf ( w , " \t \n " , jobState ( jobInfo . State ) ) } }


func PrintPipelineInfo(w io.Writer, pipelineInfo *ppsclient.PipelineInfo, fullTimestamps bool) {
	fmt.Fprintf(w, "%s\t", pipelineInfo.Pipeline.Name)
	fmt.Fprintf(w, "%s\t", ShorthandInput(pipelineInfo.Input))
	if fullTimestamps {
		fmt.Fprintf(w, "%s\t", pipelineInfo.CreatedAt.String())
	} else {
		fmt.Fprintf(w, "%s\t", pretty.Ago(pipelineInfo.CreatedAt))
	}
	fmt.Fprintf(w, "%s / %s\t\n", pipelineState(pipelineInfo.State), jobState(pipelineInfo.LastJobState))
}

func PrintPipelineInfo ( w io . Writer , pipelineInfo * ppsclient . PipelineInfo , fullTimestamps bool ) { fmt . Fprintf ( w , " \t " , pipelineInfo . Pipeline . Name ) fmt . Fprintf ( w , " \t " , ShorthandInput ( pipelineInfo . Input ) ) if fullTimestamps { fmt . Fprintf ( w , " \t " , pipelineInfo . CreatedAt . String ( ) ) } else { fmt . Fprintf ( w , " \t " , pretty . Ago ( pipelineInfo . CreatedAt ) ) } fmt . Fprintf ( w , " \t \n " , pipelineState ( pipelineInfo . State ) , jobState ( pipelineInfo . LastJobState ) ) }

// PrintPipelineInfo pretty-prints pipeline info.

PrintPipelineInfo pretty - prints pipeline info .


func PrintWorkerStatus(w io.Writer, workerStatus *ppsclient.WorkerStatus, fullTimestamps bool) {
	fmt.Fprintf(w, "%s\t", workerStatus.WorkerID)
	fmt.Fprintf(w, "%s\t", workerStatus.JobID)
	for _, datum := range workerStatus.Data {
		fmt.Fprintf(w, datum.Path)
	}
	fmt.Fprintf(w, "\t")
	if fullTimestamps {
		fmt.Fprintf(w, "%s\t", workerStatus.Started.String())
	} else {
		fmt.Fprintf(w, "%s\t", pretty.Ago(workerStatus.Started))
	}
	fmt.Fprintf(w, "%d\t\n", workerStatus.QueueSize)
}

func PrintWorkerStatus ( w io . Writer , workerStatus * ppsclient . WorkerStatus , fullTimestamps bool ) { fmt . Fprintf ( w , " \t " , workerStatus . WorkerID ) fmt . Fprintf ( w , " \t " , workerStatus . JobID ) for _ , datum := range workerStatus . Data { fmt . Fprintf ( w , datum . Path ) } fmt . Fprintf ( w , " \t " ) if fullTimestamps { fmt . Fprintf ( w , " \t " , workerStatus . Started . String ( ) ) } else { fmt . Fprintf ( w , " \t " , pretty . Ago ( workerStatus . Started ) ) } fmt . Fprintf ( w , " \t \n " , workerStatus . QueueSize ) }

// PrintWorkerStatus pretty prints a worker status.

PrintWorkerStatus pretty prints a worker status .


func PrintDetailedJobInfo(jobInfo *PrintableJobInfo) error {
	template, err := template.New("JobInfo").Funcs(funcMap).Parse(
		`ID: {{.Job.ID}} {{if .Pipeline}}
Pipeline: {{.Pipeline.Name}} {{end}} {{if .ParentJob}}
Parent: {{.ParentJob.ID}} {{end}}{{if .FullTimestamps}}
Started: {{.Started}}{{else}}
Started: {{prettyAgo .Started}} {{end}}{{if .Finished}}
Duration: {{prettyTimeDifference .Started .Finished}} {{end}}
State: {{jobState .State}}
Reason: {{.Reason}}
Processed: {{.DataProcessed}}
Failed: {{.DataFailed}}
Skipped: {{.DataSkipped}}
Recovered: {{.DataRecovered}}
Total: {{.DataTotal}}
Data Downloaded: {{prettySize .Stats.DownloadBytes}}
Data Uploaded: {{prettySize .Stats.UploadBytes}}
Download Time: {{prettyDuration .Stats.DownloadTime}}
Process Time: {{prettyDuration .Stats.ProcessTime}}
Upload Time: {{prettyDuration .Stats.UploadTime}}
Datum Timeout: {{.DatumTimeout}}
Job Timeout: {{.JobTimeout}}
Worker Status:
{{workerStatus .}}Restarts: {{.Restart}}
ParallelismSpec: {{.ParallelismSpec}}
{{ if .ResourceRequests }}ResourceRequests:
  CPU: {{ .ResourceRequests.Cpu }}
  Memory: {{ .ResourceRequests.Memory }} {{end}}
{{ if .ResourceLimits }}ResourceLimits:
  CPU: {{ .ResourceLimits.Cpu }}
  Memory: {{ .ResourceLimits.Memory }}
  {{ if .ResourceLimits.Gpu }}GPU:
    Type: {{ .ResourceLimits.Gpu.Type }}
    Number: {{ .ResourceLimits.Gpu.Number }} {{end}} {{end}}
{{ if .Service }}Service:
	{{ if .Service.InternalPort }}InternalPort: {{ .Service.InternalPort }} {{end}}
	{{ if .Service.ExternalPort }}ExternalPort: {{ .Service.ExternalPort }} {{end}} {{end}}Input:
{{jobInput .}}
Transform:
{{prettyTransform .Transform}} {{if .OutputCommit}}
Output Commit: {{.OutputCommit.ID}} {{end}} {{ if .StatsCommit }}
Stats Commit: {{.StatsCommit.ID}} {{end}} {{ if .Egress }}
Egress: {{.Egress.URL}} {{end}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, jobInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedJobInfo pretty-prints detailed job info.

PrintDetailedJobInfo pretty - prints detailed job info .


func PrintDetailedPipelineInfo(pipelineInfo *PrintablePipelineInfo) error {
	template, err := template.New("PipelineInfo").Funcs(funcMap).Parse(
		`Name: {{.Pipeline.Name}}{{if .Description}}
Description: {{.Description}}{{end}}{{if .FullTimestamps }}
Created: {{.CreatedAt}}{{ else }}
Created: {{prettyAgo .CreatedAt}} {{end}}
State: {{pipelineState .State}}
Stopped: {{ .Stopped }}
Reason: {{.Reason}}
Parallelism Spec: {{.ParallelismSpec}}
{{ if .ResourceRequests }}ResourceRequests:
  CPU: {{ .ResourceRequests.Cpu }}
  Memory: {{ .ResourceRequests.Memory }} {{end}}
{{ if .ResourceLimits }}ResourceLimits:
  CPU: {{ .ResourceLimits.Cpu }}
  Memory: {{ .ResourceLimits.Memory }}
  {{ if .ResourceLimits.Gpu }}GPU:
    Type: {{ .ResourceLimits.Gpu.Type }}
    Number: {{ .ResourceLimits.Gpu.Number }} {{end}} {{end}}
Datum Timeout: {{.DatumTimeout}}
Job Timeout: {{.JobTimeout}}
Input:
{{pipelineInput .PipelineInfo}}
{{ if .GithookURL }}Githook URL: {{.GithookURL}} {{end}}
Output Branch: {{.OutputBranch}}
Transform:
{{prettyTransform .Transform}}
{{ if .Egress }}Egress: {{.Egress.URL}} {{end}}
{{if .RecentError}} Recent Error: {{.RecentError}} {{end}}
Job Counts:
{{jobCounts .JobCounts}}
`)
	if err != nil {
		return err
	}
	err = template.Execute(os.Stdout, pipelineInfo)
	if err != nil {
		return err
	}
	return nil
}

// PrintDetailedPipelineInfo pretty-prints detailed pipeline info.

PrintDetailedPipelineInfo pretty - prints detailed pipeline info .


func PrintDatumInfo(w io.Writer, datumInfo *ppsclient.DatumInfo) {
	totalTime := "-"
	if datumInfo.Stats != nil {
		totalTime = units.HumanDuration(client.GetDatumTotalTime(datumInfo.Stats))
	}
	fmt.Fprintf(w, "%s\t%s\t%s\n", datumInfo.Datum.ID, datumState(datumInfo.State), totalTime)
}

func PrintDatumInfo ( w io . Writer , datumInfo * ppsclient . DatumInfo ) { totalTime := " " if datumInfo . Stats != nil { totalTime = units . HumanDuration ( client . GetDatumTotalTime ( datumInfo . Stats ) ) } fmt . Fprintf ( w , " \t \t \n " , datumInfo . Datum . ID , datumState ( datumInfo . State ) , totalTime ) }

// PrintDatumInfo pretty-prints file info.
// If recurse is false and directory size is 0, display "-" instead
// If fast is true and file size is 0, display "-" instead

PrintDatumInfo pretty - prints file info . If recurse is false and directory size is 0 display - instead If fast is true and file size is 0 display - instead


func PrintDetailedDatumInfo(w io.Writer, datumInfo *ppsclient.DatumInfo) {
	fmt.Fprintf(w, "ID\t%s\n", datumInfo.Datum.ID)
	fmt.Fprintf(w, "Job ID\t%s\n", datumInfo.Datum.Job.ID)
	fmt.Fprintf(w, "State\t%s\n", datumInfo.State)
	fmt.Fprintf(w, "Data Downloaded\t%s\n", pretty.Size(datumInfo.Stats.DownloadBytes))
	fmt.Fprintf(w, "Data Uploaded\t%s\n", pretty.Size(datumInfo.Stats.UploadBytes))

	totalTime := client.GetDatumTotalTime(datumInfo.Stats).String()
	fmt.Fprintf(w, "Total Time\t%s\n", totalTime)

	var downloadTime string
	dl, err := types.DurationFromProto(datumInfo.Stats.DownloadTime)
	if err != nil {
		downloadTime = err.Error()
	}
	downloadTime = dl.String()
	fmt.Fprintf(w, "Download Time\t%s\n", downloadTime)

	var procTime string
	proc, err := types.DurationFromProto(datumInfo.Stats.ProcessTime)
	if err != nil {
		procTime = err.Error()
	}
	procTime = proc.String()
	fmt.Fprintf(w, "Process Time\t%s\n", procTime)

	var uploadTime string
	ul, err := types.DurationFromProto(datumInfo.Stats.UploadTime)
	if err != nil {
		uploadTime = err.Error()
	}
	uploadTime = ul.String()
	fmt.Fprintf(w, "Upload Time\t%s\n", uploadTime)

	fmt.Fprintf(w, "PFS State:\n")
	tw := ansiterm.NewTabWriter(w, 10, 1, 3, ' ', 0)
	PrintFileHeader(tw)
	PrintFile(tw, datumInfo.PfsState)
	tw.Flush()
	fmt.Fprintf(w, "Inputs:\n")
	tw = ansiterm.NewTabWriter(w, 10, 1, 3, ' ', 0)
	PrintFileHeader(tw)
	for _, d := range datumInfo.Data {
		PrintFile(tw, d.File)
	}
	tw.Flush()
}

func
PrintDetailedDatumInfo
(
w
io
.
Writer
,
datumInfo
*
ppsclient
.
DatumInfo
)
{
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
datumInfo
.
Datum
.
ID
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
datumInfo
.
Datum
.
Job
.
ID
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
datumInfo
.
State
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
pretty
.
Size
(
datumInfo
.
Stats
.
DownloadBytes
)
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
pretty
.
Size
(
datumInfo
.
Stats
.
UploadBytes
)
)
totalTime
:=
client
.
GetDatumTotalTime
(
datumInfo
.
Stats
)
.
String
(
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
totalTime
)
var
downloadTime
string
dl
,
err
:=
types
.
DurationFromProto
(
datumInfo
.
Stats
.
DownloadTime
)
if
err
!=
{
downloadTime
=
err
.
Error
(
)
}
downloadTime
=
dl
.
String
(
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
downloadTime
)
var
procTime
string
proc
,
err
:=
types
.
DurationFromProto
(
datumInfo
.
Stats
.
ProcessTime
)
if
err
!=
{
procTime
=
err
.
Error
(
)
}
procTime
=
proc
.
String
(
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
procTime
)
var
uploadTime
string
ul
,
err
:=
types
.
DurationFromProto
(
datumInfo
.
Stats
.
UploadTime
)
if
err
!=
{
uploadTime
=
err
.
Error
(
)
}
uploadTime
=
ul
.
String
(
)
fmt
.
Fprintf
(
w
,
"
\t
\n
"
,
uploadTime
)
fmt
.
Fprintf
(
w
,
"
\n
"
)
tw
:=
ansiterm
.
NewTabWriter
(
w
,
10
,
1
,
3
,
' '
,
0
)
PrintFileHeader
(
tw
)
PrintFile
(
tw
,
datumInfo
.
PfsState
)
tw
.
Flush
(
)
fmt
.
Fprintf
(
w
,
"
\n
"
)
tw
=
ansiterm
.
NewTabWriter
(
w
,
10
,
1
,
3
,
' '
,
0
)
PrintFileHeader
(
tw
)
for
_
,
d
:=
range
datumInfo
.
Data
{
PrintFile
(
tw
,
d
.
File
)
}
tw
.
Flush
(
)
}

// PrintDetailedDatumInfo pretty-prints detailed info about a datum

PrintDetailedDatumInfo pretty - prints detailed info about a datum


func PrintFile(w io.Writer, file *pfsclient.File) {
	fmt.Fprintf(w, "  %s\t%s\t%s\t\n", file.Commit.Repo.Name, file.Commit.ID, file.Path)
}

func PrintFile ( w io . Writer , file * pfsclient . File ) { fmt . Fprintf ( w , " \t \t \t \n " , file . Commit . Repo . Name , file . Commit . ID , file . Path ) }


func ShorthandInput(input *ppsclient.Input) string {
	switch {
	case input == nil:
		return "none"
	case input.Pfs != nil:
		return fmt.Sprintf("%s:%s", input.Pfs.Repo, input.Pfs.Glob)
	case input.Cross != nil:
		var subInput []string
		for _, input := range input.Cross {
			subInput = append(subInput, ShorthandInput(input))
		}
		return "(" + strings.Join(subInput, " ⨯ ") + ")"
	case input.Union != nil:
		var subInput []string
		for _, input := range input.Union {
			subInput = append(subInput, ShorthandInput(input))
		}
		return "(" + strings.Join(subInput, " ∪ ") + ")"
	case input.Cron != nil:
		return fmt.Sprintf("%s:%s", input.Cron.Name, input.Cron.Spec)
	}
	return ""
}

func ShorthandInput ( input * ppsclient . Input ) string { switch { case input == nil : return " " case input . Pfs != nil : return fmt . Sprintf ( " " , input . Pfs . Repo , input . Pfs . Glob ) case input . Cross != nil : var subInput [ ] string for _ , input := range input . Cross { subInput = append ( subInput , ShorthandInput ( input ) ) } return " " + strings . Join ( subInput , " + " " case input . Union != nil : var subInput [ ] string for _ , input := range input . Union { subInput = append ( subInput , ShorthandInput ( input ) ) } return " " + strings . Join ( subInput , " + " " case input . Cron != nil : return fmt . Sprintf ( " " , input . Cron . Name , input . Cron . Spec ) } return " " }

// ShorthandInput renders a pps.Input as a short, readable string

ShorthandInput renders a pps . Input as a short readable string


func (v *vaultCredentialsProvider) Retrieve() (credentials.Value, error) {
	var emptyCreds, result credentials.Value // result

	// retrieve AWS creds from vault
	vaultSecret, err := v.vaultClient.Logical().Read(path.Join("aws", "creds", v.vaultRole))
	if err != nil {
		return emptyCreds, fmt.Errorf("could not retrieve creds from vault: %v", err)
	}
	accessKeyIface, accessKeyOk := vaultSecret.Data["access_key"]
	awsSecretIface, awsSecretOk := vaultSecret.Data["secret_key"]
	if !accessKeyOk || !awsSecretOk {
		return emptyCreds, fmt.Errorf("aws creds not present in vault response")
	}

	// Convert access key & secret in response to strings
	result.AccessKeyID, accessKeyOk = accessKeyIface.(string)
	result.SecretAccessKey, awsSecretOk = awsSecretIface.(string)
	if !accessKeyOk || !awsSecretOk {
		return emptyCreds, fmt.Errorf("aws creds in vault response were not both strings (%T and %T)", accessKeyIface, awsSecretIface)
	}

	// update the lease values in 'v', and spawn a goroutine to renew the lease
	v.updateLease(vaultSecret)
	go func() {
		for {
			// renew at half the lease duration or one day, whichever is greater
			// (lease must expire eventually)
			renewInterval := v.getLeaseDuration()
			if renewInterval.Seconds() < oneDayInSeconds {
				renewInterval = oneDayInSeconds * time.Second
			}

			// Wait until 'renewInterval' has elapsed, then renew the lease
			time.Sleep(renewInterval)
			backoff.RetryNotify(func() error {
				// every two days, renew the lease for this node's AWS credentials
				vaultSecret, err := v.vaultClient.Sys().Renew(v.leaseID, twoDaysInSeconds)
				if err != nil {
					return err
				}
				v.updateLease(vaultSecret)
				return nil
			}, backoff.NewExponentialBackOff(), func(err error, _ time.Duration) error {
				log.Errorf("could not renew vault lease: %v", err)
				return nil
			})
		}
	}()

	// Per https://www.vaultproject.io/docs/secrets/aws/index.html#usage, wait
	// until token is usable
	time.Sleep(10 * time.Second)
	return result, nil
}

// Retrieve returns nil if it successfully retrieved the value.  Error is
// returned if the value were not obtainable, or empty.

Retrieve returns nil if it successfully retrieved the value . Error is returned if the value were not obtainable or empty .


func (v *vaultCredentialsProvider) IsExpired() bool {
	v.leaseMu.Lock()
	defer v.leaseMu.Unlock()
	return time.Now().After(v.leaseLastRenew.Add(v.leaseDuration))
}

func ( v * vaultCredentialsProvider ) IsExpired ( ) bool { v . leaseMu . Lock ( ) defer v . leaseMu . Unlock ( ) return time . Now ( ) . After ( v . leaseLastRenew . Add ( v . leaseDuration ) ) }

// IsExpired returns if the credentials are no longer valid, and need to be
// retrieved.

IsExpired returns if the credentials are no longer valid and need to be retrieved .


func NewBranch(repoName string, branchName string) *pfs.Branch {
	return &pfs.Branch{
		Repo: NewRepo(repoName),
		Name: branchName,
	}
}

func NewBranch ( repoName string , branchName string ) * pfs . Branch { return & pfs . Branch { Repo : NewRepo ( repoName ) , Name : branchName , } }


func NewCommit(repoName string, commitID string) *pfs.Commit {
	return &pfs.Commit{
		Repo: NewRepo(repoName),
		ID:   commitID,
	}
}

func NewCommit ( repoName string , commitID string ) * pfs . Commit { return & pfs . Commit { Repo : NewRepo ( repoName ) , ID : commitID , } }


func NewCommitProvenance(repoName string, branchName string, commitID string) *pfs.CommitProvenance {
	return &pfs.CommitProvenance{
		Commit: NewCommit(repoName, commitID),
		Branch: NewBranch(repoName, branchName),
	}
}

func NewCommitProvenance ( repoName string , branchName string , commitID string ) * pfs . CommitProvenance { return & pfs . CommitProvenance { Commit : NewCommit ( repoName , commitID ) , Branch : NewBranch ( repoName , branchName ) , } }

// NewCommitProvenance creates a pfs.CommitProvenance.

NewCommitProvenance creates a pfs . CommitProvenance .


func NewFile(repoName string, commitID string, path string) *pfs.File {
	return &pfs.File{
		Commit: NewCommit(repoName, commitID),
		Path:   path,
	}
}

func NewFile ( repoName string , commitID string , path string ) * pfs . File { return & pfs . File { Commit : NewCommit ( repoName , commitID ) , Path : path , } }