func (d *driver) getTreeForFile(pachClient *client.APIClient, file *pfs.File) (hashtree.HashTree, error) {
	if file.Commit == nil {
		t, err := hashtree.NewDBHashTree(d.storageRoot)
		if err != nil {
			return nil, err
		}
		return t, nil
	}
	commitInfo, err := d.inspectCommit(pachClient, file.Commit, pfs.CommitState_STARTED)
	if err != nil {
		return nil, err
	}
	if commitInfo.Finished != nil {
		tree, err := d.getTreeForCommit(pachClient, file.Commit)
		if err != nil {
			return nil, err
		}
		return tree, nil
	}
	parentTree, err := d.getTreeForCommit(pachClient, commitInfo.ParentCommit)
	if err != nil {
		return nil, err
	}
	return d.getTreeForOpenCommit(pachClient, file, parentTree)
}

func ( d * driver ) getTreeForFile ( pachClient * client . APIClient , file * pfs . File ) ( hashtree . HashTree , error ) { if file . Commit == nil { t , err := hashtree . NewDBHashTree ( d . storageRoot ) if err != nil { return nil , err } return t , nil } commitInfo , err := d . inspectCommit ( pachClient , file . Commit , pfs . CommitState_STARTED ) if err != nil { return nil , err } if commitInfo . Finished != nil { tree , err := d . getTreeForCommit ( pachClient , file . Commit ) if err != nil { return nil , err } return tree , nil } parentTree , err := d . getTreeForCommit ( pachClient , commitInfo . ParentCommit ) if err != nil { return nil , err } return d . getTreeForOpenCommit ( pachClient , file , parentTree ) }

// getTreeForFile is like getTreeForCommit except that it can handle open commits.
// It takes a file instead of a commit so that it can apply the changes for
// that path to the tree before it returns it.

getTreeForFile is like getTreeForCommit except that it can handle open commits . It takes a file instead of a commit so that it can apply the changes for that path to the tree before it returns it .


func provenantOnInput(provenance []*pfs.CommitProvenance) bool {
	provenanceCount := len(provenance)
	for _, p := range provenance {
		// in particular, we want to exclude provenance on the spec repo (used e.g. for spouts)
		if p.Commit.Repo.Name == ppsconsts.SpecRepo {
			provenanceCount--
			break
		}
	}
	return provenanceCount > 0
}

// this is a helper function to check if the given provenance has provenance on an input branch


func nodeToFileInfo(ci *pfs.CommitInfo, path string, node *hashtree.NodeProto, full bool) *pfs.FileInfo {
	fileInfo := &pfs.FileInfo{
		File: &pfs.File{
			Commit: ci.Commit,
			Path:   path,
		},
		SizeBytes: uint64(node.SubtreeSize),
		Hash:      node.Hash,
		Committed: ci.Finished,
	}
	if node.FileNode != nil {
		fileInfo.FileType = pfs.FileType_FILE
		if full {
			fileInfo.Objects = node.FileNode.Objects
			fileInfo.BlockRefs = node.FileNode.BlockRefs
		}
	} else if node.DirNode != nil {
		fileInfo.FileType = pfs.FileType_DIR
		if full {
			fileInfo.Children = node.DirNode.Children
		}
	}
	return fileInfo
}

func nodeToFileInfo ( ci * pfs . CommitInfo , path string , node * hashtree . NodeProto , full bool ) * pfs . FileInfo { fileInfo := & pfs . FileInfo { File : & pfs . File { Commit : ci . Commit , Path : path , } , SizeBytes : uint64 ( node . SubtreeSize ) , Hash : node . Hash , Committed : ci . Finished , } if node . FileNode != nil { fileInfo . FileType = pfs . FileType_FILE if full { fileInfo . Objects = node . FileNode . Objects fileInfo . BlockRefs = node . FileNode . BlockRefs } } else if node . DirNode != nil { fileInfo . FileType = pfs . FileType_DIR if full { fileInfo . Children = node . DirNode . Children } } return fileInfo }

// If full is false, exclude potentially large fields such as `Objects`
// and `Children`

If full is false exclude potentially large fields such as Objects and Children


func (d *driver) fileHistory(pachClient *client.APIClient, file *pfs.File, history int64, f func(*pfs.FileInfo) error) error {
	var fi *pfs.FileInfo
	for {
		_fi, err := d.inspectFile(pachClient, file)
		if err != nil {
			if _, ok := err.(pfsserver.ErrFileNotFound); ok {
				return f(fi)
			}
			return err
		}
		if fi != nil && bytes.Compare(fi.Hash, _fi.Hash) != 0 {
			if err := f(fi); err != nil {
				return err
			}
			if history > 0 {
				history--
				if history == 0 {
					return nil
				}
			}
		}
		fi = _fi
		ci, err := d.inspectCommit(pachClient, file.Commit, pfs.CommitState_STARTED)
		if err != nil {
			return err
		}
		if ci.ParentCommit == nil {
			return f(fi)
		}
		file.Commit = ci.ParentCommit
	}
}

func ( d * driver ) fileHistory ( pachClient * client . APIClient , file * pfs . File , history int64 , f func ( * pfs . FileInfo ) error ) error { var fi * pfs . FileInfo for { _fi , err := d . inspectFile ( pachClient , file ) if err != nil { if _ , ok := err . ( pfsserver . ErrFileNotFound ) ; ok { return f ( fi ) } return err } if fi != nil && bytes . Compare ( fi . Hash , _fi . Hash ) != 0 { if err := f ( fi ) ; err != nil { return err } if history > 0 { history -- if history == 0 { return nil } } } fi = _fi ci , err := d . inspectCommit ( pachClient , file . Commit , pfs . CommitState_STARTED ) if err != nil { return err } if ci . ParentCommit == nil { return f ( fi ) } file . Commit = ci . ParentCommit } }

// fileHistory calls f with FileInfos for the file, starting with how it looked
// at the referenced commit and then all past versions that are different.

fileHistory calls f with FileInfos for the file starting with how it looked at the referenced commit and then all past versions that are different .


func (d *driver) upsertPutFileRecords(pachClient *client.APIClient, file *pfs.File, newRecords *pfs.PutFileRecords) error {
	prefix, err := d.scratchFilePrefix(file)
	if err != nil {
		return err
	}

	ctx := pachClient.Ctx()
	_, err = col.NewSTM(ctx, d.etcdClient, func(stm col.STM) error {
		commitsCol := d.openCommits.ReadOnly(ctx)
		var commit pfs.Commit
		err := commitsCol.Get(file.Commit.ID, &commit)
		if err != nil {
			return err
		}
		// Dumb check to make sure the unmarshalled value exists (and matches the current ID)
		// to denote that the current commit is indeed open
		if commit.ID != file.Commit.ID {
			return fmt.Errorf("commit %v is not open", file.Commit.ID)
		}
		recordsCol := d.putFileRecords.ReadWrite(stm)
		var existingRecords pfs.PutFileRecords
		return recordsCol.Upsert(prefix, &existingRecords, func() error {
			if newRecords.Tombstone {
				existingRecords.Tombstone = true
				existingRecords.Records = nil
			}
			existingRecords.Split = newRecords.Split
			existingRecords.Records = append(existingRecords.Records, newRecords.Records...)
			existingRecords.Header = newRecords.Header
			existingRecords.Footer = newRecords.Footer
			return nil
		})
	})
	if err != nil {
		return err
	}

	return err
}

// Put the tree into the blob store
// Only write the records to etcd if the commit does exist and is open.
// To check that a key exists in etcd, we assert that its CreateRevision
// is greater than zero.

Put the tree into the blob store Only write the records to etcd if the commit does exist and is open . To check that a key exists in etcd we assert that its CreateRevision is greater than zero .


func (r *PGDumpReader) ReadRow() ([]byte, error) {
	if len(r.Header) == 0 {
		err := r.readHeader()
		if err != nil {
			return nil, err
		}
	}
	endLine := "\\.\n" // Trailing '\.' denotes the end of the row inserts
	row, err := r.rd.ReadBytes('\n')
	if err != nil && err != io.EOF {
		return nil, fmt.Errorf("error reading pgdump row: %v", err)
	}
	// corner case: some pgdump files separate lines with \r\n (even on linux),
	// so clean this case up so all handling below is unified
	if len(row) >= 2 && row[len(row)-2] == '\r' {
		row[len(row)-2] = '\n'
		row = row[:len(row)-1]
	}
	if string(row) == endLine {
		r.Footer = append(r.Footer, row...)
		err = r.readFooter()
		row = nil // The endline is part of the footer
	}
	if err == io.EOF && len(r.Footer) == 0 {
		return nil, fmt.Errorf("invalid pgdump - missing footer")
	}
	return row, err
}

// ReadRow parses the pgdump file and populates the header and the footer
// It returns EOF when done, and at that time both the Header and Footer will
// be populated. Both header and footer are required. If either are missing, an
// error is returned

ReadRow parses the pgdump file and populates the header and the footer It returns EOF when done and at that time both the Header and Footer will be populated . Both header and footer are required . If either are missing an error is returned


func NewReporter(clusterID string, kubeClient *kube.Clientset) *Reporter {
	reporter := &Reporter{
		segmentClient: newPersistentClient(),
		clusterID:     clusterID,
		kubeClient:    kubeClient,
	}
	go reporter.reportClusterMetrics()
	return reporter
}

func NewReporter ( clusterID string , kubeClient * kube . Clientset ) * Reporter { reporter := & Reporter { segmentClient : newPersistentClient ( ) , clusterID : clusterID , kubeClient : kubeClient , } go reporter . reportClusterMetrics ( ) return reporter }

// NewReporter creates a new reporter and kicks off the loop to report cluster
// metrics

NewReporter creates a new reporter and kicks off the loop to report cluster metrics


func ReportUserAction(ctx context.Context, r *Reporter, action string) func(time.Time, error) {
	if r == nil {
		// This happens when stubbing out metrics for testing, e.g. src/server/pfs/server/server_test.go
		return func(time.Time, error) {}
	}
	// If we report nil, segment sees it, but mixpanel omits the field
	r.reportUserAction(ctx, fmt.Sprintf("%vStarted", action), 1)
	return func(start time.Time, err error) {
		if err == nil {
			r.reportUserAction(ctx, fmt.Sprintf("%vFinished", action), time.Since(start).Seconds())
		} else {
			r.reportUserAction(ctx, fmt.Sprintf("%vErrored", action), err.Error())
		}
	}
}

//ReportUserAction pushes the action into a queue for reporting,
// and reports the start, finish, and error conditions

ReportUserAction pushes the action into a queue for reporting and reports the start finish and error conditions


func FinishReportAndFlushUserAction(action string, err error, start time.Time) func() {
	var wait func()
	if err != nil {
		wait = reportAndFlushUserAction(fmt.Sprintf("%vErrored", action), err)
	} else {
		wait = reportAndFlushUserAction(fmt.Sprintf("%vFinished", action), time.Since(start).Seconds())
	}
	return wait
}

func FinishReportAndFlushUserAction ( action string , err error , start time . Time ) func ( ) { var wait func ( ) if err != nil { wait = reportAndFlushUserAction ( fmt . Sprintf ( " " , action ) , err ) } else { wait = reportAndFlushUserAction ( fmt . Sprintf ( " " , action ) , time . Since ( start ) . Seconds ( ) ) } return wait }

// FinishReportAndFlushUserAction immediately reports the metric but does
// not block execution. It returns a wait function which waits or times
// out after 5s.
// It is used by the pachctl binary and runs on users' machines

FinishReportAndFlushUserAction immediately reports the metric but does not block execution . It returns a wait function which waits or times out after 5s . It is used by the pachctl binary and runs on users machines


func (r *Reader) Read(data []byte) (int, error) {
	var totalRead int
	for len(data) > 0 {
		n, err := r.r.Read(data)
		data = data[n:]
		totalRead += n
		if err != nil {
			// If all DataRefs have been read, then io.EOF.
			if len(r.dataRefs) == 0 {
				return totalRead, io.EOF
			}
			// Get next chunk if necessary.
			if r.curr == nil || r.curr.Chunk.Hash != r.dataRefs[0].Chunk.Hash {
				if err := r.readChunk(r.dataRefs[0].Chunk); err != nil {
					return totalRead, err
				}
			}
			r.curr = r.dataRefs[0]
			r.dataRefs = r.dataRefs[1:]
			r.r = bytes.NewReader(r.buf.Bytes()[r.curr.OffsetBytes : r.curr.OffsetBytes+r.curr.SizeBytes])
		}
	}
	return totalRead, nil

}

// Read reads from the byte stream produced by the set of DataRefs.

Read reads from the byte stream produced by the set of DataRefs .


func ActivateCmd(noMetrics, noPortForwarding *bool) *cobra.Command {
	var expires string
	activate := &cobra.Command{
		Use: "{{alias}} <activation-code>",
		Short: "Activate the enterprise features of Pachyderm with an activation " +
			"code",
		Long: "Activate the enterprise features of Pachyderm with an activation " +
			"code",
		Run: cmdutil.RunFixedArgs(1, func(args []string) error {
			c, err := client.NewOnUserMachine(!*noMetrics, !*noPortForwarding, "user")
			if err != nil {
				return fmt.Errorf("could not connect: %s", err.Error())
			}
			defer c.Close()
			req := &enterprise.ActivateRequest{}
			req.ActivationCode = args[0]
			if expires != "" {
				t, err := parseISO8601(expires)
				if err != nil {
					return fmt.Errorf("could not parse the timestamp \"%s\": %s", expires, err.Error())
				}
				req.Expires, err = types.TimestampProto(t)
				if err != nil {
					return fmt.Errorf("error converting expiration time \"%s\"; %s", t.String(), err.Error())
				}
			}
			resp, err := c.Enterprise.Activate(c.Ctx(), req)
			if err != nil {
				return err
			}
			ts, err := types.TimestampFromProto(resp.Info.Expires)
			if err != nil {
				return fmt.Errorf("Activation request succeeded, but could not "+
					"convert token expiration time to a timestamp: %s", err.Error())
			}
			fmt.Printf("Activation succeeded. Your Pachyderm Enterprise token "+
				"expires %s\n", ts.String())
			return nil
		}),
	}
	activate.PersistentFlags().StringVar(&expires, "expires", "", "A timestamp "+
		"indicating when the token provided above should expire (formatted as an "+
		"RFC 3339/ISO 8601 datetime). This is only applied if it's earlier than "+
		"the signed expiration time encoded in 'activation-code', and therefore "+
		"is only useful for testing.")

	return cmdutil.CreateAlias(activate, "enterprise activate")
}

func ActivateCmd ( noMetrics , noPortForwarding * bool ) * cobra . Command { var expires string activate := & cobra . Command { Use : " " , Short : " " + " " , Long : " " + " " , Run : cmdutil . RunFixedArgs ( 1 , func ( args [ ] string ) error { c , err := client . NewOnUserMachine ( ! * noMetrics , ! * noPortForwarding , " " ) if err != nil { return fmt . Errorf ( " " , err . Error ( ) ) } defer c . Close ( ) req := & enterprise . ActivateRequest { } req . ActivationCode = args [ 0 ] if expires != " " { t , err := parseISO8601 ( expires ) if err != nil { return fmt . Errorf ( " \" \" " , expires , err . Error ( ) ) } req . Expires , err = types . TimestampProto ( t ) if err != nil { return fmt . Errorf ( " \" \" " , t . String ( ) , err . Error ( ) ) } } resp , err := c . Enterprise . Activate ( c . Ctx ( ) , req ) if err != nil { return err } ts , err := types . TimestampFromProto ( resp . Info . Expires ) if err != nil { return fmt . Errorf ( " " + " " , err . Error ( ) ) } fmt . Printf ( " " + " \n " , ts . String ( ) ) return nil } ) , } activate . PersistentFlags ( ) . StringVar ( & expires , " " , " " , " " + " " + " " + " " + " " ) return cmdutil . CreateAlias ( activate , " " ) }

// ActivateCmd returns a cobra.Command to activate the enterprise features of
// Pachyderm within a Pachyderm cluster. All repos will go from
// publicly-accessible to accessible only by the owner, who can subsequently add
// users

ActivateCmd returns a cobra . Command to activate the enterprise features of Pachyderm within a Pachyderm cluster . All repos will go from publicly - accessible to accessible only by the owner who can subsequently add users


func GetStateCmd(noMetrics, noPortForwarding *bool) *cobra.Command {
	getState := &cobra.Command{
		Short: "Check whether the Pachyderm cluster has enterprise features " +
			"activated",
		Long: "Check whether the Pachyderm cluster has enterprise features " +
			"activated",
		Run: cmdutil.Run(func(args []string) error {
			c, err := client.NewOnUserMachine(!*noMetrics, !*noPortForwarding, "user")
			if err != nil {
				return fmt.Errorf("could not connect: %s", err.Error())
			}
			defer c.Close()
			resp, err := c.Enterprise.GetState(c.Ctx(), &enterprise.GetStateRequest{})
			if err != nil {
				return err
			}
			if resp.State == enterprise.State_NONE {
				fmt.Println("No Pachyderm Enterprise token was found")
				return nil
			}
			ts, err := types.TimestampFromProto(resp.Info.Expires)
			if err != nil {
				return fmt.Errorf("Activation request succeeded, but could not "+
					"convert token expiration time to a timestamp: %s", err.Error())
			}
			fmt.Printf("Pachyderm Enterprise token state: %s\nExpiration: %s\n",
				resp.State.String(), ts.String())
			return nil
		}),
	}
	return cmdutil.CreateAlias(getState, "enterprise get-state")
}

func GetStateCmd ( noMetrics , noPortForwarding * bool ) * cobra . Command { getState := & cobra . Command { Short : " " + " " , Long : " " + " " , Run : cmdutil . Run ( func ( args [ ] string ) error { c , err := client . NewOnUserMachine ( ! * noMetrics , ! * noPortForwarding , " " ) if err != nil { return fmt . Errorf ( " " , err . Error ( ) ) } defer c . Close ( ) resp , err := c . Enterprise . GetState ( c . Ctx ( ) , & enterprise . GetStateRequest { } ) if err != nil { return err } if resp . State == enterprise . State_NONE { fmt . Println ( " " ) return nil } ts , err := types . TimestampFromProto ( resp . Info . Expires ) if err != nil { return fmt . Errorf ( " " + " " , err . Error ( ) ) } fmt . Printf ( " \n \n " , resp . State . String ( ) , ts . String ( ) ) return nil } ) , } return cmdutil . CreateAlias ( getState , " " ) }

// GetStateCmd returns a cobra.Command to activate the enterprise features of
// Pachyderm within a Pachyderm cluster. All repos will go from
// publicly-accessible to accessible only by the owner, who can subsequently add
// users

GetStateCmd returns a cobra . Command to activate the enterprise features of Pachyderm within a Pachyderm cluster . All repos will go from publicly - accessible to accessible only by the owner who can subsequently add users


func Cmds(noMetrics, noPortForwarding *bool) []*cobra.Command {
	var commands []*cobra.Command

	enterprise := &cobra.Command{
		Short: "Enterprise commands enable Pachyderm Enterprise features",
		Long:  "Enterprise commands enable Pachyderm Enterprise features",
	}
	commands = append(commands, cmdutil.CreateAlias(enterprise, "enterprise"))

	commands = append(commands, ActivateCmd(noMetrics, noPortForwarding))
	commands = append(commands, GetStateCmd(noMetrics, noPortForwarding))

	return commands
}

func Cmds ( noMetrics , noPortForwarding * bool ) [ ] * cobra . Command { var commands [ ] * cobra . Command enterprise := & cobra . Command { Short : " " , Long : " " , } commands = append ( commands , cmdutil . CreateAlias ( enterprise , " " ) ) commands = append ( commands , ActivateCmd ( noMetrics , noPortForwarding ) ) commands = append ( commands , GetStateCmd ( noMetrics , noPortForwarding ) ) return commands }

// Cmds returns pachctl commands related to Pachyderm Enterprise


func NewConfiguration(config interface{}) *Configuration {
	configuration := &Configuration{}
	switch config.(type) {
	case *GlobalConfiguration:
		configuration.GlobalConfiguration = config.(*GlobalConfiguration)
		return configuration
	case *PachdFullConfiguration:
		configuration.GlobalConfiguration = &config.(*PachdFullConfiguration).GlobalConfiguration
		configuration.PachdSpecificConfiguration = &config.(*PachdFullConfiguration).PachdSpecificConfiguration
		return configuration
	case *WorkerFullConfiguration:
		configuration.GlobalConfiguration = &config.(*WorkerFullConfiguration).GlobalConfiguration
		configuration.WorkerSpecificConfiguration = &config.(*WorkerFullConfiguration).WorkerSpecificConfiguration
		return configuration
	default:
		return nil
	}
}

func NewConfiguration ( config interface { } ) * Configuration { configuration := & Configuration { } switch config . ( type ) { case * GlobalConfiguration : configuration . GlobalConfiguration = config . ( * GlobalConfiguration ) return configuration case * PachdFullConfiguration : configuration . GlobalConfiguration = & config . ( * PachdFullConfiguration ) . GlobalConfiguration configuration . PachdSpecificConfiguration = & config . ( * PachdFullConfiguration ) . PachdSpecificConfiguration return configuration case * WorkerFullConfiguration : configuration . GlobalConfiguration = & config . ( * WorkerFullConfiguration ) . GlobalConfiguration configuration . WorkerSpecificConfiguration = & config . ( * WorkerFullConfiguration ) . WorkerSpecificConfiguration return configuration default : return nil } }

// NewConfiguration creates a generic configuration from a specific type of configuration.

NewConfiguration creates a generic configuration from a specific type of configuration .


func Repos(etcdClient *etcd.Client, etcdPrefix string) col.Collection {
	return col.NewCollection(
		etcdClient,
		path.Join(etcdPrefix, reposPrefix),
		nil,
		&pfs.RepoInfo{},
		nil,
		nil,
	)
}

func Repos ( etcdClient * etcd . Client , etcdPrefix string ) col . Collection { return col . NewCollection ( etcdClient , path . Join ( etcdPrefix , reposPrefix ) , nil , & pfs . RepoInfo { } , nil , nil , ) }


func PutFileRecords(etcdClient *etcd.Client, etcdPrefix string) col.Collection {
	return col.NewCollection(
		etcdClient,
		path.Join(etcdPrefix, putFileRecordsPrefix),
		nil,
		&pfs.PutFileRecords{},
		nil,
		nil,
	)
}

func PutFileRecords ( etcdClient * etcd . Client , etcdPrefix string ) col . Collection { return col . NewCollection ( etcdClient , path . Join ( etcdPrefix , putFileRecordsPrefix ) , nil , & pfs . PutFileRecords { } , nil , nil , ) }

// PutFileRecords returns a collection of putFileRecords


func Commits(etcdClient *etcd.Client, etcdPrefix string, repo string) col.Collection {
	return col.NewCollection(
		etcdClient,
		path.Join(etcdPrefix, commitsPrefix, repo),
		[]*col.Index{ProvenanceIndex},
		&pfs.CommitInfo{},
		nil,
		nil,
	)
}

func Commits ( etcdClient * etcd . Client , etcdPrefix string , repo string ) col . Collection { return col . NewCollection ( etcdClient , path . Join ( etcdPrefix , commitsPrefix , repo ) , [ ] * col . Index { ProvenanceIndex } , & pfs . CommitInfo { } , nil , nil , ) }

// Commits returns a collection of commits


func Branches(etcdClient *etcd.Client, etcdPrefix string, repo string) col.Collection {
	return col.NewCollection(
		etcdClient,
		path.Join(etcdPrefix, branchesPrefix, repo),
		nil,
		&pfs.BranchInfo{},
		func(key string) error {
			if uuid.IsUUIDWithoutDashes(key) {
				return fmt.Errorf("branch name cannot be a UUID V4")
			}
			return nil
		},
		nil,
	)
}

func Branches ( etcdClient * etcd . Client , etcdPrefix string , repo string ) col . Collection { return col . NewCollection ( etcdClient , path . Join ( etcdPrefix , branchesPrefix , repo ) , nil , & pfs . BranchInfo { } , func ( key string ) error { if uuid . IsUUIDWithoutDashes ( key ) { return fmt . Errorf ( " " ) } return nil } , nil , ) }

// Branches returns a collection of branches


func OpenCommits(etcdClient *etcd.Client, etcdPrefix string) col.Collection {
	return col.NewCollection(
		etcdClient,
		path.Join(etcdPrefix, openCommitsPrefix),
		nil,
		&pfs.Commit{},
		nil,
		nil,
	)
}

func OpenCommits ( etcdClient * etcd . Client , etcdPrefix string ) col . Collection { return col . NewCollection ( etcdClient , path . Join ( etcdPrefix , openCommitsPrefix ) , nil , & pfs . Commit { } , nil , nil , ) }

// OpenCommits returns a collection of open commits


func NewDAG(nodes map[string][]string) *DAG {
	result := &DAG{
		parents:  make(map[string][]string),
		children: make(map[string][]string),
		leaves:   make(map[string]bool),
	}
	for id, parents := range nodes {
		result.NewNode(id, parents)
	}
	return result
}

func NewDAG ( nodes map [ string ] [ ] string ) * DAG { result := & DAG { parents : make ( map [ string ] [ ] string ) , children : make ( map [ string ] [ ] string ) , leaves : make ( map [ string ] bool ) , } for id , parents := range nodes { result . NewNode ( id , parents ) } return result }

// NewDAG creates a DAG and populates it with the given nodes.

NewDAG creates a DAG and populates it with the given nodes .


func (d *DAG) NewNode(id string, parents []string) {
	d.parents[id] = parents
	for _, parentID := range parents {
		d.children[parentID] = append(d.children[parentID], id)
		d.leaves[parentID] = false
	}
	if _, ok := d.leaves[id]; !ok {
		d.leaves[id] = true
	}
}

func ( d * DAG ) NewNode ( id string , parents [ ] string ) { d . parents [ id ] = parents for _ , parentID := range parents { d . children [ parentID ] = append ( d . children [ parentID ] , id ) d . leaves [ parentID ] = false } if _ , ok := d . leaves [ id ] ; ! ok { d . leaves [ id ] = true } }


func (d *DAG) Sorted() []string {
	seen := make(map[string]bool)
	var result []string
	for id := range d.parents {
		result = append(result, dfs(id, d.parents, seen)...)
	}
	return result
}

func ( d * DAG ) Sorted ( ) [ ] string { seen := make ( map [ string ] bool ) var result [ ] string for id := range d . parents { result = append ( result , dfs ( id , d . parents , seen ) ... ) } return result }

// Sorted returns all nodes in a topologically sorted order


func (d *DAG) Leaves() []string {
	var result []string
	for id, isLeaf := range d.leaves {
		// isLeaf might be false, explicit mark nodes as non leaves
		if isLeaf {
			result = append(result, id)
		}
	}
	return result
}

// Leaves returns a slice containing all leaves in d.

Leaves returns a slice containing all leaves in d .


func (d *DAG) Ancestors(id string, from []string) []string {
	seen := make(map[string]bool)
	for _, fromID := range from {
		seen[fromID] = true
	}
	return dfs(id, d.parents, seen)
}

func ( d * DAG ) Ancestors ( id string , from [ ] string ) [ ] string { seen := make ( map [ string ] bool ) for _ , fromID := range from { seen [ fromID ] = true } return dfs ( id , d . parents , seen ) }

// Ancestors returns a slice containing all ancestors of a node, 'id',
// in d which are a descendant of at least one of the nodes in 'from'.

Ancestors returns a slice containing all ancestors of a node id in d which are a descendant of at least one of the nodes in from .


func (d *DAG) Descendants(id string, to []string) []string {
	seen := make(map[string]bool)
	for _, toID := range to {
		seen[toID] = true
	}
	return bfs(id, d.children, seen)
}

func ( d * DAG ) Descendants ( id string , to [ ] string ) [ ] string { seen := make ( map [ string ] bool ) for _ , toID := range to { seen [ toID ] = true } return bfs ( id , d . children , seen ) }

// Descendants returns a slice containing all descendants of a node, 'id',
// in d which are an ancestor of at least one of the nodes in 'to'.

Descendants returns a slice containing all descendants of a node id in d which are an ancestor of at least one of the nodes in to .


func (d *DAG) Ghosts() []string {
	var result []string
	for id := range d.children {
		if _, ok := d.parents[id]; !ok {
			result = append(result, id)
		}
	}
	return result
}

func ( d * DAG ) Ghosts ( ) [ ] string { var result [ ] string for id := range d . children { if _ , ok := d . parents [ id ] ; ! ok { result = append ( result , id ) } } return result }

// Ghosts returns nodes that were referenced as parents but never created.

Ghosts returns nodes that were referenced as parents but never created .


func NewPortForwarder(namespace string) (*PortForwarder, error) {
	if namespace == "" {
		namespace = "default"
	}

	rules := clientcmd.NewDefaultClientConfigLoadingRules()
	overrides := &clientcmd.ConfigOverrides{}
	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides)
	config, err := kubeConfig.ClientConfig()
	if err != nil {
		return nil, err
	}

	client, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	core := client.CoreV1()

	return &PortForwarder{
		core:          core,
		client:        core.RESTClient(),
		config:        config,
		namespace:     namespace,
		logger:        log.StandardLogger().Writer(),
		stopChansLock: &sync.Mutex{},
		stopChans:     []chan struct{}{},
		shutdown:      false,
	}, nil
}

// NewPortForwarder creates a new port forwarder


func (f *PortForwarder) Run(appName string, localPort, remotePort uint16) error {
	podNameSelector := map[string]string{
		"suite": "pachyderm",
		"app":   appName,
	}

	podList, err := f.core.Pods(f.namespace).List(metav1.ListOptions{
		LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(podNameSelector)),
		TypeMeta: metav1.TypeMeta{
			Kind:       "ListOptions",
			APIVersion: "v1",
		},
	})
	if err != nil {
		return err
	}
	if len(podList.Items) == 0 {
		return fmt.Errorf("No pods found for app %s", appName)
	}

	// Choose a random pod
	podName := podList.Items[rand.Intn(len(podList.Items))].Name

	url := f.client.Post().
		Resource("pods").
		Namespace(f.namespace).
		Name(podName).
		SubResource("portforward").
		URL()

	transport, upgrader, err := spdy.RoundTripperFor(f.config)
	if err != nil {
		return err
	}

	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", url)
	ports := []string{fmt.Sprintf("%d:%d", localPort, remotePort)}
	readyChan := make(chan struct{}, 1)
	stopChan := make(chan struct{}, 1)

	// Ensure that the port forwarder isn't already shutdown, and append the
	// shutdown channel so this forwarder can be closed
	f.stopChansLock.Lock()
	if f.shutdown {
		f.stopChansLock.Unlock()
		return fmt.Errorf("port forwarder is shutdown")
	}
	f.stopChans = append(f.stopChans, stopChan)
	f.stopChansLock.Unlock()

	fw, err := portforward.New(dialer, ports, stopChan, readyChan, ioutil.Discard, f.logger)
	if err != nil {
		return err
	}

	errChan := make(chan error, 1)
	go func() { errChan <- fw.ForwardPorts() }()

	select {
	case err = <-errChan:
		return fmt.Errorf("port forwarding failed: %v", err)
	case <-fw.Ready:
		return nil
	}
}

// Run starts the port forwarder. Returns after initialization is begun,
// returning any initialization errors.

Run starts the port forwarder . Returns after initialization is begun returning any initialization errors .


func (f *PortForwarder) RunForDaemon(localPort, remotePort uint16) error {
	if localPort == 0 {
		localPort = pachdLocalPort
	}
	if remotePort == 0 {
		remotePort = pachdRemotePort
	}
	return f.Run("pachd", localPort, remotePort)
}

func ( f * PortForwarder ) RunForDaemon ( localPort , remotePort uint16 ) error { if localPort == 0 { localPort = pachdLocalPort } if remotePort == 0 { remotePort = pachdRemotePort } return f . Run ( " " , localPort , remotePort ) }

// RunForDaemon creates a port forwarder for the pachd daemon.

RunForDaemon creates a port forwarder for the pachd daemon .


func (f *PortForwarder) RunForSAMLACS(localPort uint16) error {
	if localPort == 0 {
		localPort = samlAcsLocalPort
	}
	return f.Run("pachd", localPort, 654)
}

func ( f * PortForwarder ) RunForSAMLACS ( localPort uint16 ) error { if localPort == 0 { localPort = samlAcsLocalPort } return f . Run ( " " , localPort , 654 ) }

// RunForSAMLACS creates a port forwarder for SAML ACS.

RunForSAMLACS creates a port forwarder for SAML ACS .


func (f *PortForwarder) RunForDashUI(localPort uint16) error {
	if localPort == 0 {
		localPort = dashUILocalPort
	}
	return f.Run("dash", localPort, 8080)
}

func ( f * PortForwarder ) RunForDashUI ( localPort uint16 ) error { if localPort == 0 { localPort = dashUILocalPort } return f . Run ( " " , localPort , 8080 ) }

// RunForDashUI creates a port forwarder for the dash UI.

RunForDashUI creates a port forwarder for the dash UI .


func (f *PortForwarder) RunForDashWebSocket(localPort uint16) error {
	if localPort == 0 {
		localPort = dashWebSocketLocalPort
	}
	return f.Run("dash", localPort, 8081)
}

func ( f * PortForwarder ) RunForDashWebSocket ( localPort uint16 ) error { if localPort == 0 { localPort = dashWebSocketLocalPort } return f . Run ( " " , localPort , 8081 ) }

// RunForDashWebSocket creates a port forwarder for the dash websocket.

RunForDashWebSocket creates a port forwarder for the dash websocket .


func (f *PortForwarder) RunForPFS(localPort uint16) error {
	if localPort == 0 {
		localPort = pfsLocalPort
	}
	return f.Run("pachd", localPort, 30652)
}

func ( f * PortForwarder ) RunForPFS ( localPort uint16 ) error { if localPort == 0 { localPort = pfsLocalPort } return f . Run ( " " , localPort , 30652 ) }

// RunForPFS creates a port forwarder for PFS over HTTP.

RunForPFS creates a port forwarder for PFS over HTTP .


func (f *PortForwarder) RunForS3Gateway(localPort uint16) error {
	if localPort == 0 {
		localPort = s3gatewayLocalPort
	}
	return f.Run("pachd", localPort, 600)
}

func ( f * PortForwarder ) RunForS3Gateway ( localPort uint16 ) error { if localPort == 0 { localPort = s3gatewayLocalPort } return f . Run ( " " , localPort , 600 ) }

// RunForS3Gateway creates a port forwarder for the s3gateway.

RunForS3Gateway creates a port forwarder for the s3gateway .


func (f *PortForwarder) Lock() error {
	pidfile.SetPidfilePath(path.Join(os.Getenv("HOME"), ".pachyderm/port-forward.pid"))
	return pidfile.Write()
}

func ( f * PortForwarder ) Lock ( ) error { pidfile . SetPidfilePath ( path . Join ( os . Getenv ( " " ) , " " ) ) return pidfile . Write ( ) }

// Lock uses pidfiles to ensure that only one port forwarder is running across
// one or more `pachctl` instances

Lock uses pidfiles to ensure that only one port forwarder is running across one or more pachctl instances


func (f *PortForwarder) Close() {
	defer f.logger.Close()

	f.stopChansLock.Lock()
	defer f.stopChansLock.Unlock()

	if f.shutdown {
		panic("port forwarder already shutdown")
	}

	f.shutdown = true

	for _, stopChan := range f.stopChans {
		close(stopChan)
	}
}

func ( f * PortForwarder ) Close ( ) { defer f . logger . Close ( ) f . stopChansLock . Lock ( ) defer f . stopChansLock . Unlock ( ) if f . shutdown { panic ( " " ) } f . shutdown = true for _ , stopChan := range f . stopChans { close ( stopChan ) } }


func (e *Event) Unmarshal(key *string, val proto.Message) error {
	if err := CheckType(e.Template, val); err != nil {
		return err
	}
	*key = string(e.Key)
	return proto.Unmarshal(e.Value, val)
}

func ( e * Event ) Unmarshal ( key * string , val proto . Message ) error { if err := CheckType ( e . Template , val ) ; err != nil { return err } * key = string ( e . Key ) return proto . Unmarshal ( e . Value , val ) }

// Unmarshal unmarshals the item in an event into a protobuf message.

Unmarshal unmarshals the item in an event into a protobuf message .


func (e *Event) UnmarshalPrev(key *string, val proto.Message) error {
	if err := CheckType(e.Template, val); err != nil {
		return err
	}
	*key = string(e.PrevKey)
	return proto.Unmarshal(e.PrevValue, val)
}

func ( e * Event ) UnmarshalPrev ( key * string , val proto . Message ) error { if err := CheckType ( e . Template , val ) ; err != nil { return err } * key = string ( e . PrevKey ) return proto . Unmarshal ( e . PrevValue , val ) }

// UnmarshalPrev unmarshals the prev item in an event into a protobuf
// message.

UnmarshalPrev unmarshals the prev item in an event into a protobuf message .


func MakeWatcher(eventCh chan *Event, done chan struct{}) Watcher {
	return &watcher{
		eventCh: eventCh,
		done:    done,
	}
}

func MakeWatcher ( eventCh chan * Event , done chan struct { } ) Watcher { return & watcher { eventCh : eventCh , done : done , } }

// MakeWatcher returns a Watcher that uses the given event channel and done
// channel internally to deliver events and signal closure, respectively.

MakeWatcher returns a Watcher that uses the given event channel and done channel internally to deliver events and signal closure respectively .


func CheckType(template proto.Message, val interface{}) error {
	if template != nil {
		valType, templateType := reflect.TypeOf(val), reflect.TypeOf(template)
		if valType != templateType {
			return fmt.Errorf("invalid type, got: %s, expected: %s", valType, templateType)
		}
	}
	return nil
}

func CheckType ( template proto . Message , val interface { } ) error { if template != nil { valType , templateType := reflect . TypeOf ( val ) , reflect . TypeOf ( template ) if valType != templateType { return fmt . Errorf ( " " , valType , templateType ) } } return nil }

// CheckType checks to make sure val has the same type as template, unless
// template is nil in which case it always returns nil.

CheckType checks to make sure val has the same type as template unless template is nil in which case it always returns nil .


func NewPool(kubeClient *kube.Clientset, namespace string, serviceName string, port int, queueSize int64, opts ...grpc.DialOption) (*Pool, error) {
	endpointsInterface := kubeClient.CoreV1().Endpoints(namespace)

	watch, err := endpointsInterface.Watch(metav1.ListOptions{
		LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
			map[string]string{"app": serviceName},
		)),
		Watch: true,
	})
	if err != nil {
		return nil, err
	}

	pool := &Pool{
		port:           port,
		endpointsWatch: watch,
		opts:           opts,
		done:           make(chan struct{}),
		queueSize:      queueSize,
	}
	pool.connsCond = sync.NewCond(&pool.connsLock)
	go pool.watchEndpoints()
	return pool, nil
}

func NewPool ( kubeClient * kube . Clientset , namespace string , serviceName string , port int , queueSize int64 , opts ... grpc . DialOption ) ( * Pool , error ) { endpointsInterface := kubeClient . CoreV1 ( ) . Endpoints ( namespace ) watch , err := endpointsInterface . Watch ( metav1 . ListOptions { LabelSelector : metav1 . FormatLabelSelector ( metav1 . SetAsLabelSelector ( map [ string ] string { " " : serviceName } , ) ) , Watch : true , } ) if err != nil { return nil , err } pool := & Pool { port : port , endpointsWatch : watch , opts : opts , done : make ( chan struct { } ) , queueSize : queueSize , } pool . connsCond = sync . NewCond ( & pool . connsLock ) go pool . watchEndpoints ( ) return pool , nil }

// NewPool creates a new connection pool with connections to pods in the
// given service.

NewPool creates a new connection pool with connections to pods in the given service .


func (p *Pool) Do(ctx context.Context, f func(cc *grpc.ClientConn) error) error {
	var conn *connCount
	if err := func() error {
		p.connsLock.Lock()
		defer p.connsLock.Unlock()
		for {
			for addr, mapConn := range p.conns {
				if mapConn.cc == nil {
					cc, err := grpc.DialContext(ctx, addr, p.opts...)
					if err != nil {
						return fmt.Errorf("failed to connect to %s: %+v", addr, err)
					}
					mapConn.cc = cc
					conn = mapConn
					// We break because this conn has a count of 0 which we know
					// we're not beating
					break
				} else {
					mapConnCount := atomic.LoadInt64(&mapConn.count)
					if mapConnCount < p.queueSize && (conn == nil || mapConnCount < atomic.LoadInt64(&conn.count)) {
						conn = mapConn
					}
				}
			}
			if conn == nil {
				p.connsCond.Wait()
			} else {
				atomic.AddInt64(&conn.count, 1)
				break
			}
		}
		return nil
	}(); err != nil {
		return err
	}
	defer p.connsCond.Broadcast()
	defer atomic.AddInt64(&conn.count, -1)
	return f(conn.cc)
}

// Do allows you to do something with a grpc.ClientConn.
// Errors returned from f will be returned by Do.

Do allows you to do something with a grpc . ClientConn . Errors returned from f will be returned by Do .


func (p *Pool) Close() error {
	close(p.done)
	var retErr error
	for _, conn := range p.conns {
		if conn.cc != nil {
			if err := conn.cc.Close(); err != nil {
				retErr = err
			}
		}
	}
	return retErr
}

func ( p * Pool ) Close ( ) error { close ( p . done ) var retErr error for _ , conn := range p . conns { if conn . cc != nil { if err := conn . cc . Close ( ) ; err != nil { retErr = err } } } return retErr }

// Close closes all connections stored in the pool, it returns an error if any
// of the calls to Close error.

Close closes all connections stored in the pool it returns an error if any of the calls to Close error .


func buildImage(client *docker.Client, repo string, contextDir string, dockerfile string, destTag string) error {
	destImage := fmt.Sprintf("%s:%s", repo, destTag)

	fmt.Printf("Building %s, this may take a while.\n", destImage)

	err := client.BuildImage(docker.BuildImageOptions{
		Name:         destImage,
		ContextDir:   contextDir,
		Dockerfile:   dockerfile,
		OutputStream: os.Stdout,
	})

	if err != nil {
		return fmt.Errorf("could not build docker image: %s", err)
	}

	return nil
}

func buildImage ( client * docker . Client , repo string , contextDir string , dockerfile string , destTag string ) error { destImage := fmt . Sprintf ( " " , repo , destTag ) fmt . Printf ( " \n " , destImage ) err := client . BuildImage ( docker . BuildImageOptions { Name : destImage , ContextDir : contextDir , Dockerfile : dockerfile , OutputStream : os . Stdout , } ) if err != nil { return fmt . Errorf ( " " , err ) } return nil }

// buildImage builds a new docker image.


func pushImage(client *docker.Client, authConfig docker.AuthConfiguration, repo string, sourceTag string, destTag string) (string, error) {
	sourceImage := fmt.Sprintf("%s:%s", repo, sourceTag)
	destImage := fmt.Sprintf("%s:%s", repo, destTag)

	fmt.Printf("Tagging/pushing %s, this may take a while.\n", destImage)

	if err := client.TagImage(sourceImage, docker.TagImageOptions{
		Repo:    repo,
		Tag:     destTag,
		Context: context.Background(),
	}); err != nil {
		err = fmt.Errorf("could not tag docker image: %s", err)
		return "", err
	}

	if err := client.PushImage(
		docker.PushImageOptions{
			Name: repo,
			Tag:  destTag,
		},
		authConfig,
	); err != nil {
		err = fmt.Errorf("could not push docker image: %s", err)
		return "", err
	}

	return destImage, nil
}

func pushImage ( client * docker . Client , authConfig docker . AuthConfiguration , repo string , sourceTag string , destTag string ) ( string , error ) { sourceImage := fmt . Sprintf ( " " , repo , sourceTag ) destImage := fmt . Sprintf ( " " , repo , destTag ) fmt . Printf ( " \n " , destImage ) if err := client . TagImage ( sourceImage , docker . TagImageOptions { Repo : repo , Tag : destTag , Context : context . Background ( ) , } ) ; err != nil { err = fmt . Errorf ( " " , err ) return " " , err } if err := client . PushImage ( docker . PushImageOptions { Name : repo , Tag : destTag , } , authConfig , ) ; err != nil { err = fmt . Errorf ( " " , err ) return " " , err } return destImage , nil }


func newMinioClient(endpoint, bucket, id, secret string, secure bool) (*minioClient, error) {
	mclient, err := minio.New(endpoint, id, secret, secure)
	if err != nil {
		return nil, err
	}
	return &minioClient{
		bucket: bucket,
		Client: mclient,
	}, nil
}

func newMinioClient ( endpoint , bucket , id , secret string , secure bool ) ( * minioClient , error ) { mclient , err := minio . New ( endpoint , id , secret , secure ) if err != nil { return nil , err } return & minioClient { bucket : bucket , Client : mclient , } , nil }

// Creates a new minioClient structure and returns


func newMinioClientV2(endpoint, bucket, id, secret string, secure bool) (*minioClient, error) {
	mclient, err := minio.NewV2(endpoint, id, secret, secure)
	if err != nil {
		return nil, err
	}
	return &minioClient{
		bucket: bucket,
		Client: mclient,
	}, nil
}

func newMinioClientV2 ( endpoint , bucket , id , secret string , secure bool ) ( * minioClient , error ) { mclient , err := minio . NewV2 ( endpoint , id , secret , secure ) if err != nil { return nil , err } return & minioClient { bucket : bucket , Client : mclient , } , nil }

// Creates a new minioClient S3V2 structure and returns


func newMinioWriter(ctx context.Context, client *minioClient, name string) *minioWriter {
	reader, writer := io.Pipe()
	w := &minioWriter{
		ctx:     ctx,
		errChan: make(chan error),
		pipe:    writer,
	}
	go func() {
		_, err := client.PutObject(client.bucket, name, reader, "application/octet-stream")
		if err != nil {
			reader.CloseWithError(err)
		}
		w.errChan <- err
	}()
	return w
}

func newMinioWriter ( ctx context . Context , client * minioClient , name string ) * minioWriter { reader , writer := io . Pipe ( ) w := & minioWriter { ctx : ctx , errChan : make ( chan error ) , pipe : writer , } go func ( ) { _ , err := client . PutObject ( client . bucket , name , reader , " " ) if err != nil { reader . CloseWithError ( err ) } w . errChan <- err } ( ) return w }

// Creates a new minio writer and a go routine to upload objects to minio server


func (w *minioWriter) Close() error {
	span, _ := tracing.AddSpanToAnyExisting(w.ctx, "minioWriter.Close")
	defer tracing.FinishAnySpan(span)
	if err := w.pipe.Close(); err != nil {
		return err
	}
	return <-w.errChan
}

func ( w * minioWriter ) Close ( ) error { span , _ := tracing . AddSpanToAnyExisting ( w . ctx , " " ) defer tracing . FinishAnySpan ( span ) if err := w . pipe . Close ( ) ; err != nil { return err } return <- w . errChan }


func PipelineRepo(pipeline *ppsclient.Pipeline) *pfs.Repo {
	return &pfs.Repo{Name: pipeline.Name}
}

func PipelineRepo ( pipeline * ppsclient . Pipeline ) * pfs . Repo { return & pfs . Repo { Name : pipeline . Name } }

// PipelineRepo creates a pfs repo for a given pipeline.

PipelineRepo creates a pfs repo for a given pipeline .


func PipelineRcName(name string, version uint64) string {
	// k8s won't allow RC names that contain upper-case letters
	// or underscores
	// TODO: deal with name collision
	name = strings.Replace(name, "_", "-", -1)
	return fmt.Sprintf("pipeline-%s-v%d", strings.ToLower(name), version)
}

// PipelineRcName generates the name of the k8s replication controller that
// manages a pipeline's workers

PipelineRcName generates the name of the k8s replication controller that manages a pipeline s workers


func GetRequestsResourceListFromPipeline(pipelineInfo *pps.PipelineInfo) (*v1.ResourceList, error) {
	return getResourceListFromSpec(pipelineInfo.ResourceRequests, pipelineInfo.CacheSize)
}

func GetRequestsResourceListFromPipeline ( pipelineInfo * pps . PipelineInfo ) ( * v1 . ResourceList , error ) { return getResourceListFromSpec ( pipelineInfo . ResourceRequests , pipelineInfo . CacheSize ) }

// GetRequestsResourceListFromPipeline returns a list of resources that the pipeline,
// minimally requires.

GetRequestsResourceListFromPipeline returns a list of resources that the pipeline minimally requires .


func GetLimitsResourceListFromPipeline(pipelineInfo *pps.PipelineInfo) (*v1.ResourceList, error) {
	return getResourceListFromSpec(pipelineInfo.ResourceLimits, pipelineInfo.CacheSize)
}

func GetLimitsResourceListFromPipeline ( pipelineInfo * pps . PipelineInfo ) ( * v1 . ResourceList , error ) { return getResourceListFromSpec ( pipelineInfo . ResourceLimits , pipelineInfo . CacheSize ) }

// GetLimitsResourceListFromPipeline returns a list of resources that the pipeline,
// maximally is limited to.

GetLimitsResourceListFromPipeline returns a list of resources that the pipeline maximally is limited to .


func getNumNodes(kubeClient *kube.Clientset) (int, error) {
	nodeList, err := kubeClient.CoreV1().Nodes().List(metav1.ListOptions{})
	if err != nil {
		return 0, fmt.Errorf("unable to retrieve node list from k8s to determine parallelism: %v", err)
	}
	if len(nodeList.Items) == 0 {
		return 0, fmt.Errorf("pachyderm.pps.jobserver: no k8s nodes found")
	}
	return len(nodeList.Items), nil
}

func getNumNodes ( kubeClient * kube . Clientset ) ( int , error ) { nodeList , err := kubeClient . CoreV1 ( ) . Nodes ( ) . List ( metav1 . ListOptions { } ) if err != nil { return 0 , fmt . Errorf ( " " , err ) } if len ( nodeList . Items ) == 0 { return 0 , fmt . Errorf ( " " ) } return len ( nodeList . Items ) , nil }

// getNumNodes attempts to retrieve the number of nodes in the current k8s
// cluster

getNumNodes attempts to retrieve the number of nodes in the current k8s cluster


func GetExpectedNumWorkers(kubeClient *kube.Clientset, spec *ppsclient.ParallelismSpec) (int, error) {
	if spec == nil || (spec.Constant == 0 && spec.Coefficient == 0) {
		return 1, nil
	} else if spec.Constant > 0 && spec.Coefficient == 0 {
		return int(spec.Constant), nil
	} else if spec.Constant == 0 && spec.Coefficient > 0 {
		// Start ('coefficient' * 'nodes') workers. Determine number of workers
		numNodes, err := getNumNodes(kubeClient)
		if err != nil {
			return 0, err
		}
		result := math.Floor(spec.Coefficient * float64(numNodes))
		return int(math.Max(result, 1)), nil
	}
	return 0, fmt.Errorf("Unable to interpret ParallelismSpec %+v", spec)
}

// GetExpectedNumWorkers computes the expected number of workers that
// pachyderm will start given the ParallelismSpec 'spec'.
//
// This is only exported for testing

GetExpectedNumWorkers computes the expected number of workers that pachyderm will start given the ParallelismSpec spec . This is only exported for testing


func GetExpectedNumHashtrees(spec *ppsclient.HashtreeSpec) (int64, error) {
	if spec == nil || spec.Constant == 0 {
		return 1, nil
	} else if spec.Constant > 0 {
		return int64(spec.Constant), nil
	}
	return 0, fmt.Errorf("unable to interpret HashtreeSpec %+v", spec)
}

func GetExpectedNumHashtrees ( spec * ppsclient . HashtreeSpec ) ( int64 , error ) { if spec == nil || spec . Constant == 0 { return 1 , nil } else if spec . Constant > 0 { return int64 ( spec . Constant ) , nil } return 0 , fmt . Errorf ( " " , spec ) }

// GetExpectedNumHashtrees computes the expected number of hashtrees that
// Pachyderm will create given the HashtreeSpec 'spec'.

GetExpectedNumHashtrees computes the expected number of hashtrees that Pachyderm will create given the HashtreeSpec spec .


func FailPipeline(ctx context.Context, etcdClient *etcd.Client, pipelinesCollection col.Collection, pipelineName string, reason string) error {
	_, err := col.NewSTM(ctx, etcdClient, func(stm col.STM) error {
		pipelines := pipelinesCollection.ReadWrite(stm)
		pipelinePtr := new(pps.EtcdPipelineInfo)
		if err := pipelines.Get(pipelineName, pipelinePtr); err != nil {
			return err
		}
		pipelinePtr.State = pps.PipelineState_PIPELINE_FAILURE
		pipelinePtr.Reason = reason
		pipelines.Put(pipelineName, pipelinePtr)
		return nil
	})
	return err
}

func FailPipeline ( ctx context . Context , etcdClient * etcd . Client , pipelinesCollection col . Collection , pipelineName string , reason string ) error { _ , err := col . NewSTM ( ctx , etcdClient , func ( stm col . STM ) error { pipelines := pipelinesCollection . ReadWrite ( stm ) pipelinePtr := new ( pps . EtcdPipelineInfo ) if err := pipelines . Get ( pipelineName , pipelinePtr ) ; err != nil { return err } pipelinePtr . State = pps . PipelineState_PIPELINE_FAILURE pipelinePtr . Reason = reason pipelines . Put ( pipelineName , pipelinePtr ) return nil } ) return err }

// FailPipeline updates the pipeline's state to failed and sets the failure reason

FailPipeline updates the pipeline s state to failed and sets the failure reason


func JobInput(pipelineInfo *pps.PipelineInfo, outputCommitInfo *pfs.CommitInfo) *pps.Input {
	// branchToCommit maps strings of the form "<repo>/<branch>" to PFS commits
	branchToCommit := make(map[string]*pfs.Commit)
	key := path.Join
	for _, prov := range outputCommitInfo.Provenance {
		branchToCommit[key(prov.Commit.Repo.Name, prov.Branch.Name)] = prov.Commit
	}
	jobInput := proto.Clone(pipelineInfo.Input).(*pps.Input)
	pps.VisitInput(jobInput, func(input *pps.Input) {
		if input.Pfs != nil {
			if commit, ok := branchToCommit[key(input.Pfs.Repo, input.Pfs.Branch)]; ok {
				input.Pfs.Commit = commit.ID
			}
		}
		if input.Cron != nil {
			if commit, ok := branchToCommit[key(input.Cron.Repo, "master")]; ok {
				input.Cron.Commit = commit.ID
			}
		}
		if input.Git != nil {
			if commit, ok := branchToCommit[key(input.Git.Name, input.Git.Branch)]; ok {
				input.Git.Commit = commit.ID
			}
		}
	})
	return jobInput
}

// JobInput fills in the commits for a JobInfo


func PipelineReqFromInfo(pipelineInfo *ppsclient.PipelineInfo) *ppsclient.CreatePipelineRequest {
	return &ppsclient.CreatePipelineRequest{
		Pipeline:           pipelineInfo.Pipeline,
		Transform:          pipelineInfo.Transform,
		ParallelismSpec:    pipelineInfo.ParallelismSpec,
		HashtreeSpec:       pipelineInfo.HashtreeSpec,
		Egress:             pipelineInfo.Egress,
		OutputBranch:       pipelineInfo.OutputBranch,
		ScaleDownThreshold: pipelineInfo.ScaleDownThreshold,
		ResourceRequests:   pipelineInfo.ResourceRequests,
		ResourceLimits:     pipelineInfo.ResourceLimits,
		Input:              pipelineInfo.Input,
		Description:        pipelineInfo.Description,
		CacheSize:          pipelineInfo.CacheSize,
		EnableStats:        pipelineInfo.EnableStats,
		Batch:              pipelineInfo.Batch,
		MaxQueueSize:       pipelineInfo.MaxQueueSize,
		Service:            pipelineInfo.Service,
		ChunkSpec:          pipelineInfo.ChunkSpec,
		DatumTimeout:       pipelineInfo.DatumTimeout,
		JobTimeout:         pipelineInfo.JobTimeout,
		Salt:               pipelineInfo.Salt,
	}
}

func PipelineReqFromInfo ( pipelineInfo * ppsclient . PipelineInfo ) * ppsclient . CreatePipelineRequest { return & ppsclient . CreatePipelineRequest { Pipeline : pipelineInfo . Pipeline , Transform : pipelineInfo . Transform , ParallelismSpec : pipelineInfo . ParallelismSpec , HashtreeSpec : pipelineInfo . HashtreeSpec , Egress : pipelineInfo . Egress , OutputBranch : pipelineInfo . OutputBranch , ScaleDownThreshold : pipelineInfo . ScaleDownThreshold , ResourceRequests : pipelineInfo . ResourceRequests , ResourceLimits : pipelineInfo . ResourceLimits , Input : pipelineInfo . Input , Description : pipelineInfo . Description , CacheSize : pipelineInfo . CacheSize , EnableStats : pipelineInfo . EnableStats , Batch : pipelineInfo . Batch , MaxQueueSize : pipelineInfo . MaxQueueSize , Service : pipelineInfo . Service , ChunkSpec : pipelineInfo . ChunkSpec , DatumTimeout : pipelineInfo . DatumTimeout , JobTimeout : pipelineInfo . JobTimeout , Salt : pipelineInfo . Salt , } }

// PipelineReqFromInfo converts a PipelineInfo into a CreatePipelineRequest.

PipelineReqFromInfo converts a PipelineInfo into a CreatePipelineRequest .


func NewPipelineManifestReader(path string) (result *PipelineManifestReader, retErr error) {
	result = &PipelineManifestReader{}
	var pipelineReader io.Reader
	if path == "-" {
		pipelineReader = io.TeeReader(os.Stdin, &result.buf)
		fmt.Print("Reading from stdin.\n")
	} else if url, err := url.Parse(path); err == nil && url.Scheme != "" {
		resp, err := http.Get(url.String())
		if err != nil {
			return nil, err
		}
		defer func() {
			if err := resp.Body.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()
		rawBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		pipelineReader = io.TeeReader(strings.NewReader(string(rawBytes)), &result.buf)
	} else {
		rawBytes, err := ioutil.ReadFile(path)
		if err != nil {
			return nil, err
		}

		pipelineReader = io.TeeReader(strings.NewReader(string(rawBytes)), &result.buf)
	}
	result.decoder = json.NewDecoder(pipelineReader)
	return result, nil
}

func NewPipelineManifestReader ( path string ) ( result * PipelineManifestReader , retErr error ) { result = & PipelineManifestReader { } var pipelineReader io . Reader if path == " " { pipelineReader = io . TeeReader ( os . Stdin , & result . buf ) fmt . Print ( " \n " ) } else if url , err := url . Parse ( path ) ; err == nil && url . Scheme != " " { resp , err := http . Get ( url . String ( ) ) if err != nil { return nil , err } defer func ( ) { if err := resp . Body . Close ( ) ; err != nil && retErr == nil { retErr = err } } ( ) rawBytes , err := ioutil . ReadAll ( resp . Body ) if err != nil { return nil , err } pipelineReader = io . TeeReader ( strings . NewReader ( string ( rawBytes ) ) , & result . buf ) } else { rawBytes , err := ioutil . ReadFile ( path ) if err != nil { return nil , err } pipelineReader = io . TeeReader ( strings . NewReader ( string ( rawBytes ) ) , & result . buf ) } result . decoder = json . NewDecoder ( pipelineReader ) return result , nil }

// NewPipelineManifestReader creates a new manifest reader from a path.

NewPipelineManifestReader creates a new manifest reader from a path .


func (r *PipelineManifestReader) NextCreatePipelineRequest() (*ppsclient.CreatePipelineRequest, error) {
	var result ppsclient.CreatePipelineRequest
	if err := jsonpb.UnmarshalNext(r.decoder, &result); err != nil {
		if err == io.EOF {
			return nil, err
		}
		return nil, fmt.Errorf("malformed pipeline spec: %s", err)
	}
	return &result, nil
}

func ( r * PipelineManifestReader ) NextCreatePipelineRequest ( ) ( * ppsclient . CreatePipelineRequest , error ) { var result ppsclient . CreatePipelineRequest if err := jsonpb . UnmarshalNext ( r . decoder , & result ) ; err != nil { if err == io . EOF { return nil , err } return nil , fmt . Errorf ( " " , err ) } return & result , nil }

// NextCreatePipelineRequest gets the next request from the manifest reader.

NextCreatePipelineRequest gets the next request from the manifest reader .


func DescribeSyntaxError(originalErr error, parsedBuffer bytes.Buffer) error {

	sErr, ok := originalErr.(*json.SyntaxError)
	if !ok {
		return originalErr
	}

	buffer := make([]byte, sErr.Offset)
	parsedBuffer.Read(buffer)

	lineOffset := strings.LastIndex(string(buffer[:len(buffer)-1]), "\n")
	if lineOffset == -1 {
		lineOffset = 0
	}

	lines := strings.Split(string(buffer[:len(buffer)-1]), "\n")
	lineNumber := len(lines)

	descriptiveErrorString := fmt.Sprintf("Syntax Error on line %v:\n%v\n%v^\n%v\n",
		lineNumber,
		string(buffer[lineOffset:]),
		strings.Repeat(" ", int(sErr.Offset)-2-lineOffset),
		originalErr,
	)

	return errors.New(descriptiveErrorString)
}

func DescribeSyntaxError ( originalErr error , parsedBuffer bytes . Buffer ) error { sErr , ok := originalErr . ( * json . SyntaxError ) if ! ok { return originalErr } buffer := make ( [ ] byte , sErr . Offset ) parsedBuffer . Read ( buffer ) lineOffset := strings . LastIndex ( string ( buffer [ : len ( buffer ) - 1 ] ) , " \n " ) if lineOffset == - 1 { lineOffset = 0 } lines := strings . Split ( string ( buffer [ : len ( buffer ) - 1 ] ) , " \n " ) lineNumber := len ( lines ) descriptiveErrorString := fmt . Sprintf ( " \n \n \n \n " , lineNumber , string ( buffer [ lineOffset : ] ) , strings . Repeat ( " " , int ( sErr . Offset ) - 2 - lineOffset ) , originalErr , ) return errors . New ( descriptiveErrorString ) }

// DescribeSyntaxError describes a syntax error encountered parsing json.

DescribeSyntaxError describes a syntax error encountered parsing json .


func UpdateJobState(pipelines col.ReadWriteCollection, jobs col.ReadWriteCollection, jobPtr *pps.EtcdJobInfo, state pps.JobState, reason string) error {
	// Update pipeline
	pipelinePtr := &pps.EtcdPipelineInfo{}
	if err := pipelines.Get(jobPtr.Pipeline.Name, pipelinePtr); err != nil {
		return err
	}
	if pipelinePtr.JobCounts == nil {
		pipelinePtr.JobCounts = make(map[int32]int32)
	}
	if pipelinePtr.JobCounts[int32(jobPtr.State)] != 0 {
		pipelinePtr.JobCounts[int32(jobPtr.State)]--
	}
	pipelinePtr.JobCounts[int32(state)]++
	pipelinePtr.LastJobState = state
	if err := pipelines.Put(jobPtr.Pipeline.Name, pipelinePtr); err != nil {
		return err
	}

	// Update job info
	var err error
	if state == pps.JobState_JOB_STARTING {
		jobPtr.Started, err = types.TimestampProto(time.Now())
	} else if IsTerminal(state) {
		jobPtr.Finished, err = types.TimestampProto(time.Now())
	}
	if err != nil {
		return err
	}
	jobPtr.State = state
	jobPtr.Reason = reason
	return jobs.Put(jobPtr.Job.ID, jobPtr)
}

func
UpdateJobState
(
pipelines
col
.
ReadWriteCollection
,
jobs
col
.
ReadWriteCollection
,
jobPtr
*
pps
.
EtcdJobInfo
,
state
pps
.
JobState
,
reason
string
)
error
{
// Update pipeline
pipelinePtr
:=
&
pps
.
EtcdPipelineInfo
{
}
if
err
:=
pipelines
.
Get
(
jobPtr
.
Pipeline
.
Name
,
pipelinePtr
)
;
err
!=
{
return
err
}
if
pipelinePtr
.
JobCounts
==
{
pipelinePtr
.
JobCounts
=
make
(
map
[
int32
]
int32
)
}
if
pipelinePtr
.
JobCounts
[
int32
(
jobPtr
.
State
)
]
!=
0
{
pipelinePtr
.
JobCounts
[
int32
(
jobPtr
.
State
)
]
}
pipelinePtr
.
JobCounts
[
int32
(
state
)
]
++
pipelinePtr
.
LastJobState
=
state
if
err
:=
pipelines
.
Put
(
jobPtr
.
Pipeline
.
Name
,
pipelinePtr
)
;
err
!=
{
return
err
}
// Update job info
var
err
error
if
state
==
pps
.
JobState_JOB_STARTING
{
jobPtr
.
Started
,
err
=
types
.
TimestampProto
(
time
.
Now
(
)
)
}
else
if
IsTerminal
(
state
)
{
jobPtr
.
Finished
,
err
=
types
.
TimestampProto
(
time
.
Now
(
)
)
}
if
err
!=
{
return
err
}
jobPtr
.
State
=
state
jobPtr
.
Reason
=
reason
return
jobs
.
Put
(
jobPtr
.
Job
.
ID
,
jobPtr
)
}

// UpdateJobState performs the operations involved with a job state transition.

UpdateJobState performs the operations involved with a job state transition .


func New() string {
	var result string
	backoff.RetryNotify(func() error {
		uuid, err := uuid.NewV4()
		if err != nil {
			return err
		}
		result = uuid.String()
		return nil
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		fmt.Printf("error from uuid.NewV4: %v", err)
		return nil
	})
	return result
}

func New ( ) string { var result string backoff . RetryNotify ( func ( ) error { uuid , err := uuid . NewV4 ( ) if err != nil { return err } result = uuid . String ( ) return nil } , backoff . NewInfiniteBackOff ( ) , func ( err error , d time . Duration ) error { fmt . Printf ( " " , err ) return nil } ) return result }


func (h *HTTPError) Code() int {
	if h == nil {
		return http.StatusOK
	}
	return h.code
}

func ( h * HTTPError ) Code ( ) int { if h == nil { return http . StatusOK } return h . code }

// Code returns the HTTP error code associated with 'h'

Code returns the HTTP error code associated with h


func NewHTTPError(code int, formatStr string, args ...interface{}) *HTTPError {
	return &HTTPError{
		code: code,
		err:  fmt.Sprintf(formatStr, args...),
	}
}

func NewHTTPError ( code int , formatStr string , args ... interface { } ) * HTTPError { return & HTTPError { code : code , err : fmt . Sprintf ( formatStr , args ... ) , } }

// NewHTTPError returns a new HTTPError where the HTTP error code is 'code' and
// the error message is based on 'formatStr' and 'args'

NewHTTPError returns a new HTTPError where the HTTP error code is code and the error message is based on formatStr and args


func NewStorage(objC obj.Client, prefix string) *Storage {
	return &Storage{
		objC:   objC,
		prefix: prefix,
	}
}

func NewStorage ( objC obj . Client , prefix string ) * Storage { return & Storage { objC : objC , prefix : prefix , } }


func (s *Storage) DeleteAll(ctx context.Context) error {
	return s.objC.Walk(ctx, s.prefix, func(hash string) error {
		return s.objC.Delete(ctx, hash)
	})
}

func ( s * Storage ) DeleteAll ( ctx context . Context ) error { return s . objC . Walk ( ctx , s . prefix , func ( hash string ) error { return s . objC . Delete ( ctx , hash ) } ) }

// DeleteAll deletes all of the chunks in object storage.

DeleteAll deletes all of the chunks in object storage .


func Chunk(data []byte, chunkSize int) [][]byte {
	var result [][]byte
	for i := 0; i < len(data); i += chunkSize {
		end := i + chunkSize
		if end > len(data) {
			end = len(data)
		}
		result = append(result, data[i:end])
	}
	return result
}

func Chunk ( data [ ] byte , chunkSize int ) [ ] [ ] byte { var result [ ] [ ] byte for i := 0 ; i < len ( data ) ; i += chunkSize { end := i + chunkSize if end > len ( data ) { end = len ( data ) } result = append ( result , data [ i : end ] ) } return result }

// Chunk splits a piece of data up, this is useful for splitting up data that's
// bigger than MaxMsgSize

Chunk splits a piece of data up this is useful for splitting up data that s bigger than MaxMsgSize


func ChunkReader(r io.Reader, f func([]byte) error) (int, error) {
	var total int
	buf := GetBuffer()
	defer PutBuffer(buf)
	for {
		n, err := r.Read(buf)
		if n == 0 && err != nil {
			if err == io.EOF {
				return total, nil
			}
			return total, err
		}
		if err := f(buf[:n]); err != nil {
			return total, err
		}
		total += n
	}
}

func ChunkReader ( r io . Reader , f func ( [ ] byte ) error ) ( int , error ) { var total int buf := GetBuffer ( ) defer PutBuffer ( buf ) for { n , err := r . Read ( buf ) if n == 0 && err != nil { if err == io . EOF { return total , nil } return total , err } if err := f ( buf [ : n ] ) ; err != nil { return total , err } total += n } }

// ChunkReader splits a reader into reasonably sized chunks for the purpose
// of transmitting the chunks over gRPC. For each chunk, it calls the given
// function.

ChunkReader splits a reader into reasonably sized chunks for the purpose of transmitting the chunks over gRPC . For each chunk it calls the given function .


func NewStreamingBytesReader(streamingBytesClient StreamingBytesClient, cancel context.CancelFunc) io.ReadCloser {
	return &streamingBytesReader{streamingBytesClient: streamingBytesClient, cancel: cancel}
}

func NewStreamingBytesReader ( streamingBytesClient StreamingBytesClient , cancel context . CancelFunc ) io . ReadCloser { return & streamingBytesReader { streamingBytesClient : streamingBytesClient , cancel : cancel } }

// NewStreamingBytesReader returns an io.Reader for a StreamingBytesClient.

NewStreamingBytesReader returns an io . Reader for a StreamingBytesClient .


func WriteToStreamingBytesServer(reader io.Reader, streamingBytesServer StreamingBytesServer) error {
	buf := GetBuffer()
	defer PutBuffer(buf)
	_, err := io.CopyBuffer(NewStreamingBytesWriter(streamingBytesServer), ReaderWrapper{reader}, buf)
	return err
}

func WriteToStreamingBytesServer ( reader io . Reader , streamingBytesServer StreamingBytesServer ) error { buf := GetBuffer ( ) defer PutBuffer ( buf ) _ , err := io . CopyBuffer ( NewStreamingBytesWriter ( streamingBytesServer ) , ReaderWrapper { reader } , buf ) return err }

// WriteToStreamingBytesServer writes the data from the io.Reader to the StreamingBytesServer.

WriteToStreamingBytesServer writes the data from the io . Reader to the StreamingBytesServer .


func WriteFromStreamingBytesClient(streamingBytesClient StreamingBytesClient, writer io.Writer) error {
	for bytesValue, err := streamingBytesClient.Recv(); err != io.EOF; bytesValue, err = streamingBytesClient.Recv() {
		if err != nil {
			return err
		}
		if _, err = writer.Write(bytesValue.Value); err != nil {
			return err
		}
	}
	return nil
}

func WriteFromStreamingBytesClient ( streamingBytesClient StreamingBytesClient , writer io . Writer ) error { for bytesValue , err := streamingBytesClient . Recv ( ) ; err != io . EOF ; bytesValue , err = streamingBytesClient . Recv ( ) { if err != nil { return err } if _ , err = writer . Write ( bytesValue . Value ) ; err != nil { return err } } return nil }

// WriteFromStreamingBytesClient writes from the StreamingBytesClient to the io.Writer.

WriteFromStreamingBytesClient writes from the StreamingBytesClient to the io . Writer .


func NewSidecarAPIServer(
	env *serviceenv.ServiceEnv,
	etcdPrefix string,
	iamRole string,
	reporter *metrics.Reporter,
	workerGrpcPort uint16,
	pprofPort uint16,
	httpPort uint16,
	peerPort uint16,
) (ppsclient.APIServer, error) {
	apiServer := &apiServer{
		Logger:         log.NewLogger("pps.API"),
		env:            env,
		etcdPrefix:     etcdPrefix,
		iamRole:        iamRole,
		reporter:       reporter,
		workerUsesRoot: true,
		pipelines:      ppsdb.Pipelines(env.GetEtcdClient(), etcdPrefix),
		jobs:           ppsdb.Jobs(env.GetEtcdClient(), etcdPrefix),
		workerGrpcPort: workerGrpcPort,
		pprofPort:      pprofPort,
		httpPort:       httpPort,
		peerPort:       peerPort,
	}
	return apiServer, nil
}

func NewSidecarAPIServer ( env * serviceenv . ServiceEnv , etcdPrefix string , iamRole string , reporter * metrics . Reporter , workerGrpcPort uint16 , pprofPort uint16 , httpPort uint16 , peerPort uint16 , ) ( ppsclient . APIServer , error ) { apiServer := & apiServer { Logger : log . NewLogger ( " " ) , env : env , etcdPrefix : etcdPrefix , iamRole : iamRole , reporter : reporter , workerUsesRoot : true , pipelines : ppsdb . Pipelines ( env . GetEtcdClient ( ) , etcdPrefix ) , jobs : ppsdb . Jobs ( env . GetEtcdClient ( ) , etcdPrefix ) , workerGrpcPort : workerGrpcPort , pprofPort : pprofPort , httpPort : httpPort , peerPort : peerPort , } return apiServer , nil }

// NewSidecarAPIServer creates an APIServer that has limited functionalities
// and is meant to be run as a worker sidecar.  It cannot, for instance,
// create pipelines.

NewSidecarAPIServer creates an APIServer that has limited functionalities and is meant to be run as a worker sidecar . It cannot for instance create pipelines .


func NewEnterpriseServer(env *serviceenv.ServiceEnv, etcdPrefix string) (ec.APIServer, error) {
	s := &apiServer{
		pachLogger: log.NewLogger("enterprise.API"),
		env:        env,
		enterpriseToken: col.NewCollection(
			env.GetEtcdClient(),
			etcdPrefix, // only one collection--no extra prefix needed
			nil,
			&ec.EnterpriseRecord{},
			nil,
			nil,
		),
	}
	s.enterpriseExpiration.Store(time.Time{})
	go s.watchEnterpriseToken(etcdPrefix)
	return s, nil
}

// NewEnterpriseServer returns an implementation of ec.APIServer.

NewEnterpriseServer returns an implementation of ec . APIServer .


func validateActivationCode(code string) (expiration time.Time, err error) {
	// Parse the public key.  If these steps fail, something is seriously
	// wrong and we should crash the service by panicking.
	block, _ := pem.Decode([]byte(publicKey))
	if block == nil {
		return time.Time{}, fmt.Errorf("failed to pem decode public key")
	}
	pub, err := x509.ParsePKIXPublicKey(block.Bytes)
	if err != nil {
		return time.Time{}, fmt.Errorf("failed to parse DER encoded public key: %s", err.Error())
	}
	rsaPub, ok := pub.(*rsa.PublicKey)
	if !ok {
		return time.Time{}, fmt.Errorf("public key isn't an RSA key")
	}

	// Decode the base64-encoded activation code
	decodedActivationCode, err := base64.StdEncoding.DecodeString(code)
	if err != nil {
		return time.Time{}, fmt.Errorf("activation code is not base64 encoded")
	}
	activationCode := &activationCode{}
	if err := json.Unmarshal(decodedActivationCode, &activationCode); err != nil {
		return time.Time{}, fmt.Errorf("activation code is not valid JSON")
	}

	// Decode the signature
	decodedSignature, err := base64.StdEncoding.DecodeString(activationCode.Signature)
	if err != nil {
		return time.Time{}, fmt.Errorf("signature is not base64 encoded")
	}

	// Compute the sha256 checksum of the token
	hashedToken := sha256.Sum256([]byte(activationCode.Token))

	// Verify that the signature is valid
	if err := rsa.VerifyPKCS1v15(rsaPub, crypto.SHA256, hashedToken[:], decodedSignature); err != nil {
		return time.Time{}, fmt.Errorf("invalid signature in activation code")
	}

	// Unmarshal the token
	token := token{}
	if err := json.Unmarshal([]byte(activationCode.Token), &token); err != nil {
		return time.Time{}, fmt.Errorf("token is not valid JSON")
	}

	// Parse the expiration. Note that this string is generated by Date.toJSON()
	// running in node, so Go's definition of RFC 3339 timestamps (which is
	// incomplete) must be compatible with the strings that node generates. So far
	// it seems to work.
	expiration, err = time.Parse(time.RFC3339, token.Expiry)
	if err != nil {
		return time.Time{}, fmt.Errorf("expiration is not valid ISO 8601 string")
	}
	// Check that the activation code has not expired
	if time.Now().After(expiration) {
		return time.Time{}, fmt.Errorf("the activation code has expired")
	}
	return expiration, nil
}

// validateActivationCode checks the validity of an activation code


func (a *apiServer) Activate(ctx context.Context, req *ec.ActivateRequest) (resp *ec.ActivateResponse, retErr error) {
	a.LogReq(req)
	defer func(start time.Time) { a.pachLogger.Log(req, resp, retErr, time.Since(start)) }(time.Now())

	// Validate the activation code
	expiration, err := validateActivationCode(req.ActivationCode)
	if err != nil {
		return nil, fmt.Errorf("error validating activation code: %s", err.Error())
	}
	// Allow request to override expiration in the activation code, for testing
	if req.Expires != nil {
		customExpiration, err := types.TimestampFromProto(req.Expires)
		if err == nil && expiration.After(customExpiration) {
			expiration = customExpiration
		}
	}
	expirationProto, err := types.TimestampProto(expiration)
	if err != nil {
		return nil, fmt.Errorf("could not convert expiration time \"%s\" to proto: %s", expiration.String(), err.Error())
	}
	if _, err := col.NewSTM(ctx, a.env.GetEtcdClient(), func(stm col.STM) error {
		e := a.enterpriseToken.ReadWrite(stm)
		// blind write
		return e.Put(enterpriseTokenKey, &ec.EnterpriseRecord{
			ActivationCode: req.ActivationCode,
			Expires:        expirationProto,
		})
	}); err != nil {
		return nil, err
	}

	// Wait until watcher observes the write
	if err := backoff.Retry(func() error {
		if t := a.enterpriseExpiration.Load().(time.Time); t.IsZero() {
			return fmt.Errorf("enterprise not activated")
		}
		return nil
	}, backoff.RetryEvery(time.Second)); err != nil {
		return nil, err
	}
	time.Sleep(time.Second) // give other pachd nodes time to observe the write

	return &ec.ActivateResponse{
		Info: &ec.TokenInfo{
			Expires: expirationProto,
		},
	}, nil
}


func (a *apiServer) Deactivate(ctx context.Context, req *ec.DeactivateRequest) (resp *ec.DeactivateResponse, retErr error) {
	a.LogReq(req)
	defer func(start time.Time) { a.pachLogger.Log(req, resp, retErr, time.Since(start)) }(time.Now())

	pachClient := a.env.GetPachClient(ctx)
	if err := pachClient.DeleteAll(); err != nil {
		return nil, fmt.Errorf("could not delete all pachyderm data: %v", err)
	}

	if _, err := col.NewSTM(ctx, a.env.GetEtcdClient(), func(stm col.STM) error {
		// blind delete
		return a.enterpriseToken.ReadWrite(stm).Delete(enterpriseTokenKey)
	}); err != nil {
		return nil, err
	}

	// Wait until watcher observes the write
	if err := backoff.Retry(func() error {
		if t := a.enterpriseExpiration.Load().(time.Time); !t.IsZero() {
			return fmt.Errorf("enterprise still activated")
		}
		return nil
	}, backoff.RetryEvery(time.Second)); err != nil {
		return nil, err
	}
	time.Sleep(time.Second) // give other pachd nodes time to observe the write

	return &ec.DeactivateResponse{}, nil
}

// Deactivate deletes the current cluster's enterprise token, and puts the
// cluster in the "NONE" enterprise state. It also deletes all data in the
// cluster, to avoid invalid cluster states. This call only makes sense for
// testing

Deactivate deletes the current cluster s enterprise token and puts the cluster in the NONE enterprise state . It also deletes all data in the cluster to avoid invalid cluster states . This call only makes sense for testing


func lookExtensions(path, dir string) (string, error) {
	if filepath.Base(path) == path {
		path = filepath.Join(".", path)
	}
	if dir == "" {
		return exec.LookPath(path)
	}
	if filepath.VolumeName(path) != "" {
		return exec.LookPath(path)
	}
	if len(path) > 1 && os.IsPathSeparator(path[0]) {
		return exec.LookPath(path)
	}
	dirandpath := filepath.Join(dir, path)
	// We assume that LookPath will only add file extension.
	lp, err := exec.LookPath(dirandpath)
	if err != nil {
		return "", err
	}
	ext := strings.TrimPrefix(lp, dirandpath)
	return path + ext, nil
}

// lookExtensions finds windows executable by its dir and path.
// It uses LookPath to try appropriate extensions.
// lookExtensions does not search PATH, instead it converts `prog` into `.\prog`.

lookExtensions finds windows executable by its dir and path . It uses LookPath to try appropriate extensions . lookExtensions does not search PATH instead it converts prog into . \ prog .


func (c *Cmd) Start() error {
	if c.lookPathErr != nil {
		c.closeDescriptors(c.closeAfterStart)
		c.closeDescriptors(c.closeAfterWait)
		return c.lookPathErr
	}
	if runtime.GOOS == "windows" {
		lp, err := lookExtensions(c.Path, c.Dir)
		if err != nil {
			c.closeDescriptors(c.closeAfterStart)
			c.closeDescriptors(c.closeAfterWait)
			return err
		}
		c.Path = lp
	}
	if c.Process != nil {
		return errors.New("exec: already started")
	}
	if c.ctx != nil {
		select {
		case <-c.ctx.Done():
			c.closeDescriptors(c.closeAfterStart)
			c.closeDescriptors(c.closeAfterWait)
			return c.ctx.Err()
		default:
		}
	}

	type F func(*Cmd) (*os.File, error)
	for _, setupFd := range []F{(*Cmd).stdin, (*Cmd).stdout, (*Cmd).stderr} {
		fd, err := setupFd(c)
		if err != nil {
			c.closeDescriptors(c.closeAfterStart)
			c.closeDescriptors(c.closeAfterWait)
			return err
		}
		c.childFiles = append(c.childFiles, fd)
	}
	c.childFiles = append(c.childFiles, c.ExtraFiles...)

	var err error
	c.Process, err = os.StartProcess(c.Path, c.argv(), &os.ProcAttr{
		Dir:   c.Dir,
		Files: c.childFiles,
		Env:   dedupEnv(c.envv()),
		Sys:   c.SysProcAttr,
	})
	if err != nil {
		c.closeDescriptors(c.closeAfterStart)
		c.closeDescriptors(c.closeAfterWait)
		return err
	}

	c.closeDescriptors(c.closeAfterStart)

	c.errch = make(chan error, len(c.goroutine))
	for _, fn := range c.goroutine {
		go func(fn func() error) {
			c.errch <- fn()
		}(fn)
	}

	if c.ctx != nil {
		c.waitDone = make(chan struct{})
		go func() {
			select {
			case <-c.ctx.Done():
				c.Process.Kill()
			case <-c.waitDone:
			}
		}()
	}

	return nil
}

func ( c * Cmd ) Start ( ) error { if c . lookPathErr != nil { c . closeDescriptors ( c . closeAfterStart ) c . closeDescriptors ( c . closeAfterWait ) return c . lookPathErr } if runtime . GOOS == " " { lp , err := lookExtensions ( c . Path , c . Dir ) if err != nil { c . closeDescriptors ( c . closeAfterStart ) c . closeDescriptors ( c . closeAfterWait ) return err } c . Path = lp } if c . Process != nil { return errors . New ( " " ) } if c . ctx != nil { select { case <- c . ctx . Done ( ) : c . closeDescriptors ( c . closeAfterStart ) c . closeDescriptors ( c . closeAfterWait ) return c . ctx . Err ( ) default : } } type F func ( * Cmd ) ( * os . File , error ) for _ , setupFd := range [ ] F { ( * Cmd ) . stdin , ( * Cmd ) . stdout , ( * Cmd ) . stderr } { fd , err := setupFd ( c ) if err != nil { c . closeDescriptors ( c . closeAfterStart ) c . closeDescriptors ( c . closeAfterWait ) return err } c . childFiles = append ( c . childFiles , fd ) } c . childFiles = append ( c . childFiles , c . ExtraFiles ... ) var err error c . Process , err = os . StartProcess ( c . Path , c . argv ( ) , & os . ProcAttr { Dir : c . Dir , Files : c . childFiles , Env : dedupEnv ( c . envv ( ) ) , Sys : c . SysProcAttr , } ) if err != nil { c . closeDescriptors ( c . closeAfterStart ) c . closeDescriptors ( c . closeAfterWait ) return err } c . closeDescriptors ( c . closeAfterStart ) c . errch = make ( chan error , len ( c . goroutine ) ) for _ , fn := range c . goroutine { go func ( fn func ( ) error ) { c . errch <- fn ( ) } ( fn ) } if c . ctx != nil { c . waitDone = make ( chan struct { } ) go func ( ) { select { case <- c . ctx . Done ( ) : c . Process . Kill ( ) case <- c . waitDone : } } ( ) } return nil }

// Start starts the specified command but does not wait for it to complete.
//
// The Wait method will return the exit code and release associated resources
// once the command exits.

Start starts the specified command but does not wait for it to complete . The Wait method will return the exit code and release associated resources once the command exits .


func (c *Cmd) CombinedOutput() ([]byte, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	if c.Stderr != nil {
		return nil, errors.New("exec: Stderr already set")
	}
	var b bytes.Buffer
	c.Stdout = &b
	c.Stderr = &b
	err := c.Run()
	return b.Bytes(), err
}

func ( c * Cmd ) CombinedOutput ( ) ( [ ] byte , error ) { if c . Stdout != nil { return nil , errors . New ( " " ) } if c . Stderr != nil { return nil , errors . New ( " " ) } var b bytes . Buffer c . Stdout = & b c . Stderr = & b err := c . Run ( ) return b . Bytes ( ) , err }

// CombinedOutput runs the command and returns its combined standard
// output and standard error.

CombinedOutput runs the command and returns its combined standard output and standard error .


func (c *Cmd) StdinPipe() (io.WriteCloser, error) {
	if c.Stdin != nil {
		return nil, errors.New("exec: Stdin already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StdinPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stdin = pr
	c.closeAfterStart = append(c.closeAfterStart, pr)
	wc := &closeOnce{File: pw}
	c.closeAfterWait = append(c.closeAfterWait, closerFunc(wc.safeClose))
	return wc, nil
}

func ( c * Cmd ) StdinPipe ( ) ( io . WriteCloser , error ) { if c . Stdin != nil { return nil , errors . New ( " " ) } if c . Process != nil { return nil , errors . New ( " " ) } pr , pw , err := os . Pipe ( ) if err != nil { return nil , err } c . Stdin = pr c . closeAfterStart = append ( c . closeAfterStart , pr ) wc := & closeOnce { File : pw } c . closeAfterWait = append ( c . closeAfterWait , closerFunc ( wc . safeClose ) ) return wc , nil }

// StdinPipe returns a pipe that will be connected to the command's
// standard input when the command starts.
// The pipe will be closed automatically after Wait sees the command exit.
// A caller need only call Close to force the pipe to close sooner.
// For example, if the command being run will not exit until standard input
// is closed, the caller must close the pipe.

StdinPipe returns a pipe that will be connected to the command s standard input when the command starts . The pipe will be closed automatically after Wait sees the command exit . A caller need only call Close to force the pipe to close sooner . For example if the command being run will not exit until standard input is closed the caller must close the pipe .


func (c *Cmd) StdoutPipe() (io.ReadCloser, error) {
	if c.Stdout != nil {
		return nil, errors.New("exec: Stdout already set")
	}
	if c.Process != nil {
		return nil, errors.New("exec: StdoutPipe after process started")
	}
	pr, pw, err := os.Pipe()
	if err != nil {
		return nil, err
	}
	c.Stdout = pw
	c.closeAfterStart = append(c.closeAfterStart, pw)
	c.closeAfterWait = append(c.closeAfterWait, pr)
	return pr, nil
}

func ( c * Cmd ) StdoutPipe ( ) ( io . ReadCloser , error ) { if c . Stdout != nil { return nil , errors . New ( " " ) } if c . Process != nil { return nil , errors . New ( " " ) } pr , pw , err := os . Pipe ( ) if err != nil { return nil , err } c . Stdout = pw c . closeAfterStart = append ( c . closeAfterStart , pw ) c . closeAfterWait = append ( c . closeAfterWait , pr ) return pr , nil }

// StdoutPipe returns a pipe that will be connected to the command's
// standard output when the command starts.
//
// Wait will close the pipe after seeing the command exit, so most callers
// need not close the pipe themselves; however, an implication is that
// it is incorrect to call Wait before all reads from the pipe have completed.
// For the same reason, it is incorrect to call Run when using StdoutPipe.
// See the example for idiomatic usage.

StdoutPipe returns a pipe that will be connected to the command s standard output when the command starts . Wait will close the pipe after seeing the command exit so most callers need not close the pipe themselves ; however an implication is that it is incorrect to call Wait before all reads from the pipe have completed . For the same reason it is incorrect to call Run when using StdoutPipe . See the example for idiomatic usage .


func dedupEnvCase(caseInsensitive bool, env []string) []string {
	out := make([]string, 0, len(env))
	saw := map[string]int{} // key => index into out
	for _, kv := range env {
		eq := strings.Index(kv, "=")
		if eq < 0 {
			out = append(out, kv)
			continue
		}
		k := kv[:eq]
		if caseInsensitive {
			k = strings.ToLower(k)
		}
		if dupIdx, isDup := saw[k]; isDup {
			out[dupIdx] = kv
			continue
		}
		saw[k] = len(out)
		out = append(out, kv)
	}
	return out
}

func
dedupEnvCase
(
caseInsensitive
bool
,
env
[
]
string
)
[
]
string
{
out
:=
make
(
[
]
string
,
0
,
len
(
env
)
)
saw
:=
map
[
string
]
int
{
}
// key => index into out
for
_
,
kv
:=
range
env
{
eq
:=
strings
.
Index
(
kv
,
"
"
)
if
eq
<
0
{
out
=
append
(
out
,
kv
)
continue
}
k
:=
kv
[
:
eq
]
if
caseInsensitive
{
k
=
strings
.
ToLower
(
k
)
}
if
dupIdx
,
isDup
:=
saw
[
k
]
;
isDup
{
out
[
dupIdx
]
=
kv
continue
}
saw
[
k
]
=
len
(
out
)
out
=
append
(
out
,
kv
)
}
return
out
}

// dedupEnvCase is dedupEnv with a case option for testing.
// If caseInsensitive is true, the case of keys is ignored.

dedupEnvCase is dedupEnv with a case option for testing . If caseInsensitive is true the case of keys is ignored .


func InputName(input *Input) string {
	switch {
	case input == nil:
		return ""
	case input.Pfs != nil:
		return input.Pfs.Name
	case input.Cross != nil:
		if len(input.Cross) > 0 {
			return InputName(input.Cross[0])
		}
	case input.Union != nil:
		if len(input.Union) > 0 {
			return InputName(input.Union[0])
		}
	}
	return ""
}

func InputName ( input * Input ) string { switch { case input == nil : return " " case input . Pfs != nil : return input . Pfs . Name case input . Cross != nil : if len ( input . Cross ) > 0 { return InputName ( input . Cross [ 0 ] ) } case input . Union != nil : if len ( input . Union ) > 0 { return InputName ( input . Union [ 0 ] ) } } return " " }

// InputName computes the name of an Input.

InputName computes the name of an Input .


func SortInput(input *Input) {
	VisitInput(input, func(input *Input) {
		SortInputs := func(inputs []*Input) {
			sort.SliceStable(inputs, func(i, j int) bool { return InputName(inputs[i]) < InputName(inputs[j]) })
		}
		switch {
		case input.Cross != nil:
			SortInputs(input.Cross)
		case input.Union != nil:
			SortInputs(input.Union)
		}
	})
}

func SortInput ( input * Input ) { VisitInput ( input , func ( input * Input ) { SortInputs := func ( inputs [ ] * Input ) { sort . SliceStable ( inputs , func ( i , j int ) bool { return InputName ( inputs [ i ] ) < InputName ( inputs [ j ] ) } ) } switch { case input . Cross != nil : SortInputs ( input . Cross ) case input . Union != nil : SortInputs ( input . Union ) } } ) }


func InputBranches(input *Input) []*pfs.Branch {
	var result []*pfs.Branch
	VisitInput(input, func(input *Input) {
		if input.Pfs != nil {
			result = append(result, &pfs.Branch{
				Repo: &pfs.Repo{Name: input.Pfs.Repo},
				Name: input.Pfs.Branch,
			})
		}
		if input.Cron != nil {
			result = append(result, &pfs.Branch{
				Repo: &pfs.Repo{Name: input.Cron.Repo},
				Name: "master",
			})
		}
		if input.Git != nil {
			result = append(result, &pfs.Branch{
				Repo: &pfs.Repo{Name: input.Git.Name},
				Name: input.Git.Branch,
			})
		}
	})
	return result
}

func InputBranches ( input * Input ) [ ] * pfs . Branch { var result [ ] * pfs . Branch VisitInput ( input , func ( input * Input ) { if input . Pfs != nil { result = append ( result , & pfs . Branch { Repo : & pfs . Repo { Name : input . Pfs . Repo } , Name : input . Pfs . Branch , } ) } if input . Cron != nil { result = append ( result , & pfs . Branch { Repo : & pfs . Repo { Name : input . Cron . Repo } , Name : " " , } ) } if input . Git != nil { result = append ( result , & pfs . Branch { Repo : & pfs . Repo { Name : input . Git . Name } , Name : input . Git . Branch , } ) } } ) return result }

// InputBranches returns the branches in an Input.

InputBranches returns the branches in an Input .


func ValidateGitCloneURL(url string) error {
	exampleURL := "https://github.com/org/foo.git"
	if url == "" {
		return fmt.Errorf("clone URL is missing (example clone URL %v)", exampleURL)
	}
	// Use the git client's validator to make sure its a valid URL
	o := &git.CloneOptions{
		URL: url,
	}
	if err := o.Validate(); err != nil {
		return err
	}

	// Make sure its the type that we want. Of the following we
	// only accept the 'clone' type of url:
	//     git_url: "git://github.com/sjezewski/testgithook.git",
	//     ssh_url: "git@github.com:sjezewski/testgithook.git",
	//     clone_url: "https://github.com/sjezewski/testgithook.git",
	//     svn_url: "https://github.com/sjezewski/testgithook",
	invalidErr := fmt.Errorf("clone URL is missing .git suffix (example clone URL %v)", exampleURL)

	if !strings.HasSuffix(url, ".git") {
		// svn_url case
		return invalidErr
	}
	if !strings.HasPrefix(url, "https://") {
		// git_url or ssh_url cases
		return invalidErr
	}

	return nil
}

// ValidateGitCloneURL returns an error if the provided URL is invalid


func containsEmpty(vals []string) bool {
	for _, val := range vals {
		if val == "" {
			return true
		}
	}
	return false
}

func containsEmpty ( vals [ ] string ) bool { for _ , val := range vals { if val == " " { return true } } return false }

// containsEmpty is a helper function used for validation (particularly for
// validating that creds arguments aren't empty

containsEmpty is a helper function used for validation ( particularly for validating that creds arguments aren t empty


func NewAPIServer(address string, storageRoot string, clusterInfo *admin.ClusterInfo) APIServer {
	return &apiServer{
		Logger:      log.NewLogger("admin.API"),
		address:     address,
		storageRoot: storageRoot,
		clusterInfo: clusterInfo,
	}
}

func NewAPIServer ( address string , storageRoot string , clusterInfo * admin . ClusterInfo ) APIServer { return & apiServer { Logger : log . NewLogger ( " " ) , address : address , storageRoot : storageRoot , clusterInfo : clusterInfo , } }

// NewAPIServer returns a new admin.APIServer

NewAPIServer returns a new admin . APIServer


func Ago(timestamp *types.Timestamp) string {
	t, _ := types.TimestampFromProto(timestamp)
	if t.Equal(time.Time{}) {
		return ""
	}
	return fmt.Sprintf("%s ago", units.HumanDuration(time.Since(t)))
}

func Ago ( timestamp * types . Timestamp ) string { t , _ := types . TimestampFromProto ( timestamp ) if t . Equal ( time . Time { } ) { return " " } return fmt . Sprintf ( " " , units . HumanDuration ( time . Since ( t ) ) ) }

// Ago pretty-prints the amount of time that has passed
// since timestamp as a human-readable string.

Ago pretty - prints the amount of time that has passed since timestamp as a human - readable string .


func TimeDifference(from *types.Timestamp, to *types.Timestamp) string {
	tFrom, _ := types.TimestampFromProto(from)
	tTo, _ := types.TimestampFromProto(to)
	return units.HumanDuration(tTo.Sub(tFrom))
}

func TimeDifference ( from * types . Timestamp , to * types . Timestamp ) string { tFrom , _ := types . TimestampFromProto ( from ) tTo , _ := types . TimestampFromProto ( to ) return units . HumanDuration ( tTo . Sub ( tFrom ) ) }

// TimeDifference pretty-prints the duration of time between from
// and to as a human-reabable string.

TimeDifference pretty - prints the duration of time between from and to as a human - reabable string .


func Duration(d *types.Duration) string {
	duration, _ := types.DurationFromProto(d)
	return units.HumanDuration(duration)
}

func Duration ( d * types . Duration ) string { duration , _ := types . DurationFromProto ( d ) return units . HumanDuration ( duration ) }

// Duration pretty prints a duration in a human readable way.

Duration pretty prints a duration in a human readable way .


func (c APIClient) InspectCluster() (*admin.ClusterInfo, error) {
	clusterInfo, err := c.AdminAPIClient.InspectCluster(c.Ctx(), &types.Empty{})
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return clusterInfo, nil
}

func ( c APIClient ) InspectCluster ( ) ( * admin . ClusterInfo , error ) { clusterInfo , err := c . AdminAPIClient . InspectCluster ( c . Ctx ( ) , & types . Empty { } ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return clusterInfo , nil }

// InspectCluster retrieves cluster state


func (c APIClient) Extract(objects bool, f func(op *admin.Op) error) error {
	extractClient, err := c.AdminAPIClient.Extract(c.Ctx(), &admin.ExtractRequest{NoObjects: !objects})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		op, err := extractClient.Recv()
		if err == io.EOF {
			break
		}
		if err != nil {
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(op); err != nil {
			return err
		}
	}
	return nil
}

func ( c APIClient ) Extract ( objects bool , f func ( op * admin . Op ) error ) error { extractClient , err := c . AdminAPIClient . Extract ( c . Ctx ( ) , & admin . ExtractRequest { NoObjects : ! objects } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { op , err := extractClient . Recv ( ) if err == io . EOF { break } if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := f ( op ) ; err != nil { return err } } return nil }

// Extract all cluster state, call f with each operation.

Extract all cluster state call f with each operation .


func (c APIClient) ExtractAll(objects bool) ([]*admin.Op, error) {
	var result []*admin.Op
	if err := c.Extract(objects, func(op *admin.Op) error {
		result = append(result, op)
		return nil
	}); err != nil {
		return nil, err
	}
	return result, nil
}

func ( c APIClient ) ExtractAll ( objects bool ) ( [ ] * admin . Op , error ) { var result [ ] * admin . Op if err := c . Extract ( objects , func ( op * admin . Op ) error { result = append ( result , op ) return nil } ) ; err != nil { return nil , err } return result , nil }

// ExtractAll cluster state as a slice of operations.

ExtractAll cluster state as a slice of operations .


func (c APIClient) ExtractWriter(objects bool, w io.Writer) error {
	writer := pbutil.NewWriter(w)
	return c.Extract(objects, func(op *admin.Op) error {
		_, err := writer.Write(op)
		return err
	})
}

func ( c APIClient ) ExtractWriter ( objects bool , w io . Writer ) error { writer := pbutil . NewWriter ( w ) return c . Extract ( objects , func ( op * admin . Op ) error { _ , err := writer . Write ( op ) return err } ) }

// ExtractWriter extracts all cluster state and marshals it to w.

ExtractWriter extracts all cluster state and marshals it to w .


func (c APIClient) ExtractURL(url string) error {
	extractClient, err := c.AdminAPIClient.Extract(c.Ctx(), &admin.ExtractRequest{URL: url})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	resp, err := extractClient.Recv()
	if err == nil {
		return fmt.Errorf("unexpected response from extract: %v", resp)
	}
	if err != io.EOF {
		return err
	}
	return nil
}

func ( c APIClient ) ExtractURL ( url string ) error { extractClient , err := c . AdminAPIClient . Extract ( c . Ctx ( ) , & admin . ExtractRequest { URL : url } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } resp , err := extractClient . Recv ( ) if err == nil { return fmt . Errorf ( " " , resp ) } if err != io . EOF { return err } return nil }

// ExtractURL extracts all cluster state and marshalls it to object storage.

ExtractURL extracts all cluster state and marshalls it to object storage .


func (c APIClient) ExtractPipeline(pipelineName string) (*pps.CreatePipelineRequest, error) {
	op, err := c.AdminAPIClient.ExtractPipeline(c.Ctx(), &admin.ExtractPipelineRequest{Pipeline: NewPipeline(pipelineName)})
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	if op.Op1_9 == nil || op.Op1_9.Pipeline == nil {
		return nil, fmt.Errorf("malformed response is missing pipeline")
	}
	return op.Op1_9.Pipeline, nil
}

func ( c APIClient ) ExtractPipeline ( pipelineName string ) ( * pps . CreatePipelineRequest , error ) { op , err := c . AdminAPIClient . ExtractPipeline ( c . Ctx ( ) , & admin . ExtractPipelineRequest { Pipeline : NewPipeline ( pipelineName ) } ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } if op . Op1_9 == nil || op . Op1_9 . Pipeline == nil { return nil , fmt . Errorf ( " " ) } return op . Op1_9 . Pipeline , nil }

// ExtractPipeline extracts a single pipeline.

ExtractPipeline extracts a single pipeline .


func (c APIClient) Restore(ops []*admin.Op) (retErr error) {
	restoreClient, err := c.AdminAPIClient.Restore(c.Ctx())
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	defer func() {
		if _, err := restoreClient.CloseAndRecv(); err != nil && retErr == nil {
			retErr = grpcutil.ScrubGRPC(err)
		}
	}()
	for _, op := range ops {
		if err := restoreClient.Send(&admin.RestoreRequest{Op: op}); err != nil {
			return grpcutil.ScrubGRPC(err)
		}
	}
	return nil
}

func ( c APIClient ) Restore ( ops [ ] * admin . Op ) ( retErr error ) { restoreClient , err := c . AdminAPIClient . Restore ( c . Ctx ( ) ) if err != nil { return grpcutil . ScrubGRPC ( err ) } defer func ( ) { if _ , err := restoreClient . CloseAndRecv ( ) ; err != nil && retErr == nil { retErr = grpcutil . ScrubGRPC ( err ) } } ( ) for _ , op := range ops { if err := restoreClient . Send ( & admin . RestoreRequest { Op : op } ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } } return nil }

// Restore cluster state from an extract series of operations.

Restore cluster state from an extract series of operations .


func (c APIClient) RestoreReader(r io.Reader) (retErr error) {
	restoreClient, err := c.AdminAPIClient.Restore(c.Ctx())
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	defer func() {
		if _, err := restoreClient.CloseAndRecv(); err != nil && retErr == nil {
			retErr = grpcutil.ScrubGRPC(err)
		}
	}()
	reader := pbutil.NewReader(r)
	op := &admin.Op{}
	for {
		if err := reader.Read(op); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		if err := restoreClient.Send(&admin.RestoreRequest{Op: op}); err != nil {
			return grpcutil.ScrubGRPC(err)
		}
	}
	return nil
}

func ( c APIClient ) RestoreReader ( r io . Reader ) ( retErr error ) { restoreClient , err := c . AdminAPIClient . Restore ( c . Ctx ( ) ) if err != nil { return grpcutil . ScrubGRPC ( err ) } defer func ( ) { if _ , err := restoreClient . CloseAndRecv ( ) ; err != nil && retErr == nil { retErr = grpcutil . ScrubGRPC ( err ) } } ( ) reader := pbutil . NewReader ( r ) op := & admin . Op { } for { if err := reader . Read ( op ) ; err != nil { if err == io . EOF { break } return err } if err := restoreClient . Send ( & admin . RestoreRequest { Op : op } ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } } return nil }

// RestoreReader restores cluster state from a reader containing marshaled ops.
// Such as those written by ExtractWriter.

RestoreReader restores cluster state from a reader containing marshaled ops . Such as those written by ExtractWriter .


func (c APIClient) RestoreFrom(objects bool, otherC *APIClient) (retErr error) {
	restoreClient, err := c.AdminAPIClient.Restore(c.Ctx())
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	defer func() {
		if _, err := restoreClient.CloseAndRecv(); err != nil && retErr == nil {
			retErr = grpcutil.ScrubGRPC(err)
		}
	}()
	return otherC.Extract(objects, func(op *admin.Op) error {
		return restoreClient.Send(&admin.RestoreRequest{Op: op})
	})
}

func ( c APIClient ) RestoreFrom ( objects bool , otherC * APIClient ) ( retErr error ) { restoreClient , err := c . AdminAPIClient . Restore ( c . Ctx ( ) ) if err != nil { return grpcutil . ScrubGRPC ( err ) } defer func ( ) { if _ , err := restoreClient . CloseAndRecv ( ) ; err != nil && retErr == nil { retErr = grpcutil . ScrubGRPC ( err ) } } ( ) return otherC . Extract ( objects , func ( op * admin . Op ) error { return restoreClient . Send ( & admin . RestoreRequest { Op : op } ) } ) }

// RestoreFrom restores state from another cluster which can be access through otherC.

RestoreFrom restores state from another cluster which can be access through otherC .