func (c APIClient) CreateRepo(repoName string) error {
	_, err := c.PfsAPIClient.CreateRepo(
		c.Ctx(),
		&pfs.CreateRepoRequest{
			Repo: NewRepo(repoName),
		},
	)
	return grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) CreateRepo ( repoName string ) error { _ , err := c . PfsAPIClient . CreateRepo ( c . Ctx ( ) , & pfs . CreateRepoRequest { Repo : NewRepo ( repoName ) , } , ) return grpcutil . ScrubGRPC ( err ) }

// CreateRepo creates a new Repo object in pfs with the given name. Repos are
// the top level data object in pfs and should be used to store data of a
// similar type. For example rather than having a single Repo for an entire
// project you might have separate Repos for logs, metrics, database dumps etc.

CreateRepo creates a new Repo object in pfs with the given name . Repos are the top level data object in pfs and should be used to store data of a similar type . For example rather than having a single Repo for an entire project you might have separate Repos for logs metrics database dumps etc .


func (c APIClient) InspectRepo(repoName string) (*pfs.RepoInfo, error) {
	resp, err := c.PfsAPIClient.InspectRepo(
		c.Ctx(),
		&pfs.InspectRepoRequest{
			Repo: NewRepo(repoName),
		},
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return resp, nil
}

func ( c APIClient ) InspectRepo ( repoName string ) ( * pfs . RepoInfo , error ) { resp , err := c . PfsAPIClient . InspectRepo ( c . Ctx ( ) , & pfs . InspectRepoRequest { Repo : NewRepo ( repoName ) , } , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return resp , nil }

// InspectRepo returns info about a specific Repo.

InspectRepo returns info about a specific Repo .


func (c APIClient) ListRepo() ([]*pfs.RepoInfo, error) {
	request := &pfs.ListRepoRequest{}
	repoInfos, err := c.PfsAPIClient.ListRepo(
		c.Ctx(),
		request,
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return repoInfos.RepoInfo, nil
}

func ( c APIClient ) ListRepo ( ) ( [ ] * pfs . RepoInfo , error ) { request := & pfs . ListRepoRequest { } repoInfos , err := c . PfsAPIClient . ListRepo ( c . Ctx ( ) , request , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return repoInfos . RepoInfo , nil }

// ListRepo returns info about all Repos.
// provenance specifies a set of provenance repos, only repos which have ALL of
// the specified repos as provenance will be returned unless provenance is nil
// in which case it is ignored.

ListRepo returns info about all Repos . provenance specifies a set of provenance repos only repos which have ALL of the specified repos as provenance will be returned unless provenance is nil in which case it is ignored .


func (c APIClient) DeleteRepo(repoName string, force bool) error {
	_, err := c.PfsAPIClient.DeleteRepo(
		c.Ctx(),
		&pfs.DeleteRepoRequest{
			Repo:  NewRepo(repoName),
			Force: force,
		},
	)
	return grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) DeleteRepo ( repoName string , force bool ) error { _ , err := c . PfsAPIClient . DeleteRepo ( c . Ctx ( ) , & pfs . DeleteRepoRequest { Repo : NewRepo ( repoName ) , Force : force , } , ) return grpcutil . ScrubGRPC ( err ) }

// DeleteRepo deletes a repo and reclaims the storage space it was using. Note
// that as of 1.0 we do not reclaim the blocks that the Repo was referencing,
// this is because they may also be referenced by other Repos and deleting them
// would make those Repos inaccessible. This will be resolved in later
// versions.
// If "force" is set to true, the repo will be removed regardless of errors.
// This argument should be used with care.

DeleteRepo deletes a repo and reclaims the storage space it was using . Note that as of 1 . 0 we do not reclaim the blocks that the Repo was referencing this is because they may also be referenced by other Repos and deleting them would make those Repos inaccessible . This will be resolved in later versions . If force is set to true the repo will be removed regardless of errors . This argument should be used with care .


func (c APIClient) BuildCommit(repoName string, branch string, parent string, treeObject string) (*pfs.Commit, error) {
	commit, err := c.PfsAPIClient.BuildCommit(
		c.Ctx(),
		&pfs.BuildCommitRequest{
			Parent: NewCommit(repoName, parent),
			Branch: branch,
			Tree:   &pfs.Object{Hash: treeObject},
		},
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return commit, nil
}

func ( c APIClient ) BuildCommit ( repoName string , branch string , parent string , treeObject string ) ( * pfs . Commit , error ) { commit , err := c . PfsAPIClient . BuildCommit ( c . Ctx ( ) , & pfs . BuildCommitRequest { Parent : NewCommit ( repoName , parent ) , Branch : branch , Tree : & pfs . Object { Hash : treeObject } , } , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return commit , nil }

// BuildCommit builds a commit in a single call from an existing HashTree that
// has already been written to the object store. Note this is a more advanced
// pattern for creating commits that's mostly used internally.

BuildCommit builds a commit in a single call from an existing HashTree that has already been written to the object store . Note this is a more advanced pattern for creating commits that s mostly used internally .


func (c APIClient) StartCommitParent(repoName string, branch string, parentCommit string) (*pfs.Commit, error) {
	commit, err := c.PfsAPIClient.StartCommit(
		c.Ctx(),
		&pfs.StartCommitRequest{
			Parent: &pfs.Commit{
				Repo: &pfs.Repo{
					Name: repoName,
				},
				ID: parentCommit,
			},
			Branch: branch,
		},
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return commit, nil
}

func ( c APIClient ) StartCommitParent ( repoName string , branch string , parentCommit string ) ( * pfs . Commit , error ) { commit , err := c . PfsAPIClient . StartCommit ( c . Ctx ( ) , & pfs . StartCommitRequest { Parent : & pfs . Commit { Repo : & pfs . Repo { Name : repoName , } , ID : parentCommit , } , Branch : branch , } , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return commit , nil }

// StartCommitParent begins the process of committing data to a Repo. Once started
// you can write to the Commit with PutFile and when all the data has been
// written you must finish the Commit with FinishCommit. NOTE, data is not
// persisted until FinishCommit is called.
// branch is a more convenient way to build linear chains of commits. When a
// commit is started with a non empty branch the value of branch becomes an
// alias for the created Commit. This enables a more intuitive access pattern.
// When the commit is started on a branch the previous head of the branch is
// used as the parent of the commit.
// parentCommit specifies the parent Commit, upon creation the new Commit will
// appear identical to the parent Commit, data can safely be added to the new
// commit without affecting the contents of the parent Commit. You may pass ""
// as parentCommit in which case the new Commit will have no parent and will
// initially appear empty.

StartCommitParent begins the process of committing data to a Repo . Once started you can write to the Commit with PutFile and when all the data has been written you must finish the Commit with FinishCommit . NOTE data is not persisted until FinishCommit is called . branch is a more convenient way to build linear chains of commits . When a commit is started with a non empty branch the value of branch becomes an alias for the created Commit . This enables a more intuitive access pattern . When the commit is started on a branch the previous head of the branch is used as the parent of the commit . parentCommit specifies the parent Commit upon creation the new Commit will appear identical to the parent Commit data can safely be added to the new commit without affecting the contents of the parent Commit . You may pass as parentCommit in which case the new Commit will have no parent and will initially appear empty .


func (c APIClient) FinishCommit(repoName string, commitID string) error {
	_, err := c.PfsAPIClient.FinishCommit(
		c.Ctx(),
		&pfs.FinishCommitRequest{
			Commit: NewCommit(repoName, commitID),
		},
	)
	return grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) FinishCommit ( repoName string , commitID string ) error { _ , err := c . PfsAPIClient . FinishCommit ( c . Ctx ( ) , & pfs . FinishCommitRequest { Commit : NewCommit ( repoName , commitID ) , } , ) return grpcutil . ScrubGRPC ( err ) }

// FinishCommit ends the process of committing data to a Repo and persists the
// Commit. Once a Commit is finished the data becomes immutable and future
// attempts to write to it with PutFile will error.

FinishCommit ends the process of committing data to a Repo and persists the Commit . Once a Commit is finished the data becomes immutable and future attempts to write to it with PutFile will error .


func (c APIClient) InspectCommit(repoName string, commitID string) (*pfs.CommitInfo, error) {
	return c.inspectCommit(repoName, commitID, pfs.CommitState_STARTED)
}

func ( c APIClient ) InspectCommit ( repoName string , commitID string ) ( * pfs . CommitInfo , error ) { return c . inspectCommit ( repoName , commitID , pfs . CommitState_STARTED ) }

// InspectCommit returns info about a specific Commit.

InspectCommit returns info about a specific Commit .


func (c APIClient) BlockCommit(repoName string, commitID string) (*pfs.CommitInfo, error) {
	return c.inspectCommit(repoName, commitID, pfs.CommitState_FINISHED)
}

func ( c APIClient ) BlockCommit ( repoName string , commitID string ) ( * pfs . CommitInfo , error ) { return c . inspectCommit ( repoName , commitID , pfs . CommitState_FINISHED ) }

// BlockCommit returns info about a specific Commit, but blocks until that
// commit has been finished.

BlockCommit returns info about a specific Commit but blocks until that commit has been finished .


func (c APIClient) ListCommit(repoName string, to string, from string, number uint64) ([]*pfs.CommitInfo, error) {
	var result []*pfs.CommitInfo
	if err := c.ListCommitF(repoName, to, from, number, func(ci *pfs.CommitInfo) error {
		result = append(result, ci)
		return nil
	}); err != nil {
		return nil, err
	}
	return result, nil
}

func ( c APIClient ) ListCommit ( repoName string , to string , from string , number uint64 ) ( [ ] * pfs . CommitInfo , error ) { var result [ ] * pfs . CommitInfo if err := c . ListCommitF ( repoName , to , from , number , func ( ci * pfs . CommitInfo ) error { result = append ( result , ci ) return nil } ) ; err != nil { return nil , err } return result , nil }

// ListCommit lists commits.
// If only `repo` is given, all commits in the repo are returned.
// If `to` is given, only the ancestors of `to`, including `to` itself,
// are considered.
// If `from` is given, only the descendents of `from`, including `from`
// itself, are considered.
// `number` determines how many commits are returned.  If `number` is 0,
// all commits that match the aforementioned criteria are returned.

ListCommit lists commits . If only repo is given all commits in the repo are returned . If to is given only the ancestors of to including to itself are considered . If from is given only the descendents of from including from itself are considered . number determines how many commits are returned . If number is 0 all commits that match the aforementioned criteria are returned .


func (c APIClient) ListCommitF(repoName string, to string, from string, number uint64, f func(*pfs.CommitInfo) error) error {
	req := &pfs.ListCommitRequest{
		Repo:   NewRepo(repoName),
		Number: number,
	}
	if from != "" {
		req.From = NewCommit(repoName, from)
	}
	if to != "" {
		req.To = NewCommit(repoName, to)
	}
	stream, err := c.PfsAPIClient.ListCommitStream(c.Ctx(), req)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		ci, err := stream.Recv()
		if err == io.EOF {
			break
		} else if err != nil {
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(ci); err != nil {
			if err == errutil.ErrBreak {
				return nil
			}
			return err
		}
	}
	return nil
}

func ( c APIClient ) ListCommitF ( repoName string , to string , from string , number uint64 , f func ( * pfs . CommitInfo ) error ) error { req := & pfs . ListCommitRequest { Repo : NewRepo ( repoName ) , Number : number , } if from != " " { req . From = NewCommit ( repoName , from ) } if to != " " { req . To = NewCommit ( repoName , to ) } stream , err := c . PfsAPIClient . ListCommitStream ( c . Ctx ( ) , req ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { ci , err := stream . Recv ( ) if err == io . EOF { break } else if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := f ( ci ) ; err != nil { if err == errutil . ErrBreak { return nil } return err } } return nil }

// ListCommitF lists commits, calling f with each commit.
// If only `repo` is given, all commits in the repo are returned.
// If `to` is given, only the ancestors of `to`, including `to` itself,
// are considered.
// If `from` is given, only the descendents of `from`, including `from`
// itself, are considered.
// `number` determines how many commits are returned.  If `number` is 0,
// all commits that match the aforementioned criteria are returned.

ListCommitF lists commits calling f with each commit . If only repo is given all commits in the repo are returned . If to is given only the ancestors of to including to itself are considered . If from is given only the descendents of from including from itself are considered . number determines how many commits are returned . If number is 0 all commits that match the aforementioned criteria are returned .


func (c APIClient) ListCommitByRepo(repoName string) ([]*pfs.CommitInfo, error) {
	return c.ListCommit(repoName, "", "", 0)
}

func ( c APIClient ) ListCommitByRepo ( repoName string ) ( [ ] * pfs . CommitInfo , error ) { return c . ListCommit ( repoName , " " , " " , 0 ) }

// ListCommitByRepo lists all commits in a repo.

ListCommitByRepo lists all commits in a repo .


func (c APIClient) CreateBranch(repoName string, branch string, commit string, provenance []*pfs.Branch) error {
	var head *pfs.Commit
	if commit != "" {
		head = NewCommit(repoName, commit)
	}
	_, err := c.PfsAPIClient.CreateBranch(
		c.Ctx(),
		&pfs.CreateBranchRequest{
			Branch:     NewBranch(repoName, branch),
			Head:       head,
			Provenance: provenance,
		},
	)
	return grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) CreateBranch ( repoName string , branch string , commit string , provenance [ ] * pfs . Branch ) error { var head * pfs . Commit if commit != " " { head = NewCommit ( repoName , commit ) } _ , err := c . PfsAPIClient . CreateBranch ( c . Ctx ( ) , & pfs . CreateBranchRequest { Branch : NewBranch ( repoName , branch ) , Head : head , Provenance : provenance , } , ) return grpcutil . ScrubGRPC ( err ) }


func (c APIClient) InspectBranch(repoName string, branch string) (*pfs.BranchInfo, error) {
	branchInfo, err := c.PfsAPIClient.InspectBranch(
		c.Ctx(),
		&pfs.InspectBranchRequest{
			Branch: NewBranch(repoName, branch),
		},
	)
	return branchInfo, grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) InspectBranch ( repoName string , branch string ) ( * pfs . BranchInfo , error ) { branchInfo , err := c . PfsAPIClient . InspectBranch ( c . Ctx ( ) , & pfs . InspectBranchRequest { Branch : NewBranch ( repoName , branch ) , } , ) return branchInfo , grpcutil . ScrubGRPC ( err ) }

// InspectBranch returns information on a specific PFS branch


func (c APIClient) ListBranch(repoName string) ([]*pfs.BranchInfo, error) {
	branchInfos, err := c.PfsAPIClient.ListBranch(
		c.Ctx(),
		&pfs.ListBranchRequest{
			Repo: NewRepo(repoName),
		},
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return branchInfos.BranchInfo, nil
}

func ( c APIClient ) ListBranch ( repoName string ) ( [ ] * pfs . BranchInfo , error ) { branchInfos , err := c . PfsAPIClient . ListBranch ( c . Ctx ( ) , & pfs . ListBranchRequest { Repo : NewRepo ( repoName ) , } , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return branchInfos . BranchInfo , nil }

// ListBranch lists the active branches on a Repo.

ListBranch lists the active branches on a Repo .


func (c APIClient) SetBranch(repoName string, commit string, branch string) error {
	return c.CreateBranch(repoName, branch, commit, nil)
}

func ( c APIClient ) SetBranch ( repoName string , commit string , branch string ) error { return c . CreateBranch ( repoName , branch , commit , nil ) }

// SetBranch sets a commit and its ancestors as a branch.
// SetBranch is deprecated in favor of CommitBranch.

SetBranch sets a commit and its ancestors as a branch . SetBranch is deprecated in favor of CommitBranch .


func (c APIClient) DeleteBranch(repoName string, branch string, force bool) error {
	_, err := c.PfsAPIClient.DeleteBranch(
		c.Ctx(),
		&pfs.DeleteBranchRequest{
			Branch: NewBranch(repoName, branch),
			Force:  force,
		},
	)
	return grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) DeleteBranch ( repoName string , branch string , force bool ) error { _ , err := c . PfsAPIClient . DeleteBranch ( c . Ctx ( ) , & pfs . DeleteBranchRequest { Branch : NewBranch ( repoName , branch ) , Force : force , } , ) return grpcutil . ScrubGRPC ( err ) }

// DeleteBranch deletes a branch, but leaves the commits themselves intact.
// In other words, those commits can still be accessed via commit IDs and
// other branches they happen to be on.

DeleteBranch deletes a branch but leaves the commits themselves intact . In other words those commits can still be accessed via commit IDs and other branches they happen to be on .


func (c APIClient) DeleteCommit(repoName string, commitID string) error {
	_, err := c.PfsAPIClient.DeleteCommit(
		c.Ctx(),
		&pfs.DeleteCommitRequest{
			Commit: NewCommit(repoName, commitID),
		},
	)
	return grpcutil.ScrubGRPC(err)
}

func ( c APIClient ) DeleteCommit ( repoName string , commitID string ) error { _ , err := c . PfsAPIClient . DeleteCommit ( c . Ctx ( ) , & pfs . DeleteCommitRequest { Commit : NewCommit ( repoName , commitID ) , } , ) return grpcutil . ScrubGRPC ( err ) }


func (c APIClient) FlushCommit(commits []*pfs.Commit, toRepos []*pfs.Repo) (CommitInfoIterator, error) {
	ctx, cancel := context.WithCancel(c.Ctx())
	stream, err := c.PfsAPIClient.FlushCommit(
		ctx,
		&pfs.FlushCommitRequest{
			Commits: commits,
			ToRepos: toRepos,
		},
	)
	if err != nil {
		cancel()
		return nil, grpcutil.ScrubGRPC(err)
	}
	return &commitInfoIterator{stream, cancel}, nil
}

func ( c APIClient ) FlushCommit ( commits [ ] * pfs . Commit , toRepos [ ] * pfs . Repo ) ( CommitInfoIterator , error ) { ctx , cancel := context . WithCancel ( c . Ctx ( ) ) stream , err := c . PfsAPIClient . FlushCommit ( ctx , & pfs . FlushCommitRequest { Commits : commits , ToRepos : toRepos , } , ) if err != nil { cancel ( ) return nil , grpcutil . ScrubGRPC ( err ) } return & commitInfoIterator { stream , cancel } , nil }

// FlushCommit returns an iterator that returns commits that have the
// specified `commits` as provenance.  Note that the iterator can block if
// jobs have not successfully completed. This in effect waits for all of the
// jobs that are triggered by a set of commits to complete.
//
// If toRepos is not nil then only the commits up to and including those
// repos will be considered, otherwise all repos are considered.
//
// Note that it's never necessary to call FlushCommit to run jobs, they'll
// run no matter what, FlushCommit just allows you to wait for them to
// complete and see their output once they do.

FlushCommit returns an iterator that returns commits that have the specified commits as provenance . Note that the iterator can block if jobs have not successfully completed . This in effect waits for all of the jobs that are triggered by a set of commits to complete . If toRepos is not nil then only the commits up to and including those repos will be considered otherwise all repos are considered . Note that it s never necessary to call FlushCommit to run jobs they ll run no matter what FlushCommit just allows you to wait for them to complete and see their output once they do .


func (c APIClient) FlushCommitF(commits []*pfs.Commit, toRepos []*pfs.Repo, f func(*pfs.CommitInfo) error) error {
	stream, err := c.PfsAPIClient.FlushCommit(
		c.Ctx(),
		&pfs.FlushCommitRequest{
			Commits: commits,
			ToRepos: toRepos,
		},
	)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		ci, err := stream.Recv()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(ci); err != nil {
			return err
		}
	}
}

func ( c APIClient ) FlushCommitF ( commits [ ] * pfs . Commit , toRepos [ ] * pfs . Repo , f func ( * pfs . CommitInfo ) error ) error { stream , err := c . PfsAPIClient . FlushCommit ( c . Ctx ( ) , & pfs . FlushCommitRequest { Commits : commits , ToRepos : toRepos , } , ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { ci , err := stream . Recv ( ) if err != nil { if err == io . EOF { return nil } return grpcutil . ScrubGRPC ( err ) } if err := f ( ci ) ; err != nil { return err } } }

// FlushCommitF calls f with commits that have the specified `commits` as
// provenance. Note that it can block if jobs have not successfully
// completed. This in effect waits for all of the jobs that are triggered by a
// set of commits to complete.
//
// If toRepos is not nil then only the commits up to and including those repos
// will be considered, otherwise all repos are considered.
//
// Note that it's never necessary to call FlushCommit to run jobs, they'll run
// no matter what, FlushCommitF just allows you to wait for them to complete and
// see their output once they do.

FlushCommitF calls f with commits that have the specified commits as provenance . Note that it can block if jobs have not successfully completed . This in effect waits for all of the jobs that are triggered by a set of commits to complete . If toRepos is not nil then only the commits up to and including those repos will be considered otherwise all repos are considered . Note that it s never necessary to call FlushCommit to run jobs they ll run no matter what FlushCommitF just allows you to wait for them to complete and see their output once they do .


func (c APIClient) FlushCommitAll(commits []*pfs.Commit, toRepos []*pfs.Repo) ([]*pfs.CommitInfo, error) {
	var result []*pfs.CommitInfo
	if err := c.FlushCommitF(commits, toRepos, func(ci *pfs.CommitInfo) error {
		result = append(result, ci)
		return nil
	}); err != nil {
		return nil, err
	}
	return result, nil
}

func ( c APIClient ) FlushCommitAll ( commits [ ] * pfs . Commit , toRepos [ ] * pfs . Repo ) ( [ ] * pfs . CommitInfo , error ) { var result [ ] * pfs . CommitInfo if err := c . FlushCommitF ( commits , toRepos , func ( ci * pfs . CommitInfo ) error { result = append ( result , ci ) return nil } ) ; err != nil { return nil , err } return result , nil }

// FlushCommitAll returns commits that have the specified `commits` as
// provenance. Note that it can block if jobs have not successfully
// completed. This in effect waits for all of the jobs that are triggered by a
// set of commits to complete.
//
// If toRepos is not nil then only the commits up to and including those repos
// will be considered, otherwise all repos are considered.
//
// Note that it's never necessary to call FlushCommit to run jobs, they'll run
// no matter what, FlushCommitAll just allows you to wait for them to complete and
// see their output once they do.

FlushCommitAll returns commits that have the specified commits as provenance . Note that it can block if jobs have not successfully completed . This in effect waits for all of the jobs that are triggered by a set of commits to complete . If toRepos is not nil then only the commits up to and including those repos will be considered otherwise all repos are considered . Note that it s never necessary to call FlushCommit to run jobs they ll run no matter what FlushCommitAll just allows you to wait for them to complete and see their output once they do .


func (c APIClient) SubscribeCommit(repo string, branch string, from string, state pfs.CommitState) (CommitInfoIterator, error) {
	ctx, cancel := context.WithCancel(c.Ctx())
	req := &pfs.SubscribeCommitRequest{
		Repo:   NewRepo(repo),
		Branch: branch,
		State:  state,
	}
	if from != "" {
		req.From = NewCommit(repo, from)
	}
	stream, err := c.PfsAPIClient.SubscribeCommit(ctx, req)
	if err != nil {
		cancel()
		return nil, grpcutil.ScrubGRPC(err)
	}
	return &commitInfoIterator{stream, cancel}, nil
}

func ( c APIClient ) SubscribeCommit ( repo string , branch string , from string , state pfs . CommitState ) ( CommitInfoIterator , error ) { ctx , cancel := context . WithCancel ( c . Ctx ( ) ) req := & pfs . SubscribeCommitRequest { Repo : NewRepo ( repo ) , Branch : branch , State : state , } if from != " " { req . From = NewCommit ( repo , from ) } stream , err := c . PfsAPIClient . SubscribeCommit ( ctx , req ) if err != nil { cancel ( ) return nil , grpcutil . ScrubGRPC ( err ) } return & commitInfoIterator { stream , cancel } , nil }

// SubscribeCommit is like ListCommit but it keeps listening for commits as
// they come in.

SubscribeCommit is like ListCommit but it keeps listening for commits as they come in .


func (c APIClient) SubscribeCommitF(repo, branch, from string, state pfs.CommitState, f func(*pfs.CommitInfo) error) error {
	req := &pfs.SubscribeCommitRequest{
		Repo:   NewRepo(repo),
		Branch: branch,
		State:  state,
	}
	if from != "" {
		req.From = NewCommit(repo, from)
	}
	stream, err := c.PfsAPIClient.SubscribeCommit(c.Ctx(), req)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		ci, err := stream.Recv()
		if err != nil {
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(ci); err != nil {
			return grpcutil.ScrubGRPC(err)
		}
	}
}

func ( c APIClient ) SubscribeCommitF ( repo , branch , from string , state pfs . CommitState , f func ( * pfs . CommitInfo ) error ) error { req := & pfs . SubscribeCommitRequest { Repo : NewRepo ( repo ) , Branch : branch , State : state , } if from != " " { req . From = NewCommit ( repo , from ) } stream , err := c . PfsAPIClient . SubscribeCommit ( c . Ctx ( ) , req ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { ci , err := stream . Recv ( ) if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := f ( ci ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } } }

// SubscribeCommitF is like ListCommit but it calls a callback function with
// the results rather than returning an iterator.

SubscribeCommitF is like ListCommit but it calls a callback function with the results rather than returning an iterator .


func (c APIClient) PutObjectAsync(tags []*pfs.Tag) (*PutObjectWriteCloserAsync, error) {
	w, err := c.newPutObjectWriteCloserAsync(tags)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return w, nil
}

func ( c APIClient ) PutObjectAsync ( tags [ ] * pfs . Tag ) ( * PutObjectWriteCloserAsync , error ) { w , err := c . newPutObjectWriteCloserAsync ( tags ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return w , nil }

// PutObjectAsync puts a value into the object store asynchronously.

PutObjectAsync puts a value into the object store asynchronously .


func (c APIClient) PutObject(_r io.Reader, tags ...string) (object *pfs.Object, _ int64, retErr error) {
	r := grpcutil.ReaderWrapper{_r}
	w, err := c.newPutObjectWriteCloser(tags...)
	if err != nil {
		return nil, 0, grpcutil.ScrubGRPC(err)
	}
	defer func() {
		if err := w.Close(); err != nil && retErr == nil {
			retErr = grpcutil.ScrubGRPC(err)
		}
		if retErr == nil {
			object = w.object
		}
	}()
	buf := grpcutil.GetBuffer()
	defer grpcutil.PutBuffer(buf)
	written, err := io.CopyBuffer(w, r, buf)
	if err != nil {
		return nil, 0, grpcutil.ScrubGRPC(err)
	}
	// return value set by deferred function
	return nil, written, nil
}

// PutObject puts a value into the object store and tags it with 0 or more tags.

PutObject puts a value into the object store and tags it with 0 or more tags .


func (c APIClient) PutObjectSplit(_r io.Reader) (objects []*pfs.Object, _ int64, retErr error) {
	r := grpcutil.ReaderWrapper{_r}
	w, err := c.newPutObjectSplitWriteCloser()
	if err != nil {
		return nil, 0, grpcutil.ScrubGRPC(err)
	}
	defer func() {
		if err := w.Close(); err != nil && retErr == nil {
			retErr = grpcutil.ScrubGRPC(err)
		}
		if retErr == nil {
			objects = w.objects
		}
	}()
	buf := grpcutil.GetBuffer()
	defer grpcutil.PutBuffer(buf)
	written, err := io.CopyBuffer(w, r, buf)
	if err != nil {
		return nil, 0, grpcutil.ScrubGRPC(err)
	}
	// return value set by deferred function
	return nil, written, nil
}

// PutObjectSplit is the same as PutObject except that the data is splitted
// into several smaller objects.  This is primarily useful if you'd like to
// be able to resume upload.

PutObjectSplit is the same as PutObject except that the data is splitted into several smaller objects . This is primarily useful if you d like to be able to resume upload .


func (c APIClient) GetObject(hash string, writer io.Writer) error {
	getObjectClient, err := c.ObjectAPIClient.GetObject(
		c.Ctx(),
		&pfs.Object{Hash: hash},
	)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	if err := grpcutil.WriteFromStreamingBytesClient(getObjectClient, writer); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

func ( c APIClient ) GetObject ( hash string , writer io . Writer ) error { getObjectClient , err := c . ObjectAPIClient . GetObject ( c . Ctx ( ) , & pfs . Object { Hash : hash } , ) if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := grpcutil . WriteFromStreamingBytesClient ( getObjectClient , writer ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } return nil }

// GetObject gets an object out of the object store by hash.

GetObject gets an object out of the object store by hash .


func (c APIClient) GetObjectReader(hash string) (io.ReadCloser, error) {
	ctx, cancel := context.WithCancel(c.Ctx())
	getObjectClient, err := c.ObjectAPIClient.GetObject(
		ctx,
		&pfs.Object{Hash: hash},
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return grpcutil.NewStreamingBytesReader(getObjectClient, cancel), nil
}

func ( c APIClient ) GetObjectReader ( hash string ) ( io . ReadCloser , error ) { ctx , cancel := context . WithCancel ( c . Ctx ( ) ) getObjectClient , err := c . ObjectAPIClient . GetObject ( ctx , & pfs . Object { Hash : hash } , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return grpcutil . NewStreamingBytesReader ( getObjectClient , cancel ) , nil }

// GetObjectReader returns a reader for an object in object store by hash.

GetObjectReader returns a reader for an object in object store by hash .


func (c APIClient) GetObjects(hashes []string, offset uint64, size uint64, totalSize uint64, writer io.Writer) error {
	var objects []*pfs.Object
	for _, hash := range hashes {
		objects = append(objects, &pfs.Object{Hash: hash})
	}
	getObjectsClient, err := c.ObjectAPIClient.GetObjects(
		c.Ctx(),
		&pfs.GetObjectsRequest{
			Objects:     objects,
			OffsetBytes: offset,
			SizeBytes:   size,
			TotalSize:   totalSize,
		},
	)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	if err := grpcutil.WriteFromStreamingBytesClient(getObjectsClient, writer); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

func ( c APIClient ) GetObjects ( hashes [ ] string , offset uint64 , size uint64 , totalSize uint64 , writer io . Writer ) error { var objects [ ] * pfs . Object for _ , hash := range hashes { objects = append ( objects , & pfs . Object { Hash : hash } ) } getObjectsClient , err := c . ObjectAPIClient . GetObjects ( c . Ctx ( ) , & pfs . GetObjectsRequest { Objects : objects , OffsetBytes : offset , SizeBytes : size , TotalSize : totalSize , } , ) if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := grpcutil . WriteFromStreamingBytesClient ( getObjectsClient , writer ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } return nil }

// GetObjects gets several objects out of the object store by hash.

GetObjects gets several objects out of the object store by hash .


func (c APIClient) TagObject(hash string, tags ...string) error {
	var _tags []*pfs.Tag
	for _, tag := range tags {
		_tags = append(_tags, &pfs.Tag{Name: tag})
	}
	if _, err := c.ObjectAPIClient.TagObject(
		c.Ctx(),
		&pfs.TagObjectRequest{
			Object: &pfs.Object{Hash: hash},
			Tags:   _tags,
		},
	); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

func ( c APIClient ) TagObject ( hash string , tags ... string ) error { var _tags [ ] * pfs . Tag for _ , tag := range tags { _tags = append ( _tags , & pfs . Tag { Name : tag } ) } if _ , err := c . ObjectAPIClient . TagObject ( c . Ctx ( ) , & pfs . TagObjectRequest { Object : & pfs . Object { Hash : hash } , Tags : _tags , } , ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } return nil }

// TagObject applies a tag to an existing object.

TagObject applies a tag to an existing object .


func (c APIClient) ListObject(f func(*pfs.Object) error) error {
	listObjectClient, err := c.ObjectAPIClient.ListObjects(c.Ctx(), &pfs.ListObjectsRequest{})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		object, err := listObjectClient.Recv()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(object); err != nil {
			return err
		}
	}
}

func ( c APIClient ) ListObject ( f func ( * pfs . Object ) error ) error { listObjectClient , err := c . ObjectAPIClient . ListObjects ( c . Ctx ( ) , & pfs . ListObjectsRequest { } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { object , err := listObjectClient . Recv ( ) if err != nil { if err == io . EOF { return nil } return grpcutil . ScrubGRPC ( err ) } if err := f ( object ) ; err != nil { return err } } }

// ListObject lists objects stored in pfs.

ListObject lists objects stored in pfs .


func (c APIClient) InspectObject(hash string) (*pfs.ObjectInfo, error) {
	value, err := c.ObjectAPIClient.InspectObject(
		c.Ctx(),
		&pfs.Object{Hash: hash},
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return value, nil
}

func ( c APIClient ) InspectObject ( hash string ) ( * pfs . ObjectInfo , error ) { value , err := c . ObjectAPIClient . InspectObject ( c . Ctx ( ) , & pfs . Object { Hash : hash } , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return value , nil }

// InspectObject returns info about an Object.

InspectObject returns info about an Object .


func (c APIClient) GetTag(tag string, writer io.Writer) error {
	getTagClient, err := c.ObjectAPIClient.GetTag(
		c.Ctx(),
		&pfs.Tag{Name: tag},
	)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	if err := grpcutil.WriteFromStreamingBytesClient(getTagClient, writer); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

func ( c APIClient ) GetTag ( tag string , writer io . Writer ) error { getTagClient , err := c . ObjectAPIClient . GetTag ( c . Ctx ( ) , & pfs . Tag { Name : tag } , ) if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := grpcutil . WriteFromStreamingBytesClient ( getTagClient , writer ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } return nil }

// GetTag gets an object out of the object store by tag.

GetTag gets an object out of the object store by tag .


func (c APIClient) GetTagReader(tag string) (io.ReadCloser, error) {
	ctx, cancel := context.WithCancel(c.Ctx())
	getTagClient, err := c.ObjectAPIClient.GetTag(
		ctx,
		&pfs.Tag{Name: tag},
	)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return grpcutil.NewStreamingBytesReader(getTagClient, cancel), nil
}

func ( c APIClient ) GetTagReader ( tag string ) ( io . ReadCloser , error ) { ctx , cancel := context . WithCancel ( c . Ctx ( ) ) getTagClient , err := c . ObjectAPIClient . GetTag ( ctx , & pfs . Tag { Name : tag } , ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return grpcutil . NewStreamingBytesReader ( getTagClient , cancel ) , nil }

// GetTagReader returns a reader for an object in object store by tag.

GetTagReader returns a reader for an object in object store by tag .


func (c APIClient) ListTag(f func(*pfs.ListTagsResponse) error) error {
	listTagClient, err := c.ObjectAPIClient.ListTags(c.Ctx(), &pfs.ListTagsRequest{IncludeObject: true})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		listTagResponse, err := listTagClient.Recv()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(listTagResponse); err != nil {
			return err
		}
	}
}

func ( c APIClient ) ListTag ( f func ( * pfs . ListTagsResponse ) error ) error { listTagClient , err := c . ObjectAPIClient . ListTags ( c . Ctx ( ) , & pfs . ListTagsRequest { IncludeObject : true } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { listTagResponse , err := listTagClient . Recv ( ) if err != nil { if err == io . EOF { return nil } return grpcutil . ScrubGRPC ( err ) } if err := f ( listTagResponse ) ; err != nil { return err } } }


func (c APIClient) Compact() error {
	_, err := c.ObjectAPIClient.Compact(
		c.Ctx(),
		&types.Empty{},
	)
	return err
}

func ( c APIClient ) Compact ( ) error { _ , err := c . ObjectAPIClient . Compact ( c . Ctx ( ) , & types . Empty { } , ) return err }

// Compact forces compaction of objects.


func (c APIClient) NewPutFileClient() (PutFileClient, error) {
	pfc, err := c.PfsAPIClient.PutFile(c.Ctx())
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return &putFileClient{c: pfc}, nil
}

func ( c APIClient ) NewPutFileClient ( ) ( PutFileClient , error ) { pfc , err := c . PfsAPIClient . PutFile ( c . Ctx ( ) ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return & putFileClient { c : pfc } , nil }

// NewPutFileClient returns a new client for putting files into pfs in a single request.

NewPutFileClient returns a new client for putting files into pfs in a single request .


func (c *putFileClient) PutFileOverwrite(repoName string, commitID string, path string, reader io.Reader, overwriteIndex int64) (_ int, retErr error) {
	writer, err := c.newPutFileWriteCloser(repoName, commitID, path, pfs.Delimiter_NONE, 0, 0, 0, &pfs.OverwriteIndex{Index: overwriteIndex})
	if err != nil {
		return 0, grpcutil.ScrubGRPC(err)
	}
	defer func() {
		if err := writer.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	written, err := io.Copy(writer, reader)
	return int(written), grpcutil.ScrubGRPC(err)
}

func ( c * putFileClient ) PutFileOverwrite ( repoName string , commitID string , path string , reader io . Reader , overwriteIndex int64 ) ( _ int , retErr error ) { writer , err := c . newPutFileWriteCloser ( repoName , commitID , path , pfs . Delimiter_NONE , 0 , 0 , 0 , & pfs . OverwriteIndex { Index : overwriteIndex } ) if err != nil { return 0 , grpcutil . ScrubGRPC ( err ) } defer func ( ) { if err := writer . Close ( ) ; err != nil && retErr == nil { retErr = err } } ( ) written , err := io . Copy ( writer , reader ) return int ( written ) , grpcutil . ScrubGRPC ( err ) }

// PutFileOverwrite is like PutFile but it overwrites the file rather than
// appending to it.  overwriteIndex allows you to specify the index of the
// object starting from which you'd like to overwrite.  If you want to
// overwrite the entire file, specify an index of 0.

PutFileOverwrite is like PutFile but it overwrites the file rather than appending to it . overwriteIndex allows you to specify the index of the object starting from which you d like to overwrite . If you want to overwrite the entire file specify an index of 0 .


func (c *putFileClient) Close() error {
	_, err := c.c.CloseAndRecv()
	return grpcutil.ScrubGRPC(err)
}

func ( c * putFileClient ) Close ( ) error { _ , err := c . c . CloseAndRecv ( ) return grpcutil . ScrubGRPC ( err ) }

// Close must be called after you're done using a putFileClient.
// Further requests will throw errors.

Close must be called after you re done using a putFileClient . Further requests will throw errors .


func (c APIClient) CopyFile(srcRepo, srcCommit, srcPath, dstRepo, dstCommit, dstPath string, overwrite bool) error {
	if _, err := c.PfsAPIClient.CopyFile(c.Ctx(),
		&pfs.CopyFileRequest{
			Src:       NewFile(srcRepo, srcCommit, srcPath),
			Dst:       NewFile(dstRepo, dstCommit, dstPath),
			Overwrite: overwrite,
		}); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

func ( c APIClient ) CopyFile ( srcRepo , srcCommit , srcPath , dstRepo , dstCommit , dstPath string , overwrite bool ) error { if _ , err := c . PfsAPIClient . CopyFile ( c . Ctx ( ) , & pfs . CopyFileRequest { Src : NewFile ( srcRepo , srcCommit , srcPath ) , Dst : NewFile ( dstRepo , dstCommit , dstPath ) , Overwrite : overwrite , } ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } return nil }

// CopyFile copys a file from one pfs location to another. It can be used on
// directories or regular files.

CopyFile copys a file from one pfs location to another . It can be used on directories or regular files .


func (c APIClient) GetFile(repoName string, commitID string, path string, offset int64, size int64, writer io.Writer) error {
	if c.limiter != nil {
		c.limiter.Acquire()
		defer c.limiter.Release()
	}
	apiGetFileClient, err := c.getFile(repoName, commitID, path, offset, size)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	if err := grpcutil.WriteFromStreamingBytesClient(apiGetFileClient, writer); err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return nil
}

func ( c APIClient ) GetFile ( repoName string , commitID string , path string , offset int64 , size int64 , writer io . Writer ) error { if c . limiter != nil { c . limiter . Acquire ( ) defer c . limiter . Release ( ) } apiGetFileClient , err := c . getFile ( repoName , commitID , path , offset , size ) if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := grpcutil . WriteFromStreamingBytesClient ( apiGetFileClient , writer ) ; err != nil { return grpcutil . ScrubGRPC ( err ) } return nil }

// GetFile returns the contents of a file at a specific Commit.
// offset specifies a number of bytes that should be skipped in the beginning of the file.
// size limits the total amount of data returned, note you will get fewer bytes
// than size if you pass a value larger than the size of the file.
// If size is set to 0 then all of the data will be returned.

GetFile returns the contents of a file at a specific Commit . offset specifies a number of bytes that should be skipped in the beginning of the file . size limits the total amount of data returned note you will get fewer bytes than size if you pass a value larger than the size of the file . If size is set to 0 then all of the data will be returned .


func (c APIClient) GetFileReader(repoName string, commitID string, path string, offset int64, size int64) (io.Reader, error) {
	apiGetFileClient, err := c.getFile(repoName, commitID, path, offset, size)
	if err != nil {
		return nil, grpcutil.ScrubGRPC(err)
	}
	return grpcutil.NewStreamingBytesReader(apiGetFileClient, nil), nil
}

func ( c APIClient ) GetFileReader ( repoName string , commitID string , path string , offset int64 , size int64 ) ( io . Reader , error ) { apiGetFileClient , err := c . getFile ( repoName , commitID , path , offset , size ) if err != nil { return nil , grpcutil . ScrubGRPC ( err ) } return grpcutil . NewStreamingBytesReader ( apiGetFileClient , nil ) , nil }

// GetFileReader returns a reader for the contents of a file at a specific Commit.
// offset specifies a number of bytes that should be skipped in the beginning of the file.
// size limits the total amount of data returned, note you will get fewer bytes
// than size if you pass a value larger than the size of the file.
// If size is set to 0 then all of the data will be returned.

GetFileReader returns a reader for the contents of a file at a specific Commit . offset specifies a number of bytes that should be skipped in the beginning of the file . size limits the total amount of data returned note you will get fewer bytes than size if you pass a value larger than the size of the file . If size is set to 0 then all of the data will be returned .


func (c APIClient) GetFileReadSeeker(repoName string, commitID string, path string) (io.ReadSeeker, error) {
	fileInfo, err := c.InspectFile(repoName, commitID, path)
	if err != nil {
		return nil, err
	}
	reader, err := c.GetFileReader(repoName, commitID, path, 0, 0)
	if err != nil {
		return nil, err
	}
	return &getFileReadSeeker{
		Reader: reader,
		file:   NewFile(repoName, commitID, path),
		offset: 0,
		size:   int64(fileInfo.SizeBytes),
		c:      c,
	}, nil
}

func ( c APIClient ) GetFileReadSeeker ( repoName string , commitID string , path string ) ( io . ReadSeeker , error ) { fileInfo , err := c . InspectFile ( repoName , commitID , path ) if err != nil { return nil , err } reader , err := c . GetFileReader ( repoName , commitID , path , 0 , 0 ) if err != nil { return nil , err } return & getFileReadSeeker { Reader : reader , file : NewFile ( repoName , commitID , path ) , offset : 0 , size : int64 ( fileInfo . SizeBytes ) , c : c , } , nil }

// GetFileReadSeeker returns a reader for the contents of a file at a specific
// Commit that permits Seeking to different points in the file.

GetFileReadSeeker returns a reader for the contents of a file at a specific Commit that permits Seeking to different points in the file .


func (c APIClient) InspectFile(repoName string, commitID string, path string) (*pfs.FileInfo, error) {
	return c.inspectFile(repoName, commitID, path)
}

func ( c APIClient ) InspectFile ( repoName string , commitID string , path string ) ( * pfs . FileInfo , error ) { return c . inspectFile ( repoName , commitID , path ) }

// InspectFile returns info about a specific file.

InspectFile returns info about a specific file .


func (c APIClient) ListFile(repoName string, commitID string, path string) ([]*pfs.FileInfo, error) {
	var result []*pfs.FileInfo
	if err := c.ListFileF(repoName, commitID, path, 0, func(fi *pfs.FileInfo) error {
		result = append(result, fi)
		return nil
	}); err != nil {
		return nil, err
	}
	return result, nil
}

func ( c APIClient ) ListFile ( repoName string , commitID string , path string ) ( [ ] * pfs . FileInfo , error ) { var result [ ] * pfs . FileInfo if err := c . ListFileF ( repoName , commitID , path , 0 , func ( fi * pfs . FileInfo ) error { result = append ( result , fi ) return nil } ) ; err != nil { return nil , err } return result , nil }

// ListFile returns info about all files in a Commit under path.

ListFile returns info about all files in a Commit under path .


func (c APIClient) ListFileHistory(repoName string, commitID string, path string, history int64) ([]*pfs.FileInfo, error) {
	var result []*pfs.FileInfo
	if err := c.ListFileF(repoName, commitID, path, history, func(fi *pfs.FileInfo) error {
		result = append(result, fi)
		return nil
	}); err != nil {
		return nil, err
	}
	return result, nil
}

func ( c APIClient ) ListFileHistory ( repoName string , commitID string , path string , history int64 ) ( [ ] * pfs . FileInfo , error ) { var result [ ] * pfs . FileInfo if err := c . ListFileF ( repoName , commitID , path , history , func ( fi * pfs . FileInfo ) error { result = append ( result , fi ) return nil } ) ; err != nil { return nil , err } return result , nil }

// ListFileHistory returns info about all files and their history in a Commit under path.

ListFileHistory returns info about all files and their history in a Commit under path .


func (c APIClient) ListFileF(repoName string, commitID string, path string, history int64, f func(fi *pfs.FileInfo) error) error {
	fs, err := c.PfsAPIClient.ListFileStream(
		c.Ctx(),
		&pfs.ListFileRequest{
			File:    NewFile(repoName, commitID, path),
			History: history,
		},
	)
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		fi, err := fs.Recv()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(fi); err != nil {
			if err == errutil.ErrBreak {
				return nil
			}
			return err
		}
	}
}

func ( c APIClient ) ListFileF ( repoName string , commitID string , path string , history int64 , f func ( fi * pfs . FileInfo ) error ) error { fs , err := c . PfsAPIClient . ListFileStream ( c . Ctx ( ) , & pfs . ListFileRequest { File : NewFile ( repoName , commitID , path ) , History : history , } , ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { fi , err := fs . Recv ( ) if err == io . EOF { return nil } else if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := f ( fi ) ; err != nil { if err == errutil . ErrBreak { return nil } return err } } }

// ListFileF returns info about all files in a Commit under path, calling f with each FileInfo.

ListFileF returns info about all files in a Commit under path calling f with each FileInfo .


func (c APIClient) Walk(repoName string, commitID string, path string, f WalkFn) error {
	fs, err := c.PfsAPIClient.WalkFile(
		c.Ctx(),
		&pfs.WalkFileRequest{File: NewFile(repoName, commitID, path)})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	for {
		fi, err := fs.Recv()
		if err == io.EOF {
			return nil
		} else if err != nil {
			return grpcutil.ScrubGRPC(err)
		}
		if err := f(fi); err != nil {
			if err == errutil.ErrBreak {
				return nil
			}
			return err
		}
	}
}

func ( c APIClient ) Walk ( repoName string , commitID string , path string , f WalkFn ) error { fs , err := c . PfsAPIClient . WalkFile ( c . Ctx ( ) , & pfs . WalkFileRequest { File : NewFile ( repoName , commitID , path ) } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } for { fi , err := fs . Recv ( ) if err == io . EOF { return nil } else if err != nil { return grpcutil . ScrubGRPC ( err ) } if err := f ( fi ) ; err != nil { if err == errutil . ErrBreak { return nil } return err } } }

// Walk walks the pfs filesystem rooted at path. walkFn will be called for each
// file found under path in lexicographical order. This includes both regular
// files and directories.

Walk walks the pfs filesystem rooted at path . walkFn will be called for each file found under path in lexicographical order . This includes both regular files and directories .


func (c APIClient) DeleteFile(repoName string, commitID string, path string) error {
	_, err := c.PfsAPIClient.DeleteFile(
		c.Ctx(),
		&pfs.DeleteFileRequest{
			File: NewFile(repoName, commitID, path),
		},
	)
	return err
}

func ( c APIClient ) DeleteFile ( repoName string , commitID string , path string ) error { _ , err := c . PfsAPIClient . DeleteFile ( c . Ctx ( ) , & pfs . DeleteFileRequest { File : NewFile ( repoName , commitID , path ) , } , ) return err }

// DeleteFile deletes a file from a Commit.
// DeleteFile leaves a tombstone in the Commit, assuming the file isn't written
// to later attempting to get the file from the finished commit will result in
// not found error.
// The file will of course remain intact in the Commit's parent.

DeleteFile deletes a file from a Commit . DeleteFile leaves a tombstone in the Commit assuming the file isn t written to later attempting to get the file from the finished commit will result in not found error . The file will of course remain intact in the Commit s parent .


func (w *PutObjectWriteCloserAsync) Write(p []byte) (int, error) {
	select {
	case err := <-w.errChan:
		if err != nil {
			return 0, grpcutil.ScrubGRPC(err)
		}
	default:
		for len(w.buf)+len(p) > cap(w.buf) {
			// Write the bytes that fit into w.buf, then
			// remove those bytes from p.
			i := cap(w.buf) - len(w.buf)
			w.buf = append(w.buf, p[:i]...)
			p = p[i:]
			w.writeChan <- w.buf
			w.buf = grpcutil.GetBuffer()[:0]
		}
		w.buf = append(w.buf, p...)
	}
	return len(p), nil
}


func (w *PutObjectWriteCloserAsync) Close() error {
	w.writeChan <- w.buf
	close(w.writeChan)
	err := <-w.errChan
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	w.object, err = w.client.CloseAndRecv()
	return grpcutil.ScrubGRPC(err)
}

func ( w * PutObjectWriteCloserAsync ) Close ( ) error { w . writeChan <- w . buf close ( w . writeChan ) err := <- w . errChan if err != nil { return grpcutil . ScrubGRPC ( err ) } w . object , err = w . client . CloseAndRecv ( ) return grpcutil . ScrubGRPC ( err ) }


func PrettyPrintVersionNoAdditional(version *pb.Version) string {
	return fmt.Sprintf("%d.%d.%d", version.Major, version.Minor, version.Micro)
}

func PrettyPrintVersionNoAdditional ( version * pb . Version ) string { return fmt . Sprintf ( " " , version . Major , version . Minor , version . Micro ) }

// PrettyPrintVersionNoAdditional returns a version string without
// version.Additional.

PrettyPrintVersionNoAdditional returns a version string without version . Additional .


func recursiveBlockQuoteExamples(parent *cobra.Command) {
	if parent.Example != "" {
		parent.Example = fmt.Sprintf("```sh\n%s\n```", parent.Example)
	}

	for _, cmd := range parent.Commands() {
		recursiveBlockQuoteExamples(cmd)
	}
}

func recursiveBlockQuoteExamples ( parent * cobra . Command ) { if parent . Example != " " { parent . Example = fmt . Sprintf ( " \n \n " , parent . Example ) } for _ , cmd := range parent . Commands ( ) { recursiveBlockQuoteExamples ( cmd ) } }

// Walk the command tree, wrap any examples in a block-quote with shell highlighting

Walk the command tree wrap any examples in a block - quote with shell highlighting


func errMissingField(field string) *logical.Response {
	return logical.ErrorResponse(fmt.Sprintf("missing required field '%s'", field))
}

func errMissingField ( field string ) * logical . Response { return logical . ErrorResponse ( fmt . Sprintf ( " " , field ) ) }

// errMissingField returns a logical response error that prints a consistent
// error message for when a required field is missing.

errMissingField returns a logical response error that prints a consistent error message for when a required field is missing .


func validateFields(req *logical.Request, data *framework.FieldData) error {
	var unknownFields []string
	for k := range req.Data {
		if _, ok := data.Schema[k]; !ok {
			unknownFields = append(unknownFields, k)
		}
	}

	if len(unknownFields) > 0 {
		return fmt.Errorf("unknown fields: %q", unknownFields)
	}

	return nil
}

func validateFields ( req * logical . Request , data * framework . FieldData ) error { var unknownFields [ ] string for k := range req . Data { if _ , ok := data . Schema [ k ] ; ! ok { unknownFields = append ( unknownFields , k ) } } if len ( unknownFields ) > 0 { return fmt . Errorf ( " " , unknownFields ) } return nil }

// validateFields verifies that no bad arguments were given to the request.

validateFields verifies that no bad arguments were given to the request .


func putConfig(ctx context.Context, s logical.Storage, cfg *config) error {
	entry, err := logical.StorageEntryJSON("config", cfg)
	if err != nil {
		return fmt.Errorf("%v: failed to generate storage entry", err)
	}
	if err := s.Put(ctx, entry); err != nil {
		return fmt.Errorf("%v: failed to write configuration to storage", err)
	}
	return nil
}

func putConfig ( ctx context . Context , s logical . Storage , cfg * config ) error { entry , err := logical . StorageEntryJSON ( " " , cfg ) if err != nil { return fmt . Errorf ( " " , err ) } if err := s . Put ( ctx , entry ) ; err != nil { return fmt . Errorf ( " " , err ) } return nil }

// putConfig parses and returns the configuration data from the storage backend.

putConfig parses and returns the configuration data from the storage backend .


func getConfig(ctx context.Context, s logical.Storage) (*config, error) {
	entry, err := s.Get(ctx, "config")
	if err != nil {
		return nil, fmt.Errorf("%v: failed to get config from storage", err)
	}
	if entry == nil || len(entry.Value) == 0 {
		return nil, errors.New("no configuration in storage")
	}

	var result config
	if err := entry.DecodeJSON(&result); err != nil {
		return nil, fmt.Errorf("%v: failed to decode configuration", err)
	}

	return &result, nil
}

func getConfig ( ctx context . Context , s logical . Storage ) ( * config , error ) { entry , err := s . Get ( ctx , " " ) if err != nil { return nil , fmt . Errorf ( " " , err ) } if entry == nil || len ( entry . Value ) == 0 { return nil , errors . New ( " " ) } var result config if err := entry . DecodeJSON ( & result ) ; err != nil { return nil , fmt . Errorf ( " " , err ) } return & result , nil }

// getConfig parses and returns the configuration data from the storage backend.

getConfig parses and returns the configuration data from the storage backend .


func Serve(
	servers ...ServerOptions,
) (retErr error) {
	for _, server := range servers {
		if server.RegisterFunc == nil {
			return ErrMustSpecifyRegisterFunc
		}
		if server.Port == 0 {
			return ErrMustSpecifyPort
		}
		opts := []grpc.ServerOption{
			grpc.MaxConcurrentStreams(math.MaxUint32),
			grpc.MaxRecvMsgSize(server.MaxMsgSize),
			grpc.MaxSendMsgSize(server.MaxMsgSize),
			grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
				MinTime:             5 * time.Second,
				PermitWithoutStream: true,
			}),
			grpc.UnaryInterceptor(tracing.UnaryServerInterceptor()),
			grpc.StreamInterceptor(tracing.StreamServerInterceptor()),
		}
		if server.PublicPortTLSAllowed {
			// Validate environment
			certPath := path.Join(TLSVolumePath, TLSCertFile)
			keyPath := path.Join(TLSVolumePath, TLSKeyFile)
			_, certPathStatErr := os.Stat(certPath)
			_, keyPathStatErr := os.Stat(keyPath)
			if certPathStatErr != nil {
				log.Warnf("TLS disabled: could not stat public cert at %s: %v", certPath, certPathStatErr)
			}
			if keyPathStatErr != nil {
				log.Warnf("TLS disabled: could not stat private key at %s: %v", keyPath, keyPathStatErr)
			}
			if certPathStatErr == nil && keyPathStatErr == nil {
				// Read TLS cert and key
				transportCreds, err := credentials.NewServerTLSFromFile(certPath, keyPath)
				if err != nil {
					return fmt.Errorf("couldn't build transport creds: %v", err)
				}
				opts = append(opts, grpc.Creds(transportCreds))
			}
		}

		grpcServer := grpc.NewServer(opts...)
		if err := server.RegisterFunc(grpcServer); err != nil {
			return err
		}
		listener, err := net.Listen("tcp", fmt.Sprintf(":%d", server.Port))
		if err != nil {
			return err
		}
		if server.Cancel != nil {
			go func() {
				<-server.Cancel
				if err := listener.Close(); err != nil {
					fmt.Printf("listener.Close(): %v\n", err)
				}
			}()
		}
		if err := grpcServer.Serve(listener); err != nil {
			return err
		}
	}
	return nil
}

func
Serve
(
servers
...
ServerOptions
,
)
(
retErr
error
)
{
for
_
,
server
:=
range
servers
{
if
server
.
RegisterFunc
==
{
return
ErrMustSpecifyRegisterFunc
}
if
server
.
Port
==
0
{
return
ErrMustSpecifyPort
}
opts
:=
[
]
grpc
.
ServerOption
{
grpc
.
MaxConcurrentStreams
(
math
.
MaxUint32
)
,
grpc
.
MaxRecvMsgSize
(
server
.
MaxMsgSize
)
,
grpc
.
MaxSendMsgSize
(
server
.
MaxMsgSize
)
,
grpc
.
KeepaliveEnforcementPolicy
(
keepalive
.
EnforcementPolicy
{
MinTime
:
5
*
time
.
Second
,
PermitWithoutStream
:
true
,
}
)
,
grpc
.
UnaryInterceptor
(
tracing
.
UnaryServerInterceptor
(
)
)
,
grpc
.
StreamInterceptor
(
tracing
.
StreamServerInterceptor
(
)
)
,
}
if
server
.
PublicPortTLSAllowed
{
// Validate environment
certPath
:=
path
.
Join
(
TLSVolumePath
,
TLSCertFile
)
keyPath
:=
path
.
Join
(
TLSVolumePath
,
TLSKeyFile
)
_
,
certPathStatErr
:=
os
.
Stat
(
certPath
)
_
,
keyPathStatErr
:=
os
.
Stat
(
keyPath
)
if
certPathStatErr
!=
{
log
.
Warnf
(
"
"
,
certPath
,
certPathStatErr
)
}
if
keyPathStatErr
!=
{
log
.
Warnf
(
"
"
,
keyPath
,
keyPathStatErr
)
}
if
certPathStatErr
==
&&
keyPathStatErr
==
{
// Read TLS cert and key
transportCreds
,
err
:=
credentials
.
NewServerTLSFromFile
(
certPath
,
keyPath
)
if
err
!=
{
return
fmt
.
Errorf
(
"
"
,
err
)
}
opts
=
append
(
opts
,
grpc
.
Creds
(
transportCreds
)
)
}
}
grpcServer
:=
grpc
.
NewServer
(
opts
...
)
if
err
:=
server
.
RegisterFunc
(
grpcServer
)
;
err
!=
{
return
err
}
listener
,
err
:=
net
.
Listen
(
"
"
,
fmt
.
Sprintf
(
"
"
,
server
.
Port
)
)
if
err
!=
{
return
err
}
if
server
.
Cancel
!=
{
go
func
(
)
{
<-
server
.
Cancel
if
err
:=
listener
.
Close
(
)
;
err
!=
{
fmt
.
Printf
(
"
\n
"
,
err
)
}
}
(
)
}
if
err
:=
grpcServer
.
Serve
(
listener
)
;
err
!=
{
return
err
}
}
return
}


func NewPuller() *Puller {
	return &Puller{
		errCh: make(chan error, 1),
		pipes: make(map[string]bool),
	}
}

func NewPuller ( ) * Puller { return & Puller { errCh : make ( chan error , 1 ) , pipes : make ( map [ string ] bool ) , } }

// NewPuller creates a new Puller struct.


func (p *Puller) PullTree(client *pachclient.APIClient, root string, tree hashtree.HashTree, pipes bool, concurrency int) error {
	limiter := limit.New(concurrency)
	var eg errgroup.Group
	if err := tree.Walk("/", func(path string, node *hashtree.NodeProto) error {
		if node.FileNode != nil {
			path := filepath.Join(root, path)
			var hashes []string
			for _, object := range node.FileNode.Objects {
				hashes = append(hashes, object.Hash)
			}
			if pipes {
				return p.makePipe(path, func(w io.Writer) error {
					return client.GetObjects(hashes, 0, 0, uint64(node.SubtreeSize), w)
				})
			}
			limiter.Acquire()
			eg.Go(func() (retErr error) {
				defer limiter.Release()
				return p.makeFile(path, func(w io.Writer) error {
					return client.GetObjects(hashes, 0, 0, uint64(node.SubtreeSize), w)
				})
			})
		}
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}

func ( p * Puller ) PullTree ( client * pachclient . APIClient , root string , tree hashtree . HashTree , pipes bool , concurrency int ) error { limiter := limit . New ( concurrency ) var eg errgroup . Group if err := tree . Walk ( " " , func ( path string , node * hashtree . NodeProto ) error { if node . FileNode != nil { path := filepath . Join ( root , path ) var hashes [ ] string for _ , object := range node . FileNode . Objects { hashes = append ( hashes , object . Hash ) } if pipes { return p . makePipe ( path , func ( w io . Writer ) error { return client . GetObjects ( hashes , 0 , 0 , uint64 ( node . SubtreeSize ) , w ) } ) } limiter . Acquire ( ) eg . Go ( func ( ) ( retErr error ) { defer limiter . Release ( ) return p . makeFile ( path , func ( w io . Writer ) error { return client . GetObjects ( hashes , 0 , 0 , uint64 ( node . SubtreeSize ) , w ) } ) } ) } return nil } ) ; err != nil { return err } return eg . Wait ( ) }

// PullTree pulls from a raw HashTree rather than a repo.

PullTree pulls from a raw HashTree rather than a repo .


func Push(client *pachclient.APIClient, root string, commit *pfs.Commit, overwrite bool) error {
	var g errgroup.Group
	if err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		g.Go(func() (retErr error) {
			if path == root || info.IsDir() {
				return nil
			}

			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()

			relPath, err := filepath.Rel(root, path)
			if err != nil {
				return err
			}

			if overwrite {
				if err := client.DeleteFile(commit.Repo.Name, commit.ID, relPath); err != nil {
					return err
				}
			}

			_, err = client.PutFile(commit.Repo.Name, commit.ID, relPath, f)
			return err
		})
		return nil
	}); err != nil {
		return err
	}

	return g.Wait()
}

func Push ( client * pachclient . APIClient , root string , commit * pfs . Commit , overwrite bool ) error { var g errgroup . Group if err := filepath . Walk ( root , func ( path string , info os . FileInfo , err error ) error { g . Go ( func ( ) ( retErr error ) { if path == root || info . IsDir ( ) { return nil } f , err := os . Open ( path ) if err != nil { return err } defer func ( ) { if err := f . Close ( ) ; err != nil && retErr == nil { retErr = err } } ( ) relPath , err := filepath . Rel ( root , path ) if err != nil { return err } if overwrite { if err := client . DeleteFile ( commit . Repo . Name , commit . ID , relPath ) ; err != nil { return err } } _ , err = client . PutFile ( commit . Repo . Name , commit . ID , relPath , f ) return err } ) return nil } ) ; err != nil { return err } return g . Wait ( ) }

// Push puts files under root into an open commit.

Push puts files under root into an open commit .


func PushObj(pachClient *pachclient.APIClient, commit *pfs.Commit, objClient obj.Client, root string) error {
	var eg errgroup.Group
	sem := make(chan struct{}, 200)
	if err := pachClient.Walk(commit.Repo.Name, commit.ID, "", func(fileInfo *pfs.FileInfo) error {
		if fileInfo.FileType != pfs.FileType_FILE {
			return nil
		}
		eg.Go(func() (retErr error) {
			sem <- struct{}{}
			defer func() { <-sem }()
			w, err := objClient.Writer(pachClient.Ctx(), filepath.Join(root, fileInfo.File.Path))
			if err != nil {
				return err
			}
			defer func() {
				if err := w.Close(); err != nil && retErr == nil {
					retErr = err
				}
			}()
			return pachClient.GetFile(commit.Repo.Name, commit.ID, fileInfo.File.Path, 0, 0, w)
		})
		return nil
	}); err != nil {
		return err
	}
	return eg.Wait()
}

func PushObj ( pachClient * pachclient . APIClient , commit * pfs . Commit , objClient obj . Client , root string ) error { var eg errgroup . Group sem := make ( chan struct { } , 200 ) if err := pachClient . Walk ( commit . Repo . Name , commit . ID , " " , func ( fileInfo * pfs . FileInfo ) error { if fileInfo . FileType != pfs . FileType_FILE { return nil } eg . Go ( func ( ) ( retErr error ) { sem <- struct { } { } defer func ( ) { <- sem } ( ) w , err := objClient . Writer ( pachClient . Ctx ( ) , filepath . Join ( root , fileInfo . File . Path ) ) if err != nil { return err } defer func ( ) { if err := w . Close ( ) ; err != nil && retErr == nil { retErr = err } } ( ) return pachClient . GetFile ( commit . Repo . Name , commit . ID , fileInfo . File . Path , 0 , 0 , w ) } ) return nil } ) ; err != nil { return err } return eg . Wait ( ) }

// PushObj pushes data from commit to an object store.

PushObj pushes data from commit to an object store .


func PushFile(c *pachclient.APIClient, pfc pachclient.PutFileClient, pfsFile *pfs.File, osFile io.ReadSeeker) error {
	fileInfo, err := c.InspectFile(pfsFile.Commit.Repo.Name, pfsFile.Commit.ID, pfsFile.Path)
	if err != nil && !isNotExist(err) {
		return err
	}

	var i int
	var object *pfs.Object
	if fileInfo != nil {
		for i, object = range fileInfo.Objects {
			hash := pfs.NewHash()
			if _, err := io.CopyN(hash, osFile, pfs.ChunkSize); err != nil {
				if err == io.EOF {
					break
				}
				return err
			}

			if object.Hash != pfs.EncodeHash(hash.Sum(nil)) {
				break
			}
		}
	}

	if _, err := osFile.Seek(int64(i)*pfs.ChunkSize, 0); err != nil {
		return err
	}

	_, err = pfc.PutFileOverwrite(pfsFile.Commit.Repo.Name, pfsFile.Commit.ID, pfsFile.Path, osFile, int64(i))
	return err
}

func PushFile ( c * pachclient . APIClient , pfc pachclient . PutFileClient , pfsFile * pfs . File , osFile io . ReadSeeker ) error { fileInfo , err := c . InspectFile ( pfsFile . Commit . Repo . Name , pfsFile . Commit . ID , pfsFile . Path ) if err != nil && ! isNotExist ( err ) { return err } var i int var object * pfs . Object if fileInfo != nil { for i , object = range fileInfo . Objects { hash := pfs . NewHash ( ) if _ , err := io . CopyN ( hash , osFile , pfs . ChunkSize ) ; err != nil { if err == io . EOF { break } return err } if object . Hash != pfs . EncodeHash ( hash . Sum ( nil ) ) { break } } } if _ , err := osFile . Seek ( int64 ( i ) * pfs . ChunkSize , 0 ) ; err != nil { return err } _ , err = pfc . PutFileOverwrite ( pfsFile . Commit . Repo . Name , pfsFile . Commit . ID , pfsFile . Path , osFile , int64 ( i ) ) return err }

// PushFile makes sure that pfsFile has the same content as osFile.

PushFile makes sure that pfsFile has the same content as osFile .


func (c APIClient) Dump(w io.Writer) error {
	goroClient, err := c.DebugClient.Dump(c.Ctx(), &debug.DumpRequest{})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return grpcutil.ScrubGRPC(grpcutil.WriteFromStreamingBytesClient(goroClient, w))
}

func ( c APIClient ) Dump ( w io . Writer ) error { goroClient , err := c . DebugClient . Dump ( c . Ctx ( ) , & debug . DumpRequest { } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } return grpcutil . ScrubGRPC ( grpcutil . WriteFromStreamingBytesClient ( goroClient , w ) ) }

// Dump writes debug information from the server to w.

Dump writes debug information from the server to w .


func (c APIClient) Profile(profile string, duration time.Duration, w io.Writer) error {
	var d *types.Duration
	if duration != 0 {
		d = types.DurationProto(duration)
	}
	profileClient, err := c.DebugClient.Profile(c.Ctx(), &debug.ProfileRequest{
		Profile:  profile,
		Duration: d,
	})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return grpcutil.ScrubGRPC(grpcutil.WriteFromStreamingBytesClient(profileClient, w))
}

func ( c APIClient ) Profile ( profile string , duration time . Duration , w io . Writer ) error { var d * types . Duration if duration != 0 { d = types . DurationProto ( duration ) } profileClient , err := c . DebugClient . Profile ( c . Ctx ( ) , & debug . ProfileRequest { Profile : profile , Duration : d , } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } return grpcutil . ScrubGRPC ( grpcutil . WriteFromStreamingBytesClient ( profileClient , w ) ) }

// Profile writes a pprof profile for pachd to w.

Profile writes a pprof profile for pachd to w .


func (c APIClient) Binary(w io.Writer) error {
	binaryClient, err := c.DebugClient.Binary(c.Ctx(), &debug.BinaryRequest{})
	if err != nil {
		return grpcutil.ScrubGRPC(err)
	}
	return grpcutil.ScrubGRPC(grpcutil.WriteFromStreamingBytesClient(binaryClient, w))
}

func ( c APIClient ) Binary ( w io . Writer ) error { binaryClient , err := c . DebugClient . Binary ( c . Ctx ( ) , & debug . BinaryRequest { } ) if err != nil { return grpcutil . ScrubGRPC ( err ) } return grpcutil . ScrubGRPC ( grpcutil . WriteFromStreamingBytesClient ( binaryClient , w ) ) }

// Binary writes the running pachd binary to w.

Binary writes the running pachd binary to w .


func RegisterCacheStats(cacheName string, groupCacheStats *groupcache.Stats) {
	c := &cacheStats{
		cacheName:    cacheName,
		descriptions: make(map[string]*prometheus.Desc),
		stats:        groupCacheStats,
	}
	if err := prometheus.Register(c); err != nil {
		// metrics may be redundantly registered; ignore these errors
		if _, ok := err.(prometheus.AlreadyRegisteredError); !ok {
			logrus.Infof("error registering prometheus metric: %v", err)
		}
	}
}

// RegisterCacheStats creates a new wrapper for groupcache stats that implements
// the prometheus.Collector interface, and registers it

RegisterCacheStats creates a new wrapper for groupcache stats that implements the prometheus . Collector interface and registers it


func (c *counter) wait(n int64) {
	c.mu.Lock()
	defer c.mu.Unlock()
	for c.n < n {
		c.cond.Wait()
	}
}

func ( c * counter ) wait ( n int64 ) { c . mu . Lock ( ) defer c . mu . Unlock ( ) for c . n < n { c . cond . Wait ( ) } }

// wait until more than n bytes have been written


func RunWorkload(
	client *client.APIClient,
	rand *rand.Rand,
	size int,
) error {
	worker := newWorker(rand)
	for i := 0; i < size; i++ {
		if err := worker.work(client); err != nil {
			return err
		}
	}
	for _, job := range worker.startedJobs {
		jobInfo, err := client.InspectJob(job.ID, true)
		if err != nil {
			return err
		}
		if jobInfo.State != pps.JobState_JOB_SUCCESS {
			return fmt.Errorf("job %s failed", job.ID)
		}
	}
	return nil
}

func RunWorkload ( client * client . APIClient , rand * rand . Rand , size int , ) error { worker := newWorker ( rand ) for i := 0 ; i < size ; i ++ { if err := worker . work ( client ) ; err != nil { return err } } for _ , job := range worker . startedJobs { jobInfo , err := client . InspectJob ( job . ID , true ) if err != nil { return err } if jobInfo . State != pps . JobState_JOB_SUCCESS { return fmt . Errorf ( " " , job . ID ) } } return nil }

// RunWorkload runs a test workload against a Pachyderm cluster.

RunWorkload runs a test workload against a Pachyderm cluster .


func (w *worker) createRepo(c *client.APIClient) error {
	repoName := w.randString(10)
	if err := c.CreateRepo(repoName); err != nil {
		return err
	}
	w.repos = append(w.repos, &pfs.Repo{Name: repoName})

	// Start the first commit in the repo (no parent). This is critical to
	// advanceCommit(), which will try to finish a commit the first time it's
	// called, and therefore must have an open commit to finish.
	commit, err := c.StartCommit(repoName, "")
	if err != nil {
		return err
	}
	w.started = append(w.started, commit)
	return nil
}

// createRepo creates a new repo in the cluster


func (w *worker) advanceCommit(c *client.APIClient) error {
	if len(w.started) >= maxStartedCommits || len(w.finished) == 0 {
		// Randomly select a commit that has been started and finish it
		if len(w.started) == 0 {
			return nil
		}
		i := w.rand.Intn(len(w.started))
		commit := w.started[i]

		// Before we finish a commit, we add a file. This assures that there
		// won't be any empty commits which will later crash jobs
		if _, err := c.PutFile(commit.Repo.Name, commit.ID, w.randString(10), w.reader()); err != nil {
			return err
		}
		if err := c.FinishCommit(commit.Repo.Name, commit.ID); err != nil {
			return err
		}
		// remove commit[i] from 'started' and add it to 'finished'
		w.started = append(w.started[:i], w.started[i+1:]...)
		w.finished = append(w.finished, commit)
	} else {
		// Start a new commmit (parented off of a commit that we've finished)
		commit := w.finished[w.rand.Intn(len(w.finished))]
		commit, err := c.StartCommitParent(commit.Repo.Name, "", commit.ID)
		if err != nil {
			return err
		}
		w.started = append(w.started, commit)
	}
	return nil
}

// advanceCommit either starts or finishes a commit, depending on the state of
// the cluster.

advanceCommit either starts or finishes a commit depending on the state of the cluster .


func RandString(r *rand.Rand, n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[r.Intn(len(letters))]
	}
	return string(b)
}

func RandString ( r * rand . Rand , n int ) string { b := make ( [ ] byte , n ) for i := range b { b [ i ] = letters [ r . Intn ( len ( letters ) ) ] } return string ( b ) }

// RandString returns a random alphabetical string of size n


func NewReader(rand *rand.Rand, bytes int64) io.Reader {
	return &reader{
		rand:  rand,
		bytes: bytes,
	}
}

func NewReader ( rand * rand . Rand , bytes int64 ) io . Reader { return & reader { rand : rand , bytes : bytes , } }

// NewReader returns a Reader which generates strings of characters.

NewReader returns a Reader which generates strings of characters .


func iterDir(tx *bolt.Tx, path string, f func(k, v []byte, c *bolt.Cursor) error) error {
	node, err := get(tx, path)
	if err != nil {
		return err
	}
	if node.DirNode == nil {
		return errorf(PathConflict, "the file at \"%s\" is not a directory",
			path)
	}
	c := NewChildCursor(tx, path)
	for k, v := c.K(), c.V(); k != nil; k, v = c.Next() {
		if err := f(k, v, c.c); err != nil {
			if err == errutil.ErrBreak {
				return nil
			}
			return err
		}
	}
	return nil
}

func iterDir ( tx * bolt . Tx , path string , f func ( k , v [ ] byte , c * bolt . Cursor ) error ) error { node , err := get ( tx , path ) if err != nil { return err } if node . DirNode == nil { return errorf ( PathConflict , " \" \" " , path ) } c := NewChildCursor ( tx , path ) for k , v := c . K ( ) , c . V ( ) ; k != nil ; k , v = c . Next ( ) { if err := f ( k , v , c . c ) ; err != nil { if err == errutil . ErrBreak { return nil } return err } } return nil }

// iterDir iterates through the nodes under path, it errors with PathNotFound if path doesn't exist, it errors with PathConflict if path exists but isn't a directory.

iterDir iterates through the nodes under path it errors with PathNotFound if path doesn t exist it errors with PathConflict if path exists but isn t a directory .


func (h *dbHashTree) FSSize() int64 {
	rootNode, err := h.Get("/")
	if err != nil {
		return 0
	}
	return rootNode.SubtreeSize
}

func ( h * dbHashTree ) FSSize ( ) int64 { rootNode , err := h . Get ( " " ) if err != nil { return 0 } return rootNode . SubtreeSize }


func (h *dbHashTree) Diff(oldHashTree HashTree, newPath string, oldPath string, recursiveDepth int64, f func(path string, node *NodeProto, new bool) error) (retErr error) {
	// Setup a txn for each hashtree, this is a bit complicated because we don't want to make 2 read tx to the same tree, if we did then should someone start a write tx inbetween them we would have a deadlock
	old := oldHashTree.(*dbHashTree)
	if old == nil {
		return fmt.Errorf("unrecognized HashTree type")
	}
	rollback := func(tx *bolt.Tx) {
		if err := tx.Rollback(); err != nil && retErr == nil {
			retErr = err
		}
	}
	var newTx *bolt.Tx
	var oldTx *bolt.Tx
	if h == oldHashTree {
		tx, err := h.Begin(false)
		if err != nil {
			return err
		}
		newTx = tx
		oldTx = tx
		defer rollback(tx)
	} else {
		var err error
		newTx, err = h.Begin(false)
		if err != nil {
			return err
		}
		defer rollback(newTx)
		oldTx, err = old.Begin(false)
		if err != nil {
			return err
		}
		defer rollback(oldTx)
	}
	return diff(newTx, oldTx, newPath, oldPath, recursiveDepth, f)
}

// Diff returns the diff of two hashtrees at particular paths.

Diff returns the diff of two hashtrees at particular paths .


func (h *dbHashTree) Serialize(_w io.Writer) error {
	w := pbutil.NewWriter(_w)
	return h.View(func(tx *bolt.Tx) error {
		for _, bucket := range buckets {
			b := tx.Bucket(b(bucket))
			if _, err := w.Write(
				&BucketHeader{
					Bucket: bucket,
				}); err != nil {
				return err
			}
			if err := b.ForEach(func(k, v []byte) error {
				if _, err := w.WriteBytes(k); err != nil {
					return err
				}
				_, err := w.WriteBytes(v)
				return err
			}); err != nil {
				return err
			}
			if _, err := w.WriteBytes(SentinelByte); err != nil {
				return err
			}
		}
		return nil
	})
}

func ( h * dbHashTree ) Serialize ( _w io . Writer ) error { w := pbutil . NewWriter ( _w ) return h . View ( func ( tx * bolt . Tx ) error { for _ , bucket := range buckets { b := tx . Bucket ( b ( bucket ) ) if _ , err := w . Write ( & BucketHeader { Bucket : bucket , } ) ; err != nil { return err } if err := b . ForEach ( func ( k , v [ ] byte ) error { if _ , err := w . WriteBytes ( k ) ; err != nil { return err } _ , err := w . WriteBytes ( v ) return err } ) ; err != nil { return err } if _ , err := w . WriteBytes ( SentinelByte ) ; err != nil { return err } } return nil } ) }

// Serialize serializes a binary version of the hashtree.

Serialize serializes a binary version of the hashtree .


func (h *dbHashTree) Deserialize(_r io.Reader) error {
	r := pbutil.NewReader(_r)
	hdr := &BucketHeader{}
	batchSize := 10000

	kvs := make(chan *keyValue, batchSize/10)
	// create cancellable ctx in case bolt writer encounters error
	eg, copyCtx := errgroup.WithContext(context.Background())
	eg.Go(func() error {
		var bucket []byte
		for {
			count := 0
			if err := h.Update(func(tx *bolt.Tx) error {
				if bucket != nil {
					tx.Bucket(bucket).FillPercent = 1
				}
				for kv := range kvs {
					if kv.k == nil {
						bucket = kv.v
						continue
					}
					if err := tx.Bucket(bucket).Put(kv.k, kv.v); err != nil {
						return err
					}
					count++
					if count >= batchSize {
						return nil
					}
				}
				return nil
			}); err != nil || copyCtx.Err() != nil {
				return err // may return nil if copyCtx was closed
			}
			if count <= 0 {
				return nil
			}
		}
	})
	eg.Go(func() error {
		defer close(kvs)
		for {
			hdr.Reset()
			// TODO(msteffen): don't block on Read if copyCtx() is cancelled?
			if err := r.Read(hdr); err != nil {
				if err == io.EOF {
					break
				}
				return err
			}
			bucket := b(hdr.Bucket)
			select {
			case kvs <- &keyValue{nil, bucket}:
			case <-copyCtx.Done():
				return nil
			}
			for {
				_k, err := r.ReadBytes()
				if err != nil {
					return err
				}
				if bytes.Equal(_k, SentinelByte) {
					break
				}
				// we need to make copies of k and v because the memory will be reused
				k := make([]byte, len(_k))
				copy(k, _k)
				_v, err := r.ReadBytes()
				if err != nil {
					return err
				}
				v := make([]byte, len(_v))
				copy(v, _v)
				select {
				case kvs <- &keyValue{k, v}:
				case <-copyCtx.Done():
					return nil
				}
			}
		}
		return nil
	})
	return eg.Wait()
}


func (h *dbHashTree) Copy() (HashTree, error) {
	if err := h.Hash(); err != nil {
		return nil, err
	}
	r, w := io.Pipe()
	var eg errgroup.Group
	eg.Go(func() (retErr error) {
		defer func() {
			if err := w.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()
		return h.Serialize(w)
	})
	var result HashTree
	eg.Go(func() error {
		var err error
		result, err = DeserializeDBHashTree(pathlib.Dir(h.Path()), r)
		return err
	})
	if err := eg.Wait(); err != nil {
		return nil, err
	}
	return result, nil
}

func ( h * dbHashTree ) Copy ( ) ( HashTree , error ) { if err := h . Hash ( ) ; err != nil { return nil , err } r , w := io . Pipe ( ) var eg errgroup . Group eg . Go ( func ( ) ( retErr error ) { defer func ( ) { if err := w . Close ( ) ; err != nil && retErr == nil { retErr = err } } ( ) return h . Serialize ( w ) } ) var result HashTree eg . Go ( func ( ) error { var err error result , err = DeserializeDBHashTree ( pathlib . Dir ( h . Path ( ) ) , r ) return err } ) if err := eg . Wait ( ) ; err != nil { return nil , err } return result , nil }


func (h *dbHashTree) Destroy() error {
	path := h.Path()
	if err := h.Close(); err != nil {
		return err
	}
	return os.Remove(path)
}

func ( h * dbHashTree ) Destroy ( ) error { path := h . Path ( ) if err := h . Close ( ) ; err != nil { return err } return os . Remove ( path ) }

// Destroy cleans up the on disk structures for the hashtree.

Destroy cleans up the on disk structures for the hashtree .


func (h *dbHashTree) PutFileOverwrite(path string, objects []*pfs.Object, overwriteIndex *pfs.OverwriteIndex, sizeDelta int64) error {
	return h.putFile(path, objects, overwriteIndex, sizeDelta, false)
}

func ( h * dbHashTree ) PutFileOverwrite ( path string , objects [ ] * pfs . Object , overwriteIndex * pfs . OverwriteIndex , sizeDelta int64 ) error { return h . putFile ( path , objects , overwriteIndex , sizeDelta , false ) }

// PutFileOverwrite is the same as PutFile, except that instead of
// appending the objects to the end of the given file, the objects
// are inserted to the given index, and the existing objects starting
// from the given index are removed.

PutFileOverwrite is the same as PutFile except that instead of appending the objects to the end of the given file the objects are inserted to the given index and the existing objects starting from the given index are removed .


func (h *dbHashTree) PutDirHeaderFooter(path string, header, footer *pfs.Object, headerSize, footerSize int64) error {
	path = clean(path)
	return h.Batch(func(tx *bolt.Tx) error {
		// validation: 'path' must point to directory (or nothing--may not be
		// created yet)
		node, err := get(tx, path)
		if err != nil && Code(err) != PathNotFound {
			return errorf(Internal, "could not get node at %q: %v", path, err)
		}
		if node != nil && node.nodetype() != directory {
			return errorf(PathConflict, "cannot add header to non-directory file "+
				"at %q; a file of type %s is already there", path, node.nodetype())
		}

		// Upsert directory at 'path' with 'Shared' field
		var newNode bool
		if node == nil {
			newNode = true
			node = &NodeProto{
				Name: base(path),
				DirNode: &DirectoryNodeProto{
					Shared: &Shared{},
				},
				// header/footer size are also stored in Shared (for CopyFile) but
				// adding it here makes size calculation in canonicalize() work
				SubtreeSize: headerSize + footerSize,
			}
		}

		// only write node to db if the node is new, or the header or footer
		// changed
		headerSame := (node.DirNode.Shared.Header == nil && header == nil) ||
			(node.DirNode.Shared.Header != nil && node.DirNode.Shared.Header.Hash == header.Hash)
		footerSame := (node.DirNode.Shared.Footer == nil && footer == nil) ||
			(node.DirNode.Shared.Footer != nil && node.DirNode.Shared.Footer.Hash == footer.Hash)
		if newNode || !headerSame || !footerSame {
			node.DirNode.Shared = &Shared{
				Header:     header,
				Footer:     footer,
				HeaderSize: headerSize,
				FooterSize: footerSize,
			}
			return put(tx, path, node)
		}
		return nil
	})
}

// PutDirHeaderFooter implements the hashtree.PutDirHeaderFooter interface
// method

PutDirHeaderFooter implements the hashtree . PutDirHeaderFooter interface method


func (h *dbHashTree) PutFileHeaderFooter(path string, objects []*pfs.Object, size int64) error {
	return h.putFile(path, objects, nil, size, true)
}

func ( h * dbHashTree ) PutFileHeaderFooter ( path string , objects [ ] * pfs . Object , size int64 ) error { return h . putFile ( path , objects , nil , size , true ) }

// PutFileHeaderFooter implements the HashTree PutFileHeaderFooter method


func deleteDir(tx *bolt.Tx, path string) error {
	c := fs(tx).Cursor()
	prefix := append(b(path), nullByte[0])
	for k, _ := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, _ = c.Next() {
		if err := c.Delete(); err != nil {
			return err
		}
	}
	return fs(tx).Delete(b(path))
}

func deleteDir ( tx * bolt . Tx , path string ) error { c := fs ( tx ) . Cursor ( ) prefix := append ( b ( path ) , nullByte [ 0 ] ) for k , _ := c . Seek ( prefix ) ; bytes . HasPrefix ( k , prefix ) ; k , _ = c . Next ( ) { if err := c . Delete ( ) ; err != nil { return err } } return fs ( tx ) . Delete ( b ( path ) ) }

// deleteDir deletes a directory and all the children under it


func NewReader(r io.Reader, filter Filter) *Reader {
	return &Reader{
		pbr:    pbutil.NewReader(r),
		filter: filter,
	}
}

func NewReader ( r io . Reader , filter Filter ) * Reader { return & Reader { pbr : pbutil . NewReader ( r ) , filter : filter , } }

// NewReader creates a new hashtree reader.

NewReader creates a new hashtree reader .


func (r *Reader) Read() (*MergeNode, error) {
	_k, err := r.pbr.ReadBytes()
	if err != nil {
		return nil, err
	}
	if r.filter != nil {
		for {
			if r.filter(_k) {
				break
			}
			_, err = r.pbr.ReadBytes()
			if err != nil {
				return nil, err
			}
			_k, err = r.pbr.ReadBytes()
			if err != nil {
				return nil, err
			}
		}

	}
	k := make([]byte, len(_k))
	copy(k, _k)
	_v, err := r.pbr.ReadBytes()
	if err != nil {
		return nil, err
	}
	v := make([]byte, len(_v))
	copy(v, _v)
	return &MergeNode{
		k: k,
		v: v,
	}, nil
}

func ( r * Reader ) Read ( ) ( * MergeNode , error ) { _k , err := r . pbr . ReadBytes ( ) if err != nil { return nil , err } if r . filter != nil { for { if r . filter ( _k ) { break } _ , err = r . pbr . ReadBytes ( ) if err != nil { return nil , err } _k , err = r . pbr . ReadBytes ( ) if err != nil { return nil , err } } } k := make ( [ ] byte , len ( _k ) ) copy ( k , _k ) _v , err := r . pbr . ReadBytes ( ) if err != nil { return nil , err } v := make ( [ ] byte , len ( _v ) ) copy ( v , _v ) return & MergeNode { k : k , v : v , } , nil }


func NewWriter(w io.Writer) *Writer {
	return &Writer{
		pbw: pbutil.NewWriter(w),
	}
}

func NewWriter ( w io . Writer ) * Writer { return & Writer { pbw : pbutil . NewWriter ( w ) , } }

// NewWriter creates a new hashtree writer.

NewWriter creates a new hashtree writer .


func (w *Writer) Write(n *MergeNode) error {
	// Marshal node if it was merged
	if n.nodeProto != nil {
		var err error
		n.v, err = n.nodeProto.Marshal()
		if err != nil {
			return err
		}
	}
	// Get size info from root node
	if bytes.Equal(n.k, nullByte) {
		if n.nodeProto == nil {
			n.nodeProto = &NodeProto{}
			if err := n.nodeProto.Unmarshal(n.v); err != nil {
				return err
			}
		}
		w.size = uint64(n.nodeProto.SubtreeSize)
	}
	// Write index for every index size bytes
	if w.offset > uint64(len(w.idxs)+1)*IndexSize {
		w.idxs = append(w.idxs, &Index{
			K:      n.k,
			Offset: w.offset,
		})
	}
	b, err := w.pbw.WriteBytes(n.k)
	if err != nil {
		return err
	}
	w.offset += uint64(b)
	b, err = w.pbw.WriteBytes(n.v)
	if err != nil {
		return err
	}
	w.offset += uint64(b)
	return nil
}


func (w *Writer) Copy(r *Reader) error {
	for {
		n, err := r.Read()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		if err := w.Write(n); err != nil {
			return err
		}
	}
}

func ( w * Writer ) Copy ( r * Reader ) error { for { n , err := r . Read ( ) if err != nil { if err == io . EOF { return nil } return err } if err := w . Write ( n ) ; err != nil { return err } } }

// Copy copies a hashtree reader in a writer.

Copy copies a hashtree reader in a writer .


func (w *Writer) Index() ([]byte, error) {
	buf := &bytes.Buffer{}
	pbw := pbutil.NewWriter(buf)
	for _, idx := range w.idxs {
		if _, err := pbw.Write(idx); err != nil {
			return nil, err
		}
	}
	return buf.Bytes(), nil
}

func ( w * Writer ) Index ( ) ( [ ] byte , error ) { buf := & bytes . Buffer { } pbw := pbutil . NewWriter ( buf ) for _ , idx := range w . idxs { if _ , err := pbw . Write ( idx ) ; err != nil { return nil , err } } return buf . Bytes ( ) , nil }

// Index returns the index for a hashtree writer.

Index returns the index for a hashtree writer .


func GetRangeFromIndex(r io.Reader, prefix string) (uint64, uint64, error) {
	prefix = clean(prefix)
	pbr := pbutil.NewReader(r)
	idx := &Index{}
	k := b(prefix)
	var lower, upper uint64
	iter := func(f func(int) bool) error {
		for {
			if err := pbr.Read(idx); err != nil {
				if err == io.EOF {
					break
				}
				return err
			}
			var cmp int
			if len(k) < len(idx.K) {
				cmp = bytes.Compare(k, idx.K[:len(k)])
			} else {
				cmp = bytes.Compare(k[:len(idx.K)], idx.K)
			}
			if f(cmp) {
				break
			}
		}
		return nil
	}
	low := func(cmp int) bool {
		if cmp > 0 {
			lower = idx.Offset
			return false
		} else if cmp < 0 {
			// Handles the case where a prefix fits within one range
			upper = idx.Offset
		}
		return true
	}
	up := func(cmp int) bool {
		if cmp < 0 {
			upper = idx.Offset
			return true
		}
		return false
	}
	// Find lower
	iter(low)
	// Find upper
	if upper <= 0 {
		iter(up)
	}
	// Handles the case when at the end of the indexes
	if upper <= 0 {
		return lower, 0, nil
	}
	// Return offset and size
	return lower, upper - lower, nil
}

// GetRangeFromIndex returns a subtree byte range in a serialized hashtree based on a passed in prefix.

GetRangeFromIndex returns a subtree byte range in a serialized hashtree based on a passed in prefix .


func NewFilter(numTrees int64, tree int64) Filter {
	return func(k []byte) bool {
		if pathToTree(k, numTrees) == uint64(tree) {
			return true
		}
		return false
	}
}

func NewFilter ( numTrees int64 , tree int64 ) Filter { return func ( k [ ] byte ) bool { if pathToTree ( k , numTrees ) == uint64 ( tree ) { return true } return false } }

// NewFilter creates a filter for a hashtree shard.

NewFilter creates a filter for a hashtree shard .


func PathToTree(path string, numTrees int64) uint64 {
	path = clean(path)
	return pathToTree(b(path), numTrees)
}

func PathToTree ( path string , numTrees int64 ) uint64 { path = clean ( path ) return pathToTree ( b ( path ) , numTrees ) }

// PathToTree computes the hashtree shard for a path.

PathToTree computes the hashtree shard for a path .


func Merge(w *Writer, rs []*Reader) error {
	if len(rs) == 0 {
		return nil
	}
	mq := &mergePQ{q: make([]*nodeStream, len(rs)+1)}
	// Setup first set of nodes
	for _, r := range rs {
		if err := mq.insert(&nodeStream{r: r}); err != nil {
			return err
		}
	}
	for mq.q[1] != nil {
		// Get next nodes to merge
		ns, err := mq.next()
		if err != nil {
			return err
		}
		// Merge nodes
		n, err := merge(ns)
		if err != nil {
			return err
		}
		// Write out result
		if err := w.Write(n); err != nil {
			return err
		}
	}
	return nil
}

func
Merge
(
w
*
Writer
,
rs
[
]
*
Reader
)
error
{
if
len
(
rs
)
==
0
{
return
}
mq
:=
&
mergePQ
{
q
:
make
(
[
]
*
nodeStream
,
len
(
rs
)
+
1
)
}
// Setup first set of nodes
for
_
,
r
:=
range
rs
{
if
err
:=
mq
.
insert
(
&
nodeStream
{
r
:
r
}
)
;
err
!=
{
return
err
}
}
for
mq
.
q
[
1
]
!=
{
// Get next nodes to merge
ns
,
err
:=
mq
.
next
(
)
if
err
!=
{
return
err
}
// Merge nodes
n
,
err
:=
merge
(
ns
)
if
err
!=
{
return
err
}
// Write out result
if
err
:=
w
.
Write
(
n
)
;
err
!=
{
return
err
}
}
return
}

// Merge merges a collection of hashtree readers into a hashtree writer.

Merge merges a collection of hashtree readers into a hashtree writer .


func (h *dbHashTree) Hash() error {
	return h.Batch(func(tx *bolt.Tx) error {
		return canonicalize(tx, "")
	})
}

func ( h * dbHashTree ) Hash ( ) error { return h . Batch ( func ( tx * bolt . Tx ) error { return canonicalize ( tx , " " ) } ) }

// Hash updates all of the hashes and node size metadata, it also checks
// for conflicts.

Hash updates all of the hashes and node size metadata it also checks for conflicts .


func IsGlob(pattern string) bool {
	pattern = clean(pattern)
	return globRegex.Match([]byte(pattern))
}

func IsGlob ( pattern string ) bool { pattern = clean ( pattern ) return globRegex . Match ( [ ] byte ( pattern ) ) }

// IsGlob checks if the pattern contains a glob character


func GlobLiteralPrefix(pattern string) string {
	pattern = clean(pattern)
	idx := globRegex.FindStringIndex(pattern)
	if idx == nil {
		return pattern
	}
	return pattern[:idx[0]]
}

func GlobLiteralPrefix ( pattern string ) string { pattern = clean ( pattern ) idx := globRegex . FindStringIndex ( pattern ) if idx == nil { return pattern } return pattern [ : idx [ 0 ] ] }

// GlobLiteralPrefix returns the prefix before the first glob character


func GetHashTreeObject(pachClient *client.APIClient, storageRoot string, treeRef *pfs.Object) (HashTree, error) {
	return getHashTree(storageRoot, func(w io.Writer) error {
		return pachClient.GetObject(treeRef.Hash, w)
	})
}

func GetHashTreeObject ( pachClient * client . APIClient , storageRoot string , treeRef * pfs . Object ) ( HashTree , error ) { return getHashTree ( storageRoot , func ( w io . Writer ) error { return pachClient . GetObject ( treeRef . Hash , w ) } ) }

// GetHashTreeObject is a convenience function to deserialize a HashTree from an object in the object store.

GetHashTreeObject is a convenience function to deserialize a HashTree from an object in the object store .


func GetHashTreeTag(pachClient *client.APIClient, storageRoot string, treeRef *pfs.Tag) (HashTree, error) {
	return getHashTree(storageRoot, func(w io.Writer) error {
		return pachClient.GetTag(treeRef.Name, w)
	})
}

func GetHashTreeTag ( pachClient * client . APIClient , storageRoot string , treeRef * pfs . Tag ) ( HashTree , error ) { return getHashTree ( storageRoot , func ( w io . Writer ) error { return pachClient . GetTag ( treeRef . Name , w ) } ) }

// GetHashTreeTag is a convenience function to deserialize a HashTree from an tagged object in the object store.

GetHashTreeTag is a convenience function to deserialize a HashTree from an tagged object in the object store .


func PutHashTree(pachClient *client.APIClient, tree HashTree, tags ...string) (*pfs.Object, error) {
	r, w := io.Pipe()
	var eg errgroup.Group
	eg.Go(func() (retErr error) {
		defer func() {
			if err := w.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()
		return tree.Serialize(w)
	})
	var treeRef *pfs.Object
	eg.Go(func() error {
		var err error
		treeRef, _, err = pachClient.PutObject(r, tags...)
		return err
	})
	if err := eg.Wait(); err != nil {
		return nil, err
	}
	return treeRef, nil
}

func PutHashTree ( pachClient * client . APIClient , tree HashTree , tags ... string ) ( * pfs . Object , error ) { r , w := io . Pipe ( ) var eg errgroup . Group eg . Go ( func ( ) ( retErr error ) { defer func ( ) { if err := w . Close ( ) ; err != nil && retErr == nil { retErr = err } } ( ) return tree . Serialize ( w ) } ) var treeRef * pfs . Object eg . Go ( func ( ) error { var err error treeRef , _ , err = pachClient . PutObject ( r , tags ... ) return err } ) if err := eg . Wait ( ) ; err != nil { return nil , err } return treeRef , nil }

// PutHashTree is a convenience function for putting a HashTree to an object store.

PutHashTree is a convenience function for putting a HashTree to an object store .


func NewChildCursor(tx *bolt.Tx, path string) *ChildCursor {
	path = clean(path)
	c := fs(tx).Cursor()
	dir := b(path)
	k, v := c.Seek(append(dir, nullByte[0]))
	if !bytes.Equal(dir, nullByte) {
		dir = append(dir, nullByte[0])
	}
	if !bytes.HasPrefix(k, dir) {
		k, v = nil, nil
	}
	return &ChildCursor{
		c:   c,
		dir: dir,
		k:   k,
		v:   v,
	}
}

func NewChildCursor ( tx * bolt . Tx , path string ) * ChildCursor { path = clean ( path ) c := fs ( tx ) . Cursor ( ) dir := b ( path ) k , v := c . Seek ( append ( dir , nullByte [ 0 ] ) ) if ! bytes . Equal ( dir , nullByte ) { dir = append ( dir , nullByte [ 0 ] ) } if ! bytes . HasPrefix ( k , dir ) { k , v = nil , nil } return & ChildCursor { c : c , dir : dir , k : k , v : v , } }

// NewChildCursor creates a new child cursor.

NewChildCursor creates a new child cursor .


func (d *ChildCursor) Next() ([]byte, []byte) {
	if d.k == nil {
		return nil, nil
	}
	k, v := d.c.Seek(append(d.k, 1))
	if !bytes.HasPrefix(k, d.dir) {
		k, v = nil, nil
	}
	d.k, d.v = k, v
	return k, v
}

func ( d * ChildCursor ) Next ( ) ( [ ] byte , [ ] byte ) { if d . k == nil { return nil , nil } k , v := d . c . Seek ( append ( d . k , 1 ) ) if ! bytes . HasPrefix ( k , d . dir ) { k , v = nil , nil } d . k , d . v = k , v return k , v }


func NewOrdered(root string) *Ordered {
	root = clean(root)
	o := &Ordered{}
	n := &node{
		path: "",
		nodeProto: &NodeProto{
			Name:    "",
			DirNode: &DirectoryNodeProto{},
		},
		hash: sha256.New(),
	}
	o.fs = append(o.fs, n)
	o.dirStack = append(o.dirStack, n)
	o.MkdirAll(root)
	o.root = root
	return o
}

func NewOrdered ( root string ) * Ordered { root = clean ( root ) o := & Ordered { } n := & node { path : " " , nodeProto : & NodeProto { Name : " " , DirNode : & DirectoryNodeProto { } , } , hash : sha256 . New ( ) , } o . fs = append ( o . fs , n ) o . dirStack = append ( o . dirStack , n ) o . MkdirAll ( root ) o . root = root return o }

// NewOrdered creates a new ordered hashtree.

NewOrdered creates a new ordered hashtree .


func (o *Ordered) MkdirAll(path string) {
	var paths []string
	for path != "" {
		paths = append(paths, path)
		path, _ = split(path)
	}
	for i := len(paths) - 1; i >= 0; i-- {
		o.PutDir(paths[i])
	}
}

func ( o * Ordered ) MkdirAll ( path string ) { var paths [ ] string for path != " " { paths = append ( paths , path ) path , _ = split ( path ) } for i := len ( paths ) - 1 ; i >= 0 ; i -- { o . PutDir ( paths [ i ] ) } }

// MkdirAll puts all of the parent directories of a given
// path into the hashtree.

MkdirAll puts all of the parent directories of a given path into the hashtree .