Repository: dolthub/dolt
File: go/store/nbs/bs_persister.go (Go)

Code before:
	})
	eg.Go(func() error {
		_, err := bsp.bs.Put(ectx, name+tableTailExt, int64(len(tail)), bytes.NewBuffer(tail))
		return err
	})
	if err = eg.Wait(); err != nil {
		return nil, gcBehavior_Continue, err
	}

	// then concatenate into a final blob
	if _, err = bsp.bs.Concatenate(ctx, name, []string{name + tableRecordsExt, name + tableTailExt}); err != nil {
		return emptyChunkSource{}, gcBehavior_Continue, err
	}
	rdr := &bsTableReaderAt{key: name, bs: bsp.bs}
	src, err := newReaderFromIndexData(ctx, bsp.q, data, address, rdr, bsp.blockSize)
	if err != nil {
		return emptyChunkSource{}, gcBehavior_Continue, err
	}
	return src, gcBehavior_Continue, nil
}

// ConjoinAll implements tablePersister.
func (bsp *blobstorePersister) ConjoinAll(ctx context.Context, sources chunkSources, stats *Stats) (chunkSource, cleanupFunc, error) {
	plan, err := planRangeCopyConjoin(ctx, sources, stats)
	if err != nil {
		return nil, nil, err
	}

	if plan.chunkCount == 0 {
		return emptyChunkSource{}, nil, nil
	}

	name := plan.name.String() + plan.suffix

	// conjoin must contiguously append the chunk records of |sources|, but the raw content
	// of each source contains a chunk index in the tail. Blobstore does not expose a range
	// copy (GCP Storage limitation), so we must create sub-objects from each source that
	// contain only chunk records. We make an effort to store these sub-objects on Persist(),
	// but we will create them in getRecordsSubObjects if necessary.

	conjoinees := make([]string, 0, len(sources)+1)
	for _, src := range plan.sources.sws {
		sub, err := bsp.getRecordsSubObject(ctx, src.source)
		if err != nil {
			return nil, nil, err
		}
		conjoinees = append(conjoinees, sub)
	}

	// first concatenate all the sub-objects to create a composite sub-object
	recordsName := name + tableRecordsExt

Review comment (bug):
In my reading of the code this looks like it might be a bug, because planArchiveConjoin is using the data sizes to track offsets for the byte spans entries. I don't know why it wouldn't have come in testing though...

Code after:
	eg.Go(func() error {
		_, err := bsp.bs.Put(ectx, recordsName, int64(len(records)), bytes.NewBuffer(records))
		return err
	})
	eg.Go(func() error {
		_, err := bsp.bs.Put(ectx, tailName, int64(len(tail)), bytes.NewBuffer(tail))
		return err
	})
	if err = eg.Wait(); err != nil {
		return nil, gcBehavior_Continue, err
	}

	// then concatenate into a final blob
	if _, err = bsp.bs.Concatenate(ctx, name, []string{recordsName, tailName}); err != nil {
		return emptyChunkSource{}, gcBehavior_Continue, err
	}
	rdr := &bsTableReaderAt{key: name, bs: bsp.bs}
	src, err := newReaderFromIndexData(ctx, bsp.q, data, address, rdr, bsp.blockSize)
	if err != nil {
		return emptyChunkSource{}, gcBehavior_Continue, err
	}
	return src, gcBehavior_Continue, nil
}

// ConjoinAll implements tablePersister.
func (bsp *blobstorePersister) ConjoinAll(ctx context.Context, sources chunkSources, stats *Stats) (chunkSource, cleanupFunc, error) {
	plan, err := planRangeCopyConjoin(ctx, sources, stats)
	if err != nil {
		return nil, nil, err
	}

	if plan.chunkCount == 0 {
		return emptyChunkSource{}, nil, nil
	}

	name := plan.name.String() + plan.suffix

	// conjoin must contiguously append the chunk records of |sources|, but the raw content
	// of each source contains a chunk index in the tail. Blobstore does not expose a range
	// copy (GCP Storage limitation), so we must create sub-objects from each source that
	// contain only chunk records. We make an effort to store these sub-objects on Persist(),
	// but we will create them in getRecordsSubObjects if necessary.

	conjoinees := make([]string, 0, len(sources)+1)
	for _, src := range plan.sources.sws {
		sub, err := bsp.getRecordsSubObject(ctx, src.source)
		if err != nil {
			return nil, nil, err
		}
		conjoinees = append(conjoinees, sub)
	}