Repository: pingcap/tidb
File: pkg/lightning/mydump/region.go (Go)

Code before:
	return parser.Columns(), startOffset, nil
}

// SplitLargeCSV splits a large csv file into multiple regions, the size of
// each regions is specified by `config.MaxRegionSize`.
// Note: We split the file coarsely, thus the format of csv file is needed to be
// strict.
// e.g.
// - CSV file with header is invalid
// - a complete tuple split into multiple lines is invalid
func SplitLargeCSV(
	ctx context.Context,
	cfg *DataDivideConfig,
	dataFile FileInfo,
	parallel bool,
) (regions []*TableRegion, dataFileSizes []float64, err error) {
	maxRegionSize := cfg.MaxChunkSize
	regionCnt := dataFile.FileMeta.FileSize/maxRegionSize + 1
	dataFileSizes = make([]float64, 0, regionCnt)

	headerColumns, dataStart, err := getHeaderColumn(ctx, cfg, dataFile.FileMeta.Path)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}

	splitEndOffsets := make([]int64, 0, regionCnt)
	endOffset := dataStart + maxRegionSize
	for endOffset < dataFile.FileMeta.FileSize {
		splitEndOffsets = append(splitEndOffsets, endOffset)
		endOffset += maxRegionSize
	}

	concurrency := 1
	if parallel {
		concurrency = max(cfg.Concurrency, 1)
	}

	eg, egCtx := errgroup.WithContext(ctx)
	eg.SetLimit(concurrency)
	for i, endOffset := range splitEndOffsets {
		eg.Go(func() error {
			select {
			case <-egCtx.Done():
				return egCtx.Err()
			default:
			}

			parser, err := openCSVParser(egCtx, cfg, dataFile.FileMeta.Path, endOffset)
			if err != nil {
				return errors.Trace(err)
			}

Review comment (suggestion):
i prefer we calculate the split point, not by this offset which strictly added by `maxRegionSize` , so the splitted chunk will be more even, such as for a 512M file, current impl will try read at offset 256M and 512M

but if we use split point, we only need to read at 256M

Code after:
	if err = parser.ReadColumns(); err != nil {
		return nil, 0, err
	}
	startOffset, _ := parser.Pos()
	return parser.Columns(), startOffset, nil
}

// SplitLargeCSV splits a large csv file into multiple regions, the size of
// each regions is specified by `config.MaxRegionSize`.
// Note: We split the file coarsely, thus the format of csv file is needed to be
// strict.
// e.g.
// - CSV file with header is invalid
// - a complete tuple split into multiple lines is invalid
func SplitLargeCSV(
	ctx context.Context,
	cfg *DataDivideConfig,
	dataFile FileInfo,
	parallel bool,
) (regions []*TableRegion, dataFileSizes []float64, err error) {
	maxRegionSize := cfg.MaxChunkSize

	headerColumns, dataStart, err := getHeaderColumn(ctx, cfg, dataFile.FileMeta.Path)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}

	regionCnt := (dataFile.FileMeta.FileSize - dataStart + maxRegionSize - 1) / maxRegionSize
	dataFileSizes = make([]float64, 0, regionCnt)

	splitPoints := mathutil.Divide2Batches(dataFile.FileMeta.FileSize-dataStart, regionCnt)
	splitPoints = splitPoints[:len(splitPoints)-1]
	for i := range splitPoints {
		if i == 0 {
			splitPoints[i] += dataStart
		} else {
			splitPoints[i] += splitPoints[i-1]
		}
	}

	concurrency := 1
	if parallel {
		concurrency = max(cfg.Concurrency, 1)
	}

	eg, egCtx := errgroup.WithContext(ctx)
	eg.SetLimit(concurrency)
	for i, splitPoint := range splitPoints {
		eg.Go(func() error {
			select {
			case <-egCtx.Done():