Code review comment: Please add example of mv column and schema change column

Code before review:
private void analyzeSubquery(Analyzer analyzer) throws UserException {
Set<String> mentionedColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
if (targetColumnNames == null) {
for (Column col : targetTable.getBaseSchema()) {
mentionedColumns.add(col.getName());
targetColumns.add(col);
}
} else {
for (String colName : targetColumnNames) {
Column col = targetTable.getColumn(colName);
if (col == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_FIELD_ERROR, colName, targetTable.getName());
}
if (!mentionedColumns.add(colName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_FIELD_SPECIFIED_TWICE, colName);
}
targetColumns.add(col);
}
for (Column col : targetTable.getBaseSchema()) {
if (col.getType().isHllType() && !mentionedColumns.contains(col.getName())) {
throw new AnalysisException (" hll column " + col.getName() + " mush in insert into columns");
}
if (col.getType().isBitmapType() && !mentionedColumns.contains(col.getName())) {
throw new AnalysisException (" object column " + col.getName() + " mush in insert into columns");
}
}
}
/*
* When doing schema change, there may be some shadow columns. we should add
* them to the end of targetColumns. And use 'origColIdxsForShadowCols' to save
* the index of column in 'targetColumns' which the shadow column related to.
* eg: origin targetColumns: (A,B,C), shadow column: __doris_shadow_B after
* processing, targetColumns: (A, B, C, __doris_shadow_B), and
* origColIdxsForShadowCols has 1 element: "1", which is the index of column B
* in targetColumns.
*
* Rule A: If the column which the shadow column related to is not mentioned,
* then do not add the shadow column to targetColumns. They will be filled by
* null or default value when loading.
*/
List<Integer> origColIdxsForShadowCols = Lists.newArrayList();
for (Column column : targetTable.getFullSchema()) {
if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) {
String origName = Column.removeNamePrefix(column.getName());
for (int i = 0; i < targetColumns.size(); i++) {
if (targetColumns.get(i).nameEquals(origName, false)) {
origColIdxsForShadowCols.add(i);
targetColumns.add(column);
break;
}
}
}
}
Map<Integer, Integer> origColIdx2MvColIdx = Maps.newHashMap();
for (int mvColumnIdx = 0; mvColumnIdx < targetTable.getFullSchema().size(); ++mvColumnIdx) {
Column column = targetTable.getFullSchema().get(mvColumnIdx);
if (column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PRFIX)) {
List<Expr> slots = new ArrayList<>();
column.getDefineExpr().collect(SlotRef.class, slots);
Preconditions.checkArgument(slots.size() == 1);
String origName = ((SlotRef) slots.get(0)).getColumnName();
for (int originColumnIdx = 0; originColumnIdx < targetColumns.size(); originColumnIdx++) {
if (targetColumns.get(originColumnIdx).nameEquals(origName, false)) {
origColIdx2MvColIdx.put(mvColumnIdx, originColumnIdx);
targetColumns.add(column);
break;
}
}
}
}
queryStmt.setFromInsert(true);
queryStmt.analyze(analyzer);
if (mentionedColumns.size() != queryStmt.getResultExprs().size()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT);
}
checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ;
if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
if (selectStmt.getValueList() != null) {
List<ArrayList<Expr>> rows = selectStmt.getValueList().getRows();
for (int rowIdx = 0; rowIdx < rows.size(); ++rowIdx) {
analyzeRow(analyzer, targetColumns, rows, rowIdx, origColIdxsForShadowCols, origColIdx2MvColIdx);
}
selectStmt.getResultExprs().clear();
selectStmt.getBaseTblResultExprs().clear();
for (int i = 0; i < selectStmt.getValueList().getFirstRow().size(); ++i) {
selectStmt.getResultExprs().add(selectStmt.getValueList().getFirstRow().get(i));
selectStmt.getBaseTblResultExprs().add(selectStmt.getValueList().getFirstRow().get(i));
}
} else {
List<ArrayList<Expr>> rows = Lists.newArrayList();
rows.add(selectStmt.getResultExprs());
analyzeRow(analyzer, targetColumns, rows, 0, origColIdxsForShadowCols, origColIdx2MvColIdx);
selectStmt.getResultExprs().clear();
for (Expr expr : rows.get(0)) {
selectStmt.getResultExprs().add(expr);
}
}
isStreaming = true;
} else {
if (!origColIdxsForShadowCols.isEmpty()) {
for (Integer idx : origColIdxsForShadowCols) {
queryStmt.getResultExprs().add(queryStmt.getResultExprs().get(idx));
}
}
if (!origColIdx2MvColIdx.isEmpty()) {
origColIdx2MvColIdx.forEach((key, value) -> {
Column mvColumn = targetTable.getFullSchema().get(key);
Expr expr = mvColumn.getDefineExpr();
ArrayList<SlotRef> slots = new ArrayList<>();
expr.collect(SlotRef.class, slots);
ExprSubstitutionMap smap = new ExprSubstitutionMap();
smap.getLhs().add(slots.get(0));
smap.getRhs().add(queryStmt.getResultExprs().get(value));
queryStmt.getResultExprs().add(Expr.substituteList(Lists.newArrayList(expr), smap, analyzer, false).get(0));
});
}
for (int i = 0; i < targetColumns.size(); ++i) {
Column column = targetColumns.get(i);
if (column.getType().isHllType()) {
Expr expr = queryStmt.getResultExprs().get(i);
checkHllCompatibility(column, expr);
}
if (column.getAggregationType() == AggregateType.BITMAP_UNION) {
Expr expr = queryStmt.getResultExprs().get(i);
checkBitmapCompatibility(column, expr);
}
}
}
if (!origColIdxsForShadowCols.isEmpty() || !origColIdx2MvColIdx.isEmpty()) {
if (queryStmt.getResultExprs().size() != queryStmt.getBaseTblResultExprs().size()) {
for (Integer idx : origColIdxsForShadowCols) {
queryStmt.getBaseTblResultExprs().add(queryStmt.getBaseTblResultExprs().get(idx));
}
for (Integer idx : origColIdx2MvColIdx.keySet()) {
queryStmt.getBaseTblResultExprs().add(queryStmt.getResultExprs().get(idx));
}
}
if (queryStmt.getResultExprs().size() != queryStmt.getColLabels().size()) {
for (Integer idx : origColIdxsForShadowCols) {
queryStmt.getColLabels().add(queryStmt.getColLabels().get(idx));
}
for (Integer idx : origColIdx2MvColIdx.values()) {
queryStmt.getColLabels().add(queryStmt.getColLabels().get(idx));
}
}
}
if (LOG.isDebugEnabled()) {
for (Expr expr : queryStmt.getResultExprs()) {
LOG.debug("final result expr: {}, {}", expr, System.identityHashCode(expr));
}
for (Expr expr : queryStmt.getBaseTblResultExprs()) {
LOG.debug("final base table result expr: {}, {}", expr, System.identityHashCode(expr));
}
for (String colLabel : queryStmt.getColLabels()) {
LOG.debug("final col label: {}", colLabel);
}
}
}


Code after addressing the review:
private void analyzeSubquery(Analyzer analyzer) throws UserException {
Set<String> mentionedColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
if (targetColumnNames == null) {
for (Column col : targetTable.getBaseSchema()) {
mentionedColumns.add(col.getName());
targetColumns.add(col);
}
} else {
for (String colName : targetColumnNames) {
Column col = targetTable.getColumn(colName);
if (col == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_FIELD_ERROR, colName, targetTable.getName());
}
if (!mentionedColumns.add(colName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_FIELD_SPECIFIED_TWICE, colName);
}
targetColumns.add(col);
}
for (Column col : targetTable.getBaseSchema()) {
if (col.getType().isHllType() && !mentionedColumns.contains(col.getName())) {
throw new AnalysisException (" hll column " + col.getName() + " mush in insert into columns");
}
if (col.getType().isBitmapType() && !mentionedColumns.contains(col.getName())) {
throw new AnalysisException (" object column " + col.getName() + " mush in insert into columns");
}
}
}
/*
* When doing schema change, there may be some shadow columns. we should add
* them to the end of targetColumns. And use 'origColIdxsForExtendCols' to save
* the index of column in 'targetColumns' which the shadow column related to.
* eg: origin targetColumns: (A,B,C), shadow column: __doris_shadow_B after
* processing, targetColumns: (A, B, C, __doris_shadow_B), and
* origColIdxsForExtendCols has 1 element: "1", which is the index of column B
* in targetColumns.
*
* Rule A: If the column which the shadow column related to is not mentioned,
* then do not add the shadow column to targetColumns. They will be filled by
* null or default value when loading.
*
* When table have materialized view, there may be some materialized view columns.
* we should add them to the end of targetColumns.
* eg: origin targetColumns: (A,B,C), shadow column: mv_bitmap_union_C
* after processing, targetColumns: (A, B, C, mv_bitmap_union_C), and
* origColIdx2MVColumn has 1 element: "2, mv_bitmap_union_C"
* will be used in as a mapping from queryStmt.getResultExprs() to targetColumns define expr
*/
List<Pair<Integer, Column>> origColIdxsForExtendCols = Lists.newArrayList();
for (Column column : targetTable.getFullSchema()) {
if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PRFIX)) {
String origName = Column.removeNamePrefix(column.getName());
for (int i = 0; i < targetColumns.size(); i++) {
if (targetColumns.get(i).nameEquals(origName, false)) {
origColIdxsForExtendCols.add(new Pair<>(i, null));
targetColumns.add(column);
break;
}
}
}
if (column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PRFIX)) {
SlotRef refColumn = column.getRefColumn();
if (refColumn == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_FIELD_ERROR, column.getName(), targetTable.getName());
}
String origName = refColumn.getColumnName();
for (int originColumnIdx = 0; originColumnIdx < targetColumns.size(); originColumnIdx++) {
if (targetColumns.get(originColumnIdx).nameEquals(origName, false)) {
origColIdxsForExtendCols.add(new Pair<>(originColumnIdx, column));
targetColumns.add(column);
break;
}
}
}
}
queryStmt.setFromInsert(true);
queryStmt.analyze(analyzer);
if (mentionedColumns.size() != queryStmt.getResultExprs().size()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT);
}
checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ;
if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getTableRefs().isEmpty()) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
if (selectStmt.getValueList() != null) {
List<ArrayList<Expr>> rows = selectStmt.getValueList().getRows();
for (int rowIdx = 0; rowIdx < rows.size(); ++rowIdx) {
analyzeRow(analyzer, targetColumns, rows, rowIdx, origColIdxsForExtendCols);
}
selectStmt.getResultExprs().clear();
selectStmt.getBaseTblResultExprs().clear();
for (int i = 0; i < selectStmt.getValueList().getFirstRow().size(); ++i) {
selectStmt.getResultExprs().add(selectStmt.getValueList().getFirstRow().get(i));
selectStmt.getBaseTblResultExprs().add(selectStmt.getValueList().getFirstRow().get(i));
}
} else {
List<ArrayList<Expr>> rows = Lists.newArrayList();
rows.add(selectStmt.getResultExprs());
analyzeRow(analyzer, targetColumns, rows, 0, origColIdxsForExtendCols);
selectStmt.getResultExprs().clear();
for (Expr expr : rows.get(0)) {
selectStmt.getResultExprs().add(expr);
}
}
isStreaming = true;
} else {
if (!origColIdxsForExtendCols.isEmpty()) {
for (Pair<Integer, Column> entry : origColIdxsForExtendCols) {
if (entry.second == null) {
queryStmt.getResultExprs().add(queryStmt.getResultExprs().get(entry.first));
} else {
ExprSubstitutionMap smap = new ExprSubstitutionMap();
smap.getLhs().add(entry.second.getRefColumn());
smap.getRhs().add(queryStmt.getResultExprs().get(entry.first));
Expr e = Expr.substituteList(Lists.newArrayList(entry.second.getDefineExpr()), smap, analyzer, false).get(0);
queryStmt.getResultExprs().add(e);
}
}
}
for (int i = 0; i < targetColumns.size(); ++i) {
Column column = targetColumns.get(i);
if (column.getType().isHllType()) {
Expr expr = queryStmt.getResultExprs().get(i);
checkHllCompatibility(column, expr);
}
if (column.getAggregationType() == AggregateType.BITMAP_UNION) {
Expr expr = queryStmt.getResultExprs().get(i);
checkBitmapCompatibility(column, expr);
}
}
}
if (!origColIdxsForExtendCols.isEmpty()) {
if (queryStmt.getResultExprs().size() != queryStmt.getBaseTblResultExprs().size()) {
for (Pair<Integer, Column> entry  : origColIdxsForExtendCols) {
if (entry.second == null) {
queryStmt.getBaseTblResultExprs().add(queryStmt.getBaseTblResultExprs().get(entry.first));
} else {
ExprSubstitutionMap smap = new ExprSubstitutionMap();
smap.getLhs().add(entry.second.getRefColumn());
smap.getRhs().add(queryStmt.getResultExprs().get(entry.first));
Expr e = Expr.substituteList(Lists.newArrayList(entry.second.getDefineExpr()), smap, analyzer, false).get(0);
queryStmt.getBaseTblResultExprs().add(e);
}
}
}
if (queryStmt.getResultExprs().size() != queryStmt.getColLabels().size()) {
for (Pair<Integer, Column> entry : origColIdxsForExtendCols) {
queryStmt.getColLabels().add(queryStmt.getColLabels().get(entry.first));
}
}
}
if (LOG.isDebugEnabled()) {
for (Expr expr : queryStmt.getResultExprs()) {
LOG.debug("final result expr: {}, {}", expr, System.identityHashCode(expr));
}
for (Expr expr : queryStmt.getBaseTblResultExprs()) {
LOG.debug("final base table result expr: {}, {}", expr, System.identityHashCode(expr));
}
for (String colLabel : queryStmt.getColLabels()) {
LOG.debug("final col label: {}", colLabel);
}
}
}