Repository: ClickHouse/ClickHouse
File: tests/queries/0_stateless/01079_parallel_alter_detach_table_zookeeper.sh (Shell)

Code before:
sleep 1

for i in $(seq $REPLICAS); do
    $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$i" 2> /dev/null
done

# This alter will finish all previous, but replica 1 maybe still not up-to-date.
# Limit retries to avoid infinite loops that cause test timeouts (especially in DatabaseReplicated mode
# where DDL operations go through a distributed queue and can take longer).
for _ in {1..5}; do
    output=$(timeout 60 ${CLICKHOUSE_CLIENT} --query "ALTER TABLE concurrent_alter_detach_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1)
    if [[ -z "$output" ]]; then
        break
    fi
    sleep 1
    # just try to attach table if it failed for some reason in the code above
    for i in $(seq $REPLICAS); do
        $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$i" 2> /dev/null
    done
done

for i in $(seq $REPLICAS); do
    $CLICKHOUSE_CLIENT --receive_timeout 120 --query "SYSTEM SYNC REPLICA concurrent_alter_detach_$i"
    $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_detach_$i"

    # Wait for all mutations and replication queue entries to finish.
    # SYSTEM SYNC REPLICA may return while a MUTATE_PART entry is postponed
    # (e.g. waiting for an in-progress MERGE_PARTS on the same part to complete).
    for _ in {1..120}; do
        mutations_count=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' AND is_done = 0 AND table = 'concurrent_alter_detach_$i'")
        queue_count=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.replication_queue WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'concurrent_alter_detach_$i' AND (type = 'ALTER_METADATA' OR type = 'MUTATE_PART')")
        [[ "$mutations_count" == "0" && "$queue_count" == "0" ]] && break
        sleep 1
    done

    $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done
    $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done
    $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'concurrent_alter_detach_$i' and (type = 'ALTER_METADATA' or type = 'MUTATE_PART')" # all mutations and alters have to be done
    $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_detach_$i" 2>&1 | grep -Fv "TIMEOUT_EXCEEDED" ||:
done


Review comment (suggestion):
Not sure about the exact reasoning. At least I'd expect that when the entry is in the queue, a subsequent SYNC REPLICA has to wait for it.

Code after:
sleep 1

for i in $(seq $REPLICAS); do
    $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$i" 2> /dev/null
done

# This alter will finish all previous, but replica 1 maybe still not up-to-date.
# Limit retries to avoid infinite loops that cause test timeouts (especially in DatabaseReplicated mode
# where DDL operations go through a distributed queue and can take longer).
for _ in {1..5}; do
    output=$(timeout 60 ${CLICKHOUSE_CLIENT} --query "ALTER TABLE concurrent_alter_detach_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>&1)
    if [[ -z "$output" ]]; then
        break
    fi
    sleep 1
    # just try to attach table if it failed for some reason in the code above
    for i in $(seq $REPLICAS); do
        $CLICKHOUSE_CLIENT --query "ATTACH TABLE concurrent_alter_detach_$i" 2> /dev/null
    done
done

for i in $(seq $REPLICAS); do
    $CLICKHOUSE_CLIENT --receive_timeout 120 --query "SYSTEM SYNC REPLICA concurrent_alter_detach_$i"
    $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_detach_$i"

    # Wait for all mutations and replication queue entries to finish.
    # In this case, it might be more robust than SYSTEM SYNC REPLICA
    # for unclear reasons.
    for _ in {1..120}; do
        mutations_count=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' AND is_done = 0 AND table = 'concurrent_alter_detach_$i'")
        queue_count=$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.replication_queue WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'concurrent_alter_detach_$i' AND (type = 'ALTER_METADATA' OR type = 'MUTATE_PART')")
        [[ "$mutations_count" == "0" && "$queue_count" == "0" ]] && break
        sleep 1
    done

    $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done
    $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE database = '$CLICKHOUSE_DATABASE' and is_done=0 and table = 'concurrent_alter_detach_$i'" # all mutations have to be done
    $CLICKHOUSE_CLIENT --query "SELECT * FROM system.replication_queue WHERE database = '$CLICKHOUSE_DATABASE' AND table = 'concurrent_alter_detach_$i' and (type = 'ALTER_METADATA' or type = 'MUTATE_PART')" # all mutations and alters have to be done
    $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_detach_$i" 2>&1 | grep -Fv "TIMEOUT_EXCEEDED" ||:
done