Repository: pytorch
Issue: CUBLAS_STATUS_NOT_SUPPORTED
Labels: needs reproduction,module: cuda,triaged,module: cublas

### 🐛 Describe the bug

python code：
import sys
import os.path
import time

import numpy as np
import random
import torch
from torch import nn
import torchaudio
import pickle
import torch.utils.data
from torch.cuda.amp import GradScaler, autocast
from torch.utils.data import Dataset

from torch.nn.parallel import DataParallel

#torch.set_float32_matmul_precision('high')
DNN_DATA_TYPE = torch.float32
HALF_TRAIN = False
HALF_TEST = False
TRAIN_BATCH_SIZE = 36
TRAIN_THREAD_SIZE = 4
TEST_BATCH_SIZE = 36
TEST_THREAD_SIZE = 4
IS_DATA_COV_IN_CPU = False


def get_device():
    device_type = 'cpu'
    is_cuda = False
    if torch.cuda.is_available():
        device_type = 'cuda'
        torch.backends.cudnn.enabled = True
        torch.backends.cudnn.benchmark = True
        is_cuda = True
        if is_cuda == True:
            print('enable cuda amp test')
    elif torch.has_mps:
        device_type = 'mps'
    elif torch.is_vulkan_available():
        device_type = 'vulkan'
    # device_type = 'cpu'
    if device_type == 'cpu':
        is_cuda = False
        if os.name.lower() == 'nt':
            # 防止没有这个模块
            import importlib
            spam_loader = importlib.util.find_spec('torch_directml')
            found = spam_loader is not None
            if found == True:
                import torch_directml
                print(f'device_type: directml')
                return torch_directml.device(), is_cuda
    print(f'device_type: {device_type}')
    device = torch.device(device_type)
    return device, is_cuda


def read_file_random(path, want_size=48000 * 12):
    data, sr = torchaudio.load(path)
    if sr != 48000:
        resample = torchaudio.transforms.Resample(orig_freq=sr, new_freq=48000)
        data = resample(data)
        sr = 48000
    data_length = len(data[0])
    add_size = int(want_size) - data_length
    if data_length < want_size:
        first_add_len = int(random.randint(0, add_size))
        end_add_len = add_size - first_add_len
        if first_add_len > 0:
            first_add = torch.zeros(1, first_add_len)
            data = torch.concatenate((first_add, data), axis=1)
        if end_add_len > 0:
            end_add = torch.zeros(1, end_add_len)
            data = torch.concatenate((data, end_add), axis=1)
    elif data_length > want_size:
        data = data[:want_size]
        print(f'cut, name:{path}')
    return data.to(DNN_DATA_TYPE)


def read_file_src(path, want_size=48000 * 12):
    data, sr = torchaudio.load(path)
    if sr != 48000:
        resample = torchaudio.transforms.Resample(orig_freq=sr, new_freq=48000)
        data = resample(data)
        sr = 48000
    data_length = len(data[0])
    add_size = int(want_size) - data_length
    if data_length < want_size:
        end_add = torch.zeros(1, add_size)
        data = torch.concatenate((data, end_add), axis=1)
    elif data_length > want_size:
        data = data[:, :want_size]
        print(f'cut, name:{path}')
    '''
    # 删除前0.5s数据
    data = data[:, 24000:]
    befor_add = torch.zeros(1, 24000)
    data = torch.concatenate((befor_add, data), axis=1)
    # 查找起点
    vad = torchaudio.transforms.Vad(sample_rate=48000)
    vad_result = vad(data)
    # 找到第一个语音段的起点位置
    speech_start_index = torch.argmax(vad_result)
    point = speech_start_index.item()
    # 截取掉没有语音的部分
    data = data[:, point:]
    befor_add = torch.zeros(1, point)
    data = torch.concatenate((befor_add, data), axis=1)
    '''
    return data.to(DNN_DATA_TYPE)


def read_file(path, want_size=48000 * 12):
    return read_file_src(path, want_size)


class MyDataSet(Dataset):

    def __init__(self, data_input_list: list, src_file_data: torch.Tensor):
        self.data_list = data_input_list
        self.src_file_data = torch.as_tensor(src_file_data).to(DNN_DATA_TYPE)
        self.src_file_data = (self.src_file_data - torch.min(self.src_file_data)) / torch.std(self.src_file_data)

    def __len__(self):
        return len(self.data_list)

    def __getitem__(self, item):
        file_name, score = self.data_list[item]
        data_in = read_file(file_name)
        data_in = torch.as_tensor(data_in)
        score = torch.as_tensor(score).view(1).to(DNN_DATA_TYPE)
        data_in.to(DNN_DATA_TYPE)
        data_in = (data_in - torch.min(data_in)) / torch.std(data_in)
        return data_in, self.src_file_data, score, file_name


class DataCov(nn.Module):
    def __init__(self):
        super(DataCov, self).__init__()

        self.transform = nn.Sequential(
            torchaudio.transforms.MelSpectrogram(sample_rate=48000, n_fft=1536, hop_length=768, f_min=20, f_max=20000)
        )

    def forward(self, x1, x2):
        # x1 = (x1 - torch.min(x1)) / torch.std(x1)
        # x2 = (x2 - torch.min(x2)) / torch.std(x2)
        return self.transform(x1), self.transform(x2)


class QANet(nn.Module):
    def __init__(self, num_classes=1):
        super(QANet, self).__init__()

        self.geul = nn.GELU(approximate='tanh')

        self.bn_for_mel_pre = nn.Sequential(
            nn.BatchNorm2d(1, eps=1e-05, momentum=0.1)
        )

        self.conv_for_mel = nn.Sequential(
            nn.Conv2d(1, 64, kernel_size=(17, 17), stride=1, padding=1)
        )

        self.conv2_for_mel = nn.Sequential(
            nn.Conv2d(128, 32, kernel_size=(17, 17), stride=11, padding=1)
        )

        self.conv3_for_mel = nn.Sequential(
            nn.Conv2d(32, 8, kernel_size=(3, 3), stride=1, padding=1)
        )

        self.fc_for_mel = nn.Sequential(
            nn.Linear(5280, 1024)
        )

        self.bn_for_src_pre = nn.Sequential(
            nn.BatchNorm1d(1, eps=1e-05, momentum=0.1)
        )

        self.conv_for_src = nn.Sequential(
            nn.Conv1d(1, 64, kernel_size=(27), stride=1, padding=1)
        )

        self.conv2_for_src = nn.Sequential(
            nn.Conv1d(128, 32, kernel_size=(27), stride=19, padding=1)
        )

        self.conv3_for_src = nn.Sequential(
            nn.Conv1d(32, 8, kernel_size=(27), stride=19, padding=1)
        )

        self.fc_for_src = nn.Sequential(
            nn.Linear(12760, 1024)
        )

        self.fc_in = nn.Sequential(
            nn.Linear(2048, 1024)
        )

        self.fc_1 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_2 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_3 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_4 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_5 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_6 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_7 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_8 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_9 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_10 = nn.Sequential(
            nn.Linear(1024, 1024)
        )

        self.fc_out = nn.Sequential(
            nn.Linear(1024, 1)
        )

    def forward(self, x1, x2, x3, x4):
        x1 = self.bn_for_mel_pre(x1)
        x1 = self.conv_for_mel(x1)
        x1 = self.geul(x1)
        x2 = self.bn_for_mel_pre(x2)
        x2 = self.conv_for_mel(x2)
        x2 = self.geul(x2)
        x_1 = torch.cat((x1, x2), dim=1)
        del x1, x2
        x_1 = self.conv2_for_mel(x_1)
        x_1 = self.geul(x_1)
        x_1 = self.conv3_for_mel(x_1)
        x_1 = self.geul(x_1)
        x_1 = x_1.flatten(1)
        x_1 = self.fc_for_mel(x_1)
        x_1 = self.geul(x_1)

        x3 = self.bn_for_src_pre(x3)
        x3 = self.conv_for_src(x3)
        x3 = self.geul(x3)
        x4 = self.bn_for_src_pre(x4)
        x4 = self.conv_for_src(x4)
        x4 = self.geul(x4)
        x_2 = torch.cat((x3, x4), dim=1)
        del x3, x4
        x_2 = self.conv2_for_src(x_2)
        x_2 = self.geul(x_2)
        x_2 = self.conv3_for_src(x_2)
        x_2 = self.geul(x_2)
        x_2 = x_2.flatten(1)
        x_2 = self.fc_for_src(x_2)
        x_2 = self.geul(x_2)
        x = torch.cat((x_1, x_2), dim=1)
        del x_1, x_2

        x = self.fc_in(x)
        x = self.geul(x)
        x = self.fc_1(x)
        x = self.geul(x)
        x = self.fc_2(x)
        x = self.geul(x)
        x = self.fc_3(x)
        x = self.geul(x)
        x = self.fc_4(x)
        x = self.geul(x)
        x = self.fc_5(x)
        x = self.geul(x)
        x = self.fc_6(x)
        x = self.geul(x)
        x = self.fc_7(x)
        x = self.geul(x)
        x = self.fc_8(x)
        x = self.geul(x)
        x = self.fc_9(x)
        x = self.geul(x)
        x = self.fc_10(x)
        x = self.geul(x)

        x = self.fc_out(x)
        return x


def read_train_data(info_file, base_dir):
    with open(info_file, 'rb') as f:
        data = pickle.load(f)
    data_len = len(data)
    ans = []
    for i in range(data_len):
        file, score = data[i]
        if os.name.lower() == 'posix':
            file = file.replace("\\", "/")
        file = os.path.abspath(os.path.join(base_dir, file))
        if os.path.exists(file):
            if os.path.isfile(file):
                if score > 0:
                    ans.append((file, score))
    return ans


def init_module():
    # 若当前 Pytorch 版本以及电脑支持GPU，则使用 GPU 训练，否则使用 CPU
    device, is_cuda = get_device()
    model = QANet().to(DNN_DATA_TYPE)

    # 实例化DNN，并将模型放在 GPU 训练
    if is_cuda:
        model.cuda()
        model = DataParallel(model, device_ids=[0, 1])
        # loss_fn = nn.SmoothL1Loss(reduction='mean').cuda()
        loss_fn = nn.MSELoss(reduction='mean').cuda()
        # loss_fn = nn.BCEloss().cuda()
        print("model , loss on cuda")
    else:
        model = model.to(device)
        # loss_fn = nn.SmoothL1Loss(reduction='mean').to(device)
        loss_fn = nn.MSELoss(reduction='mean').to(device)
        # loss_fn = nn.BCEloss().to(device)

    return model, loss_fn, device, is_cuda


def save_model(model, path):
    # 保存网络
    torch.save(model.state_dict(), path)


def init_module_with_pth(pth):
    # 若当前 Pytorch 版本以及电脑支持GPU，则使用 GPU 训练，否则使用 CPU
    device, is_cuda = get_device()
    state_dict = torch.load(pth, map_location='cpu')
    model = QANet().to(DNN_DATA_TYPE)
    set_compile = True
    if set_compile:
        state_dict_fix = {}
        for k, v in state_dict.items():
            tmp_k = k[k.find(".") + 1:]
            state_dict_fix[tmp_k] = v
        set_compile2 = False
        if set_compile2:
            state_dict_fix2 = {}
            for k, v in state_dict_fix.items():
                tmp_k = k[k.find(".") + 1:]
                state_dict_fix2[tmp_k] = v
            state_dict_fix = state_dict_fix2

        model.load_state_dict(state_dict_fix, strict=False)
    else:
        model.load_state_dict(state_dict, strict=False)
    if is_cuda:
        model = model.to("cuda")
        model = DataParallel(model, device_ids=[0, 1])
        # loss_fn = nn.SmoothL1Loss(reduction='mean').cuda()
        loss_fn = nn.MSELoss(reduction='mean').cuda()
        # loss_fn = nn.BCEloss().cuda()
        print("model , loss on cuda")
    else:
        model = model.to(device)
        # loss_fn = nn.SmoothL1Loss(reduction='mean').to(device)
        loss_fn = nn.MSELoss(reduction='mean').to(device)
        # loss_fn = nn.BCEloss().to(device)
    return model, loss_fn, device, is_cuda


def get_data_loader(src_info_path, src_base_dir, src_wav_path):
    src_wav_data = read_file_src(src_wav_path)
    train_list = read_train_data(src_info_path, src_base_dir)
    random.shuffle(train_list)
    test_list_size = int(len(train_list) * 0.05)
    test_list = train_list[:test_list_size]
    train_list = train_list[test_list_size:]
    train_loader_in = MyDataSet(data_input_list=train_list, src_file_data=src_wav_data)
    train_loader = torch.utils.data.DataLoader(
        dataset=train_loader_in,
        batch_size=TRAIN_BATCH_SIZE,  # 每批提取的数量
        shuffle=True,  # 要不要打乱数据（打乱比较好）
        num_workers=TRAIN_THREAD_SIZE,  # 多少线程来读取数据
        pin_memory=not IS_DATA_COV_IN_CPU
    )
    random.shuffle(test_list)
    test_loader_in = MyDataSet(data_input_list=test_list, src_file_data=src_wav_data)

    test_loader = torch.utils.data.DataLoader(
        dataset=test_loader_in,
        batch_size=TEST_BATCH_SIZE,  # 每批提取的数量
        shuffle=False,  # 要不要打乱数据（打乱比较好）
        num_workers=TEST_THREAD_SIZE,  # 多少线程来读取数据
        pin_memory=not IS_DATA_COV_IN_CPU
    )
    return train_loader, test_loader


def init_random():
    # 设置一个随机种子
    SEED = 1
    # 下面分别对不同模组的随机数进行设定
    np.random.seed(SEED)  # 对numpy模组进行随机数设定
    random.seed(SEED)  # 对venv2模组进行随机数设定，其实就是python中的普通模组
    torch.manual_seed(SEED)  # 对torch中的CPU部分进行随机数设定
    torch.cuda.manual_seed(SEED)  # 对torch中的GPU部分进行随机数设定


def init_train(module, pth_path):
    if module == 'empty_train' or not os.path.exists(pth_path):
        model, loss_fn, device, is_cuda = init_module()
    else:
        model, loss_fn, device, is_cuda = init_module_with_pth(pth_path)
    return model, loss_fn, device, is_cuda


def fund_max_index(input):
    outsize = len(input)
    ans = torch.zeros(outsize)
    for i in range(outsize):
        loopsize = len(input[i])
        max_index = 0
        max_value = -99999999999999
        for j in range(loopsize):
            if input[i][j] >= max_value:
                max_value = input[i][j]
                max_index = j
        ans[i] = max_index
    return ans


def get_eq_number(vector1, vector2):
    loop = len(vector1)
    eq_number = 0
    for i in range(loop):
        if vector1[i] == vector2[i]:
            eq_number += 1
    return eq_number


def train(pth_path, module, learn_rate, train_loader, test_loader, train_time):
    global scheduler, scaler, optimizer
    model, loss_fn, device, is_cuda = init_train(module, pth_path)
    num_batches = len(train_loader)
    allTime = num_batches * train_time
    test_list_size = len(test_loader)
    if is_cuda and HALF_TRAIN:
        scaler = GradScaler()
    all_diff_pr = test_list_size * 10.0 * 10000
    all_all_loss = test_list_size * 10.0 * 10000
    if IS_DATA_COV_IN_CPU:
        data_prepare = DataCov().to(DNN_DATA_TYPE).to('cpu')
    else:
        if is_cuda:
            data_prepare = DataCov().to(DNN_DATA_TYPE).to("cuda")
            data_prepare = DataParallel(data_prepare, device_ids=[0, 1])
        else:
            data_prepare = DataCov().to(DNN_DATA_TYPE).to(device)
    time_without_save = 0
    # optimizer = torch.optim.Adam(model.parameters(), lr=learn_rate)
    optimizer = torch.optim.SGD(model.parameters(), lr=learn_rate)

    for i in range(train_time):
        model.train()  # 设置网络为训练模式
        # cur_lr = optimizer.param_groups[-1]['lr']
        if i != 0 and i % 50 == 0:
            learn_rate = learn_rate * 0.5
            # optimizer = torch.optim.Adam(model.parameters(), lr=learn_rate)
            optimizer = torch.optim.SGD(model.parameters(), lr=learn_rate)
        print(f'learn_rate {learn_rate}, loop {i}, all {train_time}')
        print(f'Begin train' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
        for step1, (wav_data, src_wav_data, labels, file_name) in enumerate(train_loader):
            # with autograd.detect_anomaly():
            #    lost, output, labels = train(data, optimizer, model, loss_fn, device)
            if IS_DATA_COV_IN_CPU:
                wav_data_prepare, src_wav_data_prepare = data_prepare(wav_data.to('cpu'), src_wav_data.to('cpu'))
                wav_data = wav_data.to(device)
                src_wav_data = src_wav_data.to(device)
                wav_data_prepare = wav_data_prepare.to(device)
                src_wav_data_prepare = src_wav_data_prepare.to(device)
            else:
                wav_data_prepare, src_wav_data_prepare = data_prepare(wav_data.to(device), src_wav_data.to(device))
                wav_data = wav_data.to(device)
                src_wav_data = src_wav_data.to(device)
                wav_data_prepare = wav_data_prepare.to(device)
                src_wav_data_prepare = src_wav_data_prepare.to(device)
            optimizer.zero_grad(set_to_none=True)

            if is_cuda and HALF_TRAIN:
                with autocast():
                    output = model(wav_data_prepare, src_wav_data_prepare, wav_data, src_wav_data)
                    loss = loss_fn(output, labels.to(device)).to(device)
                    # loss.requires_grad_(True)
                    scaler.scale(loss).backward()  # 为了梯度放大
                    scaler.step(optimizer)
                    scaler.update()  # 准备着，看是否要增大scaler
            else:
                output = model(wav_data_prepare, src_wav_data_prepare, wav_data, src_wav_data)
                loss = loss_fn(output, labels.to(device)).to(device)

                if torch.any(torch.isnan(loss)):
                    print(f'{file_name}')
                    return
                loss.backward()
                optimizer.step()

            if step1 % 500 == 0:
                loss_in = float(loss.item())
                print(
                    f"current batch:{step1 + i * num_batches}/all:{allTime}, step: {step1},loss: {loss_in: .6f}")

        # scheduler.step()
        print(f'After train and begin test' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
        model.eval()
        all_loss = 0
        max_diff = -999.0
        diff_pr = 0.0

        with torch.no_grad():
            for test_step, (test_wav_data, src_wav_data, labels, _) in enumerate(test_loader):
                if IS_DATA_COV_IN_CPU:
                    test_wav_data_prepare, src_wav_data_prepare = data_prepare(test_wav_data.to('cpu'), src_wav_data.to('cpu'))
                    test_wav_data = test_wav_data.to(device)
                    src_wav_data = src_wav_data.to(device)
                    test_wav_data_prepare = test_wav_data_prepare.to(device)
                    src_wav_data_prepare = src_wav_data_prepare.to(device)
                else:
                    test_wav_data_prepare, src_wav_data_prepare = data_prepare(test_wav_data.to(device), src_wav_data.to(device))
                    test_wav_data = test_wav_data.to(device)
                    src_wav_data = src_wav_data.to(device)
                    test_wav_data_prepare = test_wav_data_prepare.to(device)
                    src_wav_data_prepare = src_wav_data_prepare.to(device)
                if is_cuda and HALF_TEST:
                    with autocast():
                        output = model(test_wav_data_prepare, src_wav_data_prepare, test_wav_data.to(device), src_wav_data.to(device))
                        loss = loss_fn(output, labels.to(device))
                else:
                    output = model(test_wav_data_prepare, src_wav_data_prepare, test_wav_data.to(device), src_wav_data.to(device))
                    loss = loss_fn(output, labels.to(device))
                out = output.to('cpu')
                labels = labels.to('cpu')
                diff_pr = diff_pr + float(torch.sum(torch.abs((out - labels) / labels)))
                diff = float(torch.max(torch.abs(out - labels)))
                if max_diff < diff:
                    max_diff = diff
                all_loss = all_loss + loss
        print(
            f'all_loss: {all_loss}, max_diff: {max_diff}, diff_p : {diff_pr}, av_diff_p : {(diff_pr / (test_list_size * TEST_BATCH_SIZE))}, min_diff_pr: {all_diff_pr}, min_lost: {all_all_loss}')
        if all_diff_pr > diff_pr:
            all_diff_pr = diff_pr
            all_all_loss = all_loss
            print(f'save module')
            save_model(model, pth_path)
            time_without_save = 0
        else:
            if all_all_loss > all_loss:
                all_all_loss = all_loss
                print(f'save module')
                save_model(model, pth_path)
                time_without_save = 0
            else:
                time_without_save = time_without_save + 1
                if time_without_save > 3:
                    print(f'save module')
                    save_model(model, pth_path)
                    time_without_save = 0
        print(f'After test' + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
        if is_cuda:
            torch.cuda.empty_cache()
    save_model(model, pth_path)


def train_data(src_info_path, src_base_dir, src_wav_path, pth_path, module='empty', train_time=50000):
    init_random()

    learn_rate = 4e-3
    train_loader, test_loader = get_data_loader(src_info_path, src_base_dir, src_wav_path)
    print(
        f'len train: {len(train_loader)}, len test:{len(test_loader)}')
    train(pth_path, module, learn_rate, train_loader, test_loader, train_time)


if __name__ == '__main__':
    is_debug = True if sys.gettrace() else False
    can_run = False
    voice_info = "d:/tmp/data2/fun3.info"
    voice_dir = "d:/tmp/data2/datasrc"
    src_file = "d:/tmp/data2/sample48k.wav"
    pth_path = "d:/tmp/data2/fun3.pth"
    module = "pre_load_train"
    if is_debug:
        can_run = True
    else:
        argc = len(sys.argv)
        if (argc >= 5):
            voice_info = sys.argv[1]
            voice_dir = sys.argv[2]
            src_file = sys.argv[3]
            pth_path = sys.argv[4]
            can_run = True
    if can_run:
        voice_info = os.path.abspath(voice_info)
        voice_dir = os.path.abspath(voice_dir)
        src_file = os.path.abspath(src_file)
        pth_path = os.path.abspath(pth_path)
        train_data(voice_info, voice_dir, src_file, pth_path, module)
    else:
        print("can't run need more args, use sample:")
        print("python3 fun3_dnn.py voice_info voice_dir src_file pth_path")

error：
Traceback (most recent call last):
  File "/root/autodl-tmp/fun3_cnn.py", line 625, in <module>
    train_data(voice_info, voice_dir, src_file, pth_path, module)
  File "/root/autodl-tmp/fun3_cnn.py", line 599, in train_data
    train(pth_path, module, learn_rate, train_loader, test_loader, train_time)
  File "/root/autodl-tmp/fun3_cnn.py", line 517, in train
    output = model(wav_data_prepare, src_wav_data_prepare, wav_data, src_wav_data)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/parallel/data_parallel.py", line 171, in forward
    outputs = self.parallel_apply(replicas, inputs, kwargs)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/parallel/data_parallel.py", line 181, in parallel_apply
    return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py", line 89, in parallel_apply
    output.reraise()
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/_utils.py", line 644, in reraise
    raise exception
RuntimeError: Caught RuntimeError in replica 1 on device 1.
Original Traceback (most recent call last):
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/parallel/parallel_apply.py", line 64, in _worker
    output = module(*input, **kwargs)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/autodl-tmp/fun3_cnn.py", line 261, in forward
    x_1 = self.fc_for_mel(x_1)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/modules/container.py", line 217, in forward
    input = module(input)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/root/miniconda3/envs/ai/lib/python3.10/site-packages/torch/nn/modules/linear.py", line 114, in forward
    return F.linear(input, self.weight, self.bias)
RuntimeError: CUDA error: CUBLAS_STATUS_NOT_SUPPORTED when calling cublasLtMatmul with transpose_mat1 1 transpose_mat2 0 m 1024 n 12 k 5280 mat1_ld 5280 mat2_ld 5280 result_ld 1024 abcType 0 computeType 77 scaleType 0

### Versions

Collecting environment information...
PyTorch version: 2.0.1
Is debug build: False
CUDA used to build PyTorch: 11.8
ROCM used to build PyTorch: N/A

OS: Ubuntu 22.04.2 LTS (x86_64)
GCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0
Clang version: Could not collect
CMake version: version 3.22.1
Libc version: glibc-2.35

Python version: 3.10.12 (main, Jul  5 2023, 18:54:27) [GCC 11.2.0] (64-bit runtime)
Python platform: Linux-5.4.0-146-generic-x86_64-with-glibc2.35
Is CUDA available: True
CUDA runtime version: 11.8.89
CUDA_MODULE_LOADING set to: LAZY
GPU models and configuration:
GPU 0: NVIDIA GeForce RTX 4090
GPU 1: NVIDIA GeForce RTX 4090

Nvidia driver version: 525.105.17
cuDNN version: Probably one of the following:
/usr/lib/x86_64-linux-gnu/libcudnn.so.8.6.0
/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.6.0
/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.6.0
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.6.0
/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.6.0
/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.6.0
/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.6.0
HIP runtime version: N/A
MIOpen runtime version: N/A
Is XNNPACK available: True

CPU:
Architecture:                    x86_64
CPU op-mode(s):                  32-bit, 64-bit
Address sizes:                   46 bits physical, 57 bits virtual
Byte Order:                      Little Endian
CPU(s):                          128
On-line CPU(s) list:             0-127
Vendor ID:                       GenuineIntel
Model name:                      Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz
CPU family:                      6
Model:                           106
Thread(s) per core:              2
Core(s) per socket:              32
Socket(s):                       2
Stepping:                        6
Frequency boost:                 enabled
CPU max MHz:                     2901.0000
CPU min MHz:                     800.0000
BogoMIPS:                        5800.00
Flags:                           fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 invpcid_single ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow vnmi flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local wbnoinvd dtherm ida arat pln pts avx512vbmi umip pku ospke avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear pconfig flush_l1d arch_capabilities
Virtualization:                  VT-x
L1d cache:                       3 MiB (64 instances)
L1i cache:                       2 MiB (64 instances)
L2 cache:                        80 MiB (64 instances)
L3 cache:                        108 MiB (2 instances)
NUMA node(s):                    2
NUMA node0 CPU(s):               0-31,64-95
NUMA node1 CPU(s):               32-63,96-127
Vulnerability Itlb multihit:     Not affected
Vulnerability L1tf:              Not affected
Vulnerability Mds:               Not affected
Vulnerability Meltdown:          Not affected
Vulnerability Mmio stale data:   Mitigation; Clear CPU buffers; SMT vulnerable
Vulnerability Retbleed:          Not affected
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp
Vulnerability Spectre v1:        Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2:        Mitigation; Enhanced IBRS, IBPB conditional, RSB filling, PBRSB-eIBRS SW sequence
Vulnerability Srbds:             Not affected
Vulnerability Tsx async abort:   Not affected

Versions of relevant libraries:
[pip3] numpy==1.25.2
[pip3] torch==2.0.1
[pip3] torchaudio==2.0.2
[pip3] torchvision==0.15.2
[pip3] triton==2.0.0
[conda] blas                      1.0                         mkl    https://mirrors.ustc.edu.cn/anaconda/pkgs/main
[conda] ffmpeg                    4.3                  hf484d3e_0    pytorch
[conda] mkl                       2023.1.0         h213fc3f_46343    https://mirrors.ustc.edu.cn/anaconda/pkgs/main
[conda] mkl-service               2.4.0           py310h5eee18b_1    https://mirrors.ustc.edu.cn/anaconda/pkgs/main
[conda] mkl_fft                   1.3.6           py310h1128e8f_1    https://mirrors.ustc.edu.cn/anaconda/pkgs/main
[conda] mkl_random                1.2.2           py310h1128e8f_1    https://mirrors.ustc.edu.cn/anaconda/pkgs/main
[conda] numpy                     1.25.2          py310h5f9d8c6_0    https://mirrors.ustc.edu.cn/anaconda/pkgs/main
[conda] numpy-base                1.25.2          py310hb5e798b_0    https://mirrors.ustc.edu.cn/anaconda/pkgs/main
[conda] pytorch                   2.0.1           py3.10_cuda11.8_cudnn8.7.0_0    pytorch
[conda] pytorch-cuda              11.8                 h7e8668a_5    pytorch
[conda] pytorch-mutex             1.0                        cuda    pytorch
[conda] torchaudio                2.0.2               py310_cu118    pytorch
[conda] torchtriton               2.0.0                     py310    pytorch
[conda] torchvision               0.15.2              py310_cu118    pytorch

cc @ptrblck @csarofeen @xwang233