How to plot auc when my classes are one hot vectoe

Saida2020: <p>I have binary classification problem, My input is vector of 561 and out put is two classes are negative class is [1,0]  and positive class is [0,1], I trained model that is return two numbers and I am trying to plot auc and find the best threshold , when I tries to use the probabilities in calculating roc curve i got 50% and when I replace it with predictions 0 or 1 i get 70 % and I do not know why that big difference. I do not know if I am calculating it wrong since I get two probability and I get the index of the largest one. I do not know if I am getting that difference because weighted random samples since I have unbalanced data. I would like to know how</p>
<p>Here is the training function</p>
<pre><code class="lang-auto">def run_gradient_descent(model, data_train, data_val, batch_size, learning_rate, weight_decay=0, num_epochs=20):
    
    model = model.to(device)
    #criterion = nn.CrossEntropyLoss()
    criterion = nn.MSELoss()
    #criterion = nn.BCEWithLogitsLoss()
    optimizer = optim.SGD(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
    iters, losses, train_losses, test_losses = [], [], [], []
    iters_sub, train_acc, val_acc = [], [] ,[]
    print(batch_size)
    
    # weight sampler
    class0, class1 =labels_count(data_train)
    dataset_counts = [class0, class1]
    print(dataset_counts)
    num_samples = sum(dataset_counts)
    labels = [tag for _, tag in data_train]
    class_weights = [1./dataset_counts[i] for i in range(len(dataset_counts))]
    labels_indics = [i.index(max(i)) for i in labels ]
    weights = [class_weights[i] for i in labels_indics] # labels.max(1, keepdim=True)[1]
    weights = numpy.array(weights)
    samples_weight = torch.from_numpy(weights)
    samples_weigth = samples_weight.double()
    sampler = torch.utils.data.sampler.WeightedRandomSampler(samples_weight, int(num_samples), replacement=True)
    
    
    train_loader = torch.utils.data.DataLoader(
        data_train,
        batch_size=batch_size,
        shuffle=False,
        sampler = sampler,
        collate_fn=lambda d: ([x[0] for x in d], [x[1] for x in d]),
        num_workers=os.cpu_count()//2
    )

    # training
    n = 0 # the number of iterations
    for epoch in tqdm(range(num_epochs), desc="epoch"):
        correct = 0
        total = 0
        for xs, ts in tqdm(train_loader, desc="train"):
            xs = torch.FloatTensor(xs).to(device)
            ts = torch.FloatTensor(ts).to(device)
            # print("batch index {}, 0/1: {}/{}".format(n,ts.tolist().count([1,0]),ts.tolist().count([0,1])))

            # if len(ts) != batch_size:
            #     print("ops")
            #     continue
            model.train()
            
            zs = model(xs)
            zs = zs.to(device)
            loss = criterion(zs, ts)
            
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)


            iters.append(n)
            loss.detach().cpu()
            losses.append(float(loss)/len(ts)) # compute *average* loss
            
            pred = zs.max(1, keepdim=True)[1] # get the index of the max logit
            target = ts.max(1, keepdim=True)[1]
            correct += pred.eq(target).sum().item()
            total += int(ts.shape[0])
            acc = correct / total

            if (n % len(train_loader) == 0) and n&gt;0 and epoch%2==0:
                test_acc, test_loss = get_accuracy(model, data_val)
            
                iters_sub.append(n)
                train_acc.append(acc)
                val_acc.append(test_acc)
        
                train_losses.append(sum(losses)/len(losses))
                test_losses.append(test_loss)
                
                print("Epoch", epoch, "train_acc", acc)
                print("Epoch", epoch, "test_acc", test_acc)
                print("Epoch", epoch, "train_loss", sum(losses)/len(losses))
                print("Epoch", epoch, "test_loss", test_loss)

             # increment the iteration number
            n += 1
        torch.save(model.state_dict(), f"{MODEL_NAME}/checkpoint_epoch{epoch}.pt")


    # plotting
    plt.title("Training Curve (batch_size={}, lr={})".format(batch_size, learning_rate))
    plt.plot(iters_sub, train_losses, label="Train")
    plt.legend(loc='best')
    plt.xlabel("Iterations")
    plt.ylabel("Loss")
    plt.savefig(f"{MODEL_NAME}/training_loss.png")
    
    #
    plt.clf()
    plt.title("Validation Curve (batch_size={}, lr={})".format(batch_size, learning_rate))
    plt.plot(iters_sub, test_losses, label="Test")
    plt.xlabel("Iterations")
    plt.ylabel("Loss")
    plt.savefig(f"{MODEL_NAME}/test_loss.png")
    
    #plt.show()
    plt.clf()
    plt.title("Training Curve (batch_size={}, lr={})".format(batch_size, learning_rate))
    plt.plot(iters_sub, train_acc, label="Train")
    plt.plot(iters_sub, val_acc, label="Test")
    plt.xlabel("Iterations")
    plt.ylabel("Accuracy")
    plt.legend(loc='best')
    plt.savefig(f"{MODEL_NAME}/training_acc.png")
    #plt.show()
    return model
</code></pre>
<p>HEre is how I get the prediction and calculate roc</p>
<pre><code class="lang-auto">def get_accuracy(model, data):
    loader = torch.utils.data.DataLoader(
        data,
        batch_size=64,
        collate_fn=lambda d: ([x[0] for x in d], [x[1] for x in d]),
        num_workers=os.cpu_count() // 2
    )
    preds_prob = []
    targets = []
    preds = []
    model.eval()
    with torch.no_grad():
        for xs, ts in tqdm(loader, desc="eval"):
            xs = torch.FloatTensor(xs).to(device)
            ts = torch.FloatTensor(ts).to(device)
            zs = model(xs)
            preds_prob.append([max(i) for i in zs.tolist()])
            preds.append(flatten_list(zs.max(1, keepdim=True)[1].tolist())) # get the index of the max logit
            targets.append(flatten_list(ts.max(1, keepdim=True)[1].tolist()))
    return preds, preds_prob, targets
</code></pre>
<pre><code class="lang-auto">preds, preds_prob, targets= get_accuracy(model, test_set)
predictions = flatten_list(preds)
target = flatten_list(targets)
predictions_prob= flatten_list(preds_prob)

</code></pre>
<p>when I use the predictions only to calculaute auc</p>
<pre><code class="lang-auto">fpr, tpr, thresholds = roc_curve(target, predictions)
</code></pre>
<p>I get 70%</p>
<p>but if I use</p>
<pre><code class="lang-auto">fpr, tpr, thresholds = roc_curve(target, predictions_prob)
</code></pre>
<p>I get around 51%, also even if I try to fin the optimal threshold it did not change much.</p>
<p>I would like to know if i get prediction probability in wrong way since y classes are [0,1] and [1,0]?</p>

Saida2020: <p>any one can help ???</p>