Where can I find cross_entropy_loss implementation

username: hwijeen
cooked: <p>Hi, I would like to see the implementation of cross entropy loss. So far, I learned that, torch.nn.funcional.py calls torch._C.nn.cross_entropy_loss but I am having trouble finding the C implementation.</p>
<p>From the releated issue(<a href="https://discuss.pytorch.org/t/where-does-torch-c-come-from/2015" class="inline-onebox">Where does `torch._C` come from?</a>), I realize that some functions are implemented in C and torch._C is the implementation. The Issue contains links to the original implementations, but it from the outdated pytorch code.</p>
<ul>
<li>Can someone point to the exact location of cross entropy loss implementation (both CPU and GPU)?</li>
<li>If possible, can someone kindly explain how one can track down to such point from python package(e.g. from torch.nn.functional.py to the C implementation, where the C code is binded with python, etc…)</li>
</ul>
<p>Many thanks!</p>

username: eqy
cooked: <p>Cross-entropy loss is implemented with a combination of existing operations, such as logsoftmax and nll_loss:</p><aside class="onebox githubblob" data-onebox-src="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L568">
  <header class="source">

      <a href="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L568" target="_blank" rel="noopener nofollow ugc">github.com</a>
  </header>

  <article class="onebox-body">
    <h4><a href="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L568" target="_blank" rel="noopener nofollow ugc">pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L568</a></h4>



    <pre class="onebox">      <code class="lang-cpp">
        <ol class="start lines" start="558" style="counter-reset: li-counter 557 ;">
            <li>        break;</li>
            <li>      case Reduction::None:</li>
            <li>        ret = smooth_loss;</li>
            <li>        break;</li>
            <li>      default:</li>
            <li>        TORCH_CHECK(false, "Invalid reduction type encountered in cross_entropy: ", reduction);</li>
            <li>    }</li>
            <li>    return (1 - label_smoothing) * nllloss + ret * (label_smoothing / n_classes);</li>
            <li>}</li>
            <li>
            </li>
<li class="selected">Tensor cross_entropy_loss(</li>
            <li>    const Tensor&amp; self,</li>
            <li>    const Tensor&amp; target,</li>
            <li>    const c10::optional&lt;Tensor&gt;&amp; weight,</li>
            <li>    int64_t reduction,</li>
            <li>    int64_t ignore_index,</li>
            <li>    double label_smoothing) {</li>
            <li>  Tensor ret;</li>
            <li>  if (self.sizes() == target.sizes()) {</li>
            <li>    // Assume soft targets when input and target shapes are the same</li>
            <li>    TORCH_CHECK(at::isFloatingType(target.scalar_type()),</li>
        </ol>
      </code>
    </pre>



  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>

<p>These individual kernels will have CPU and GPU implementations in ATen, e.g.,<br>
CPU:</p><aside class="onebox githubblob" data-onebox-src="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L274">
  <header class="source">

      <a href="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L274" target="_blank" rel="noopener nofollow ugc">github.com</a>
  </header>

  <article class="onebox-body">
    <h4><a href="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L274" target="_blank" rel="noopener nofollow ugc">pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L274</a></h4>



    <pre class="onebox">      <code class="lang-cpp">
        <ol class="start lines" start="264" style="counter-reset: li-counter 263 ;">
            <li>
            </li>
<li>  if (reduction == Reduction::Mean) {</li>
            <li>    output_val /= total_weight_val;</li>
            <li>  }</li>
            <li>
            </li>
<li>  // write result to output tensors</li>
            <li>  *output.data_ptr&lt;scalar_t&gt;() = output_val;</li>
            <li>  *total_weight_data = total_weight_val;</li>
            <li>}</li>
            <li>
            </li>
<li class="selected">void nll_loss_forward_out_cpu_template(</li>
            <li>    const Tensor&amp; output,</li>
            <li>    const Tensor&amp; total_weight,</li>
            <li>    const Tensor&amp; input,</li>
            <li>    const Tensor&amp; target,</li>
            <li>    const Tensor&amp; weight,</li>
            <li>    int64_t reduction,</li>
            <li>    int64_t ignore_index) {</li>
            <li>  AT_DISPATCH_FLOATING_TYPES_AND(</li>
            <li>      ScalarType::BFloat16, input.scalar_type(), "nll_loss_out_frame", [&amp;] {</li>
            <li>        if (target.scalar_type() == kByte) {</li>
        </ol>
      </code>
    </pre>



  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>
<aside class="onebox githubblob" data-onebox-src="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L133">
  <header class="source">

      <a href="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L133" target="_blank" rel="noopener nofollow ugc">github.com</a>
  </header>

  <article class="onebox-body">
    <h4><a href="https://github.com/pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L133" target="_blank" rel="noopener nofollow ugc">pytorch/pytorch/blob/2981534f54d49fa3a9755c9b0855e7929c2527f0/aten/src/ATen/native/LossNLL.cpp#L133</a></h4>



    <pre class="onebox">      <code class="lang-cpp">
        <ol class="start lines" start="123" style="counter-reset: li-counter 122 ;">
            <li>}</li>
            <li>
            </li>
<li>// Returns the address of the first element of a tensor</li>
            <li>// or nullptr if the tensor is undefined.</li>
            <li>template &lt;typename scalar_t&gt;</li>
            <li>inline scalar_t* optional_data(const Tensor&amp; source) {</li>
            <li>  return source.defined() ? source.data_ptr&lt;scalar_t&gt;() : nullptr;</li>
            <li>}</li>
            <li>
            </li>
<li>template &lt;typename scalar_t, typename target_t&gt;</li>
            <li class="selected">static void nll_loss_out_frame(</li>
            <li>    const Tensor&amp; output,</li>
            <li>    const Tensor&amp; total_weight,</li>
            <li>    const Tensor&amp; input,</li>
            <li>    const Tensor&amp; target,</li>
            <li>    const Tensor&amp; weight,</li>
            <li>    int64_t reduction,</li>
            <li>    int64_t ignore_index) {</li>
            <li>  const auto n_dims = input.dim();</li>
            <li>  const auto n_classes = input.size(-1);</li>
            <li>
        </li>
</ol>
      </code>
    </pre>



  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>
<aside class="onebox githubblob" data-onebox-src="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/SoftMax.cpp#L129">
  <header class="source">

      <a href="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/SoftMax.cpp#L129" target="_blank" rel="noopener nofollow ugc">github.com</a>
  </header>

  <article class="onebox-body">
    <h4><a href="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/SoftMax.cpp#L129" target="_blank" rel="noopener nofollow ugc">pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/SoftMax.cpp#L129</a></h4>



    <pre class="onebox">      <code class="lang-cpp">
        <ol class="start lines" start="119" style="counter-reset: li-counter 118 ;">
            <li>      "dim must be non-negative and less than input dimensions");</li>
            <li>
            </li>
<li>  set_output(grad.sizes(), grad_input_options);</li>
            <li>}</li>
            <li>}</li>
            <li>
            </li>
<li>namespace native {</li>
            <li>namespace {</li>
            <li>
            </li>
<li>template &lt;typename scalar_t, bool LogSoftMax, bool MaskedSoftMax = false&gt;</li>
            <li class="selected">void host_softmax(</li>
            <li>    Tensor output,</li>
            <li>    const Tensor&amp; input,</li>
            <li>    const int64_t dim,</li>
            <li>    bool* mask = nullptr) {</li>
            <li>  int64_t outer_size = 1;</li>
            <li>  int64_t dim_size = input.size(dim);</li>
            <li>  int64_t inner_size = 1;</li>
            <li>  for (const auto i : c10::irange(dim)) {</li>
            <li>    outer_size *= input.size(i);</li>
            <li>  }</li>
        </ol>
      </code>
    </pre>



  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>

<p>GPU:</p><aside class="onebox githubblob" data-onebox-src="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/Loss.cu#L180">
  <header class="source">

      <a href="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/Loss.cu#L180" target="_blank" rel="noopener nofollow ugc">github.com</a>
  </header>

  <article class="onebox-body">
    <h4><a href="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/Loss.cu#L180" target="_blank" rel="noopener nofollow ugc">pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/Loss.cu#L180</a></h4>



    <pre class="onebox">      <code class="lang-cu">
        <ol class="start lines" start="170" style="counter-reset: li-counter 169 ;">
            <li>    RECORD_KERNEL_FUNCTION_DTYPE(NAME, _it)                                 \</li>
            <li>    switch (_it) {                                                          \</li>
            <li>      AT_PRIVATE_CASE_TYPE_USING_HINT(NAME, at::ScalarType::Byte, uint8_t, index_t, __VA_ARGS__) \</li>
            <li>      AT_PRIVATE_CASE_TYPE_USING_HINT(NAME, at::ScalarType::Long, int64_t, index_t, __VA_ARGS__)\</li>
            <li>      default:                                                              \</li>
            <li>        AT_ERROR(#NAME, " not implemented for '", toString(_it), "'");      \</li>
            <li>    }                                                                       \</li>
            <li>  }()</li>
            <li>
            </li>
<li>template &lt;typename scalar_t, typename index_t&gt;</li>
            <li class="selected">__global__ void nll_loss_forward_no_reduce_cuda_kernel(</li>
            <li>    int64_t batch_size,</li>
            <li>    PackedTensorAccessor64&lt;scalar_t, 2&gt; input,</li>
            <li>    index_t* target,</li>
            <li>    scalar_t* output,</li>
            <li>    scalar_t* weights,</li>
            <li>    int n_classes,</li>
            <li>    int ignore_index) {</li>
            <li>  CUDA_KERNEL_LOOP(index, batch_size) {</li>
            <li>    int cur_target = target[index];</li>
            <li>    if (cur_target == ignore_index) {</li>
        </ol>
      </code>
    </pre>



  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>
<aside class="onebox githubblob" data-onebox-src="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/SoftMax.cu#L683">
  <header class="source">

      <a href="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/SoftMax.cu#L683" target="_blank" rel="noopener nofollow ugc">github.com</a>
  </header>

  <article class="onebox-body">
    <h4><a href="https://github.com/pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/SoftMax.cu#L683" target="_blank" rel="noopener nofollow ugc">pytorch/pytorch/blob/aab67c6dff3819d3c05b328e964f1642227adbe6/aten/src/ATen/native/cuda/SoftMax.cu#L683</a></h4>



    <pre class="onebox">      <code class="lang-cu">
        <ol class="start lines" start="673" style="counter-reset: li-counter 672 ;">
            <li>  Epilogue&lt;scalar_t, accscalar_t, outscalar_t&gt; epilogue(sum_k);</li>
            <li>
            </li>
<li>  if (shift == output_shift &amp;&amp; shift == grad_output_shift) {</li>
            <li>    WriteBpropResultsVectorized&lt;ILP, scalar_t, accscalar_t, outscalar_t, Epilogue&gt;(classes, shift, gradInput, output, gradOutput, epilogue);</li>
            <li>  } else {</li>
            <li>    WriteBpropResults&lt;ILP, scalar_t, accscalar_t, outscalar_t, Epilogue&gt;(classes, gradInput, output, gradOutput, epilogue);</li>
            <li>  }</li>
            <li>}</li>
            <li>
            </li>
<li>template&lt;template&lt;typename, typename, typename&gt; class Epilogue, bool is_log_softmax&gt;</li>
            <li class="selected">Tensor host_softmax(const Tensor &amp; input_, const int64_t dim_, const bool half_to_float, const Tensor&amp; output){</li>
            <li>  if (half_to_float) {</li>
            <li>    TORCH_CHECK(input_.scalar_type() == ScalarType::Half, "conversion is supported for Half type only");</li>
            <li>  }</li>
            <li>  auto input = input_.contiguous();</li>
            <li>  static_assert(std::is_same&lt;acc_type&lt;at::Half, true&gt;, float&gt;::value, "accscalar_t for half should be float");</li>
            <li>  if (input.dim() == 0) input = input.view(1);</li>
            <li>  int64_t dim = maybe_wrap_dim(dim_, input.dim());</li>
            <li>  TORCH_CHECK(dim &gt;=0 &amp;&amp; dim &lt; input.dim(), "dim must be non-negative and less than input dimensions");</li>
            <li>  int64_t outer_size = 1;</li>
            <li>  int64_t dim_size = input.size(dim);</li>
        </ol>
      </code>
    </pre>



  </article>

  <div class="onebox-metadata">
    
    
  </div>

  <div style="clear: both"></div>
</aside>