File size: 1,228 Bytes
2e4e201 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
"""
Applies the mish function element-wise:
mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))
"""
# import pytorch
import torch
import torch.nn.functional as F
from torch import nn
@torch.jit.script
def mish(input):
"""
Applies the mish function element-wise:
mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))
See additional documentation for mish class.
"""
return input * torch.tanh(F.softplus(input))
class Mish(nn.Module):
"""
Applies the mish function element-wise:
mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))
Shape:
- Input: (N, *) where * means, any number of additional
dimensions
- Output: (N, *), same shape as the input
Examples:
>>> m = Mish()
>>> input = torch.randn(2)
>>> output = m(input)
Reference: https://pytorch.org/docs/stable/generated/torch.nn.Mish.html
"""
def __init__(self):
"""
Init method.
"""
super().__init__()
def forward(self, input):
"""
Forward pass of the function.
"""
if torch.__version__ >= "1.9":
return F.mish(input)
else:
return mish(input) |