pirroh commited on
Commit
1c6453a
1 Parent(s): 3f27e64

Delete low_precision_layernorm.py

Browse files
Files changed (1) hide show
  1. low_precision_layernorm.py +0 -35
low_precision_layernorm.py DELETED
@@ -1,35 +0,0 @@
1
- import torch
2
- import torch.nn.functional as F
3
-
4
-
5
- class LPLayerNorm(torch.nn.LayerNorm):
6
- def __init__(self, normalized_shape, eps=1e-05, elementwise_affine=True, device=None, dtype=None):
7
- super().__init__(
8
- normalized_shape=normalized_shape,
9
- eps=eps,
10
- elementwise_affine=elementwise_affine,
11
- device=device,
12
- dtype=dtype,
13
- )
14
-
15
- def forward(self, x):
16
- module_device = x.device
17
- downcast_x = _cast_if_autocast_enabled(x)
18
- downcast_weight = _cast_if_autocast_enabled(
19
- self.weight) if self.weight is not None else self.weight
20
- downcast_bias = _cast_if_autocast_enabled(
21
- self.bias) if self.bias is not None else self.bias
22
- with torch.autocast(enabled=False, device_type=module_device.type):
23
- return F.layer_norm(downcast_x, self.normalized_shape, downcast_weight, downcast_bias, self.eps)
24
-
25
-
26
- def _cast_if_autocast_enabled(tensor):
27
- if torch.is_autocast_enabled():
28
- if tensor.device.type == 'cuda':
29
- dtype = torch.get_autocast_gpu_dtype()
30
- elif tensor.device.type == 'cpu':
31
- dtype = torch.get_autocast_cpu_dtype()
32
- else:
33
- raise NotImplementedError()
34
- return tensor.to(dtype=dtype)
35
- return tensor