Hugging Face's logo

xverse
/
XVERSE-7B-Chat-GPTQ-Int4

Text Generation
Transformers
xverse
custom_code
4-bit precision
gptq
Model card Files Community
XVERSE-7B-Chat-GPTQ-Int4
File size: 10,391 Bytes 
6bf9b72
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
import bz2
import torch
import base64
import ctypes
from transformers.utils import logging
from typing import List

logger = logging.get_logger(__name__)

try:
    from cpm_kernels.kernels.base import LazyKernelCModule, KernelFunction, round_up

    class Kernel:
        def __init__(self, code: bytes, function_names: List[str]):
            self.code = code
            self._function_names = function_names
            self._cmodule = LazyKernelCModule(self.code)

            for name in self._function_names:
                setattr(self, name, KernelFunction(self._cmodule, name))

    quantization_code = "QlpoOTFBWSZTWapgbn4ALTZ/////////9f/n9+/r/v//3/Tt7cDwfe5sdXXdZNR/9++P4BkfAfIVSVACQUFCEIJQoJUqkgAACJRSqAAAQgEqKqqoKoAAVINADQDTCMhoAGCZBpoNMg0Bk0AGRiGmQBghkNGmgAaGgGhkyAAADJoMhBoAaAaYRkNAAwTINNBpkGgMmgAyMQ0yAMEMho00ADQ0A0MmQAAAZNBkINADQDTCMhoAGCZBpoNMg0Bk0AGRiGmQBghkNGmgAaGgGhkyAAADJoMhBoAaAaYRkNAAwTINNBpkGgMmgAyMQ0yAMEMho00ADQ0A0MmQAAAZNBkCapJBTEyQ0nqeSZioDeqNlG1PU2o9Go9TTQ09QaPUD0CPSAA9Q9QPKA0AGQaANHqBo0GgNGgB6gaApSSCAJiIwmhlNGmjEmCYmTKbSnqP1J6m8gp56ppN6pp5R7U9Q9JMjxTbIk9Mj1BtUMCeoaAB+qNHqAyaNpHtoqfqcgV6L7jHRddxuUdeeq6Vy1s1q0PGdR98uY7Jv2di02u25eLvCuiUfprinG43A5DrtXzq4XdXQ7HSYXSK3F/dxtXgcbZptcVjbJtpppp/Jf7H57vPBXRiuNZXU0OXvTI3V0W5xmyvSd95rhM4s5HF6D4c/uT8etNOq3zqacJzz03vLmXS4Ti2aY77qnmeN2ze315l+oX4t89/x/uvSeN03beGb50V1vfv6lfNn12n6b0Nr8O/F31q1q6ToOydk+ONGjRhho7FPPyPwV4R1FsuRZeCbL18PZnyFyjkGq+Ies8bnvM1MaZeeT30nU/N9Pp+n+f+V+X3gHeXPR8p8ufwnqncyf4K4K2m5qaj1z2mi04YafvWjcbODfNTcysmVvfrTZjhVkHhejPQeOeZ4p5nknpOSZORxb5wdM783XnrL1K6lli2Oqt87d6wxoq9A/sudf+W6sMUfSaLwsnTN1/8P6Uyxj/fPzay3SR2UxR1BeGaFV7DFFecsUrcenP69fnv+pkybyK9J3V5ZpqYxqanlMnjm+OqO56ZxeWPhrS0vUt7DlmN1p3pckN+KtJYG+cJ3nHdBXlG5cTg3Ttraw5zi0vVz7jzst5xk3rRqVcjCLknW9BsR79+RYyYzSuLBWVvm9o+83EtLZ672k6tvu1Xo2MWpHxd6r2g8tdw5n8p/3GOByV4H8N+K7p+W5DDcuDhPi+JjMY6U+y/sP1GOntZY8C6+dj0maa1YYysvzXR0N1jWrucLof8TTZyB45+y6nFjg/z69n0TMdBu1kbTHvWR3mZvN+zc00PBYssS6HoPcW7g7bUzKwy1Y9c5ji4ufZYcurGDGGMzGMZTGVjLLBXdcro2lT5SwSfEmSqfBMpVxLGGTKxkwsr2E0T3yYkH6DPqsYy0zTGmO8ZS4Xv7G1YxjaloxJ3Y4vrmqo5m5qT7Jbm/aIca41org/rW6bJxTDgY7u31qhvMTpViiu/uvaK4LEptss0nNYk08D89snSsWLBPxFinvnjR7TGODdV8KyfMup+o2rmRjGLK/ErRxyWPKtRuYxh6DFteVdC1NnqHf1F02DtrErv3bdBsZeZP0VtpNLSxiNYV8BZDqLJ51g9ZXYmWJss2t9i4OJ8hbU3uNobm1cLibk4le1PGjxY2jnS8kcJP1JZLuhncnnjScVhX7Bf0TVXyy+CakeSMk9Bo1X0JyPKbQsrvLVLhWJTwGRXlsVdtcvwGMcN0nrsocD6Y9htSjvMReLCx9i9PaU+wwScpkj02EXpV51fKk2qHW7zRTUydk9k5W1epWXltGktGU6S7DMsmWWJcxkc1W0m6PhRhYsCxZL3C7btuqrcq3WL8A63U2WP1WO1tL4NZJvna1XblwdrlxtQ2d9p1f2WLBlqpi5Wi8U4LVjnY/SYXU3Rq4xpUPZuStUdjrvTi8StNHaOS7VVxVwrt2rmrrDTmS5jRWjE5phjDkrFwum1W9LhYai4OuNSca3JbzDYqWjAuPPedpdWzRlWWKbqV5NcDUjDI5JhoyXKsfnPOvymmOFW5RwmTzrLo78mvRdl+A4LhXCuC6vWrDsaNDNrSw4zDS9V6q3t0my3XDK1bB0VlTaHI5myxblfqn9ph0VwriA4jHMZYZRcito3m83W+sWTG4piuqytrJYLFtq1ZmWrWlixZLGGq4XdDRxOJqjz4vZWR58ZXfRlGoy/KlkNRqlvj8mS2W5ZZVe7NHKN9cpyngzbY2NWsWdWrzPM7/3H1f0/1nJV/9OXU6w0yw8NtNYGbJeJjZbXUZjzMeJuvKYvgNNGMaZuTE3G9h8dPBN28mo8L0a3E7lcK0NKyi98dIXNfoHSPUHGcbJ4U945n0jGn0V2D//d45VNjue9eC1k+Vew1Yw2N3FPZbq2sq3LFyOg8j77ctzkUeAxWLCauB1upch/ib0/JMowwyxNjIY5jUuaOjyVs6KsrnhzGczEtssWSy3WZjGWquxNDRgyMpjDEwWW1qWjKxZczLanB0vvuh6Det7gOpYzGctNLLdWaNMzTLS1WYmGWDbGGlq5lhpuatWtGpWI3GW0ppqqmcV2nfcW9Li52FojtWJXwcFeN7E/Dr48+G+292xp/hf5Z+M/4HRWXtpdq/jX+N5q8ReyK6tHoXyi1DsX0zwW1tlcHkN7H7C7l5a1au+sfwL4N/guN5li+BfvL3t7w+nPeP6Ttr6k/Xfqz91+3P3z2z+uXJWj9Gp9lZtXhpYeRf16RelfDx5tbPBs9IxVdowJ7d8Vgww0X23RWkyspfOYn4RMfiPntI+pVl819O+l+A+E3Mzsf5181q97fOMfUTc3ORjmhxXpjFi0dN4LXdsuOq0undqvur5Jj131ThWPoL9ReB1dK/E415JMC+Sb00nVdHXOqch5ZwTvpe7ucytpPsa61y1zLdMcNKfhmzcdLdGVkaYqaMF9A4tUri6XUxtI6jjNhyzlZvmuFbm0NWBqc60TVdl+df6LtVc9bzhGLmSxLtutNy6gwp1kuFcK61ul4w55LtqtVbE/kNzSvCyR4THEyI3MJyMFYeU1OUMlY7n6A69wOl2Hm2bRt2fRlz210T7Lc4hvrIOFXItTLIwadavpRz19Z+6uVe6/vLwq9B6DnH1ZNNy7l1suc1Xk8Tsq83CrvPCP5zXUaa7xyOLzPx3nfoOqNMqu5mzLXpOB/qHkk3sk00mzELVhJwq5G5vV3q3rV31jRjRpXKYlcjkYfTPVN0e+dTi7Ha6nA4Pf16L+0+S3vxntMeQ94dFs3PbPVaaY0eJ4n12LDmerNzmdVeo3jZ0Wn6TztPAY7nfr8r+XufE+rp/K9z+XyPC3zD0WGP902PXvh9ravQvM7fjz584OBXqDj4V5K9ls8TG3C9byv9tsbzDmcrGcWr23E5XqOven2Hqtzmb/OdBj4Drb3oZvxh81qTZstK3MaYx7xps9tonI3mqnla6q+BuWx2T+O/vv8b3vl9Zvdb+/XMruD0z4rpdzQxeVbOR3q5J3THBvPA3tzyP8BTnGH3jENMMyxlMsxg0sxi1aasZS633V9BdDpYx0nq+E6HvHS1fMdRp53c077vvVNjnnfdbZ52HQ6GnOx52mlxMrifkJ3P4H333X1n663ud4n8Jzl5TmdDyPmF+c/cd5xcgD8x0v9puXlOx0HfeL4ryD2fBXTXbX0a+jX5E9weUcTicTicTicTicFXA4HA4HA4HhXm+VetOVcxh4p3LV5Myp1UvGPRPcnqD8Q2NjY2NjY2NjhXuUserunePXK67236bxvkMfCcyu2uZ8dy258NqOlwrcHJZecxORjTpelVvbljocqbI53xWZjlcXl+W+Yuze35blS9+MVctlmKc2VdNh8rjTwHcX5rdXynpPYdFmPw30HBte5bNNm+1egy2vC8TqbNNmm1dK8VtX/s+e/i9jdU8BiDhed2um3V2saPM4ydlp2Mb2m+652Ox8KsbrLlnh5KPVGSded05W584ceNxVev7y3XJbo7DptKOplXrsE6WSfEN903xvyZR8B8R8R73dJyhwjtPcJ6NNyK8lWI3PXbm0bOq7/HpmrdUdC4GjvbLjYsWJ7z4J49j4Vo0f4U/bb6zMf9OGlu3zJ/2za/xP8fG4eAOc7E6ho+uWLqGPFxd727L8aq5f/b0n/MtrjX1r+Bafhm3Ce5WmVixZWNPuNGPdvj2x7rLgZcMMmrlO0Buc7fml0arVwuE47mYw0maYynSMORNTUbjhnI1bGN1XKP2zLGpfJb8ZlwXO0ZjGlz4f1l0LhG6MXzHYHcZVt+EaTxlj/cMm5h+cyHrvzGX8SenN09OZOVkYNPLfqk3e4+8WoXmr5NfbH0XD8iT435PY4uVgxo0000f8q/FvjG+vEzGSuxcLV75vrU8aT9K/GclTcTJMFjIc75taDx2DnXPbrRtjv5qnjxMrpco5Z+NacOENnW0bORzOsxwcH0HFcjneNpC/iri4t9cCfzq/erS70xHTX753jYBxrBisj1cU6LIO2bpNo2V9Qak1TF4H2k8rHsNMewfOXyvwr+McsncOT1j/vObbPYPsLvsHx33mn8R7r+ibzH9Ns7jZp8e/eb3zXBwc5+y24vmOZztqvkN6z2j13or4K9dXiV76vXeeGq9k4GyppTHvZuT4Qr5F+fW9Hxji+kux1Sb57OC+57rR7rBvmJMGTHTVonx3M7xsbN5pvPSq1VirJNVqrpbDtna3zYnfXzK/91fTVP8qb02TYjUFZGUskslT6Bf0WK9rHvY+wN1irdG6NVujaNo1G0exl3aXjnQuQdEyVo7vkktptbrL2024zUv+KZGSslZGRhixZYXVPcHSrpLIFi0/qIG5xjzv7xzvEY3SfivEafafzD+Y0MdLGnGuxWmzHQ/0Di2O9VXI2cF3pzXLdGTnzWtD4T5DsW1eCudDg4ty9qd4/sOJeivWODcvq1xbLmfj1qub6ldHQ8LsOCr126tLjw5utuV6ECx0XBrOH2XlfwMaae0556C/nmGMbDTD9+sDmI/BvcvktXvlb1v+adfWu46W4Z6S0cFvfra47b16DcvCcjg2bMq4OXfbQew9sYxjGLSfaXz13Lr21lllqy2twX75Vfu2AyH0YPf24+yjorQ5+8h0S7dR/rH2qVqDdXUk+Sk5IxXPQf9ir234LZzKm8dy41xkt0PgR76HjQ6JL3SqmSOtJ3UHKcRXRKn+Rsf8j7ir7kC6/UfLOA/TUPSgXdAu4V0VV1q4zxsP4xznEWxYnoSP1ivCrxq7wNVTdC0D0lVOeCvEqOi3R3sLnna9RMsY+MamYxjUf2V661P47v2nXtXzs5wetm+bTIdBwh6e+WvvbVc+bZP7Yy1YvBXOqxJ30PIV7fuqx9SMmMMY7LVjG+S6IyPiqp4Eakui7CvM7Kv2EnFUjjAvfFewVwVOSS7Mao9TDxTlVfClhkspcsB2PcfAe8nYOww6SmqwRlZCysNdLaej4FTwo9nwXBZaWl5uatmmrfbrZbHBYNmrFtbJ7xtbTG6xYerb7LL2UmT9ur54D+5V6tPU3ozGSyZPEV4S6JYfyT7qaNGzVNDs8Feer6i9gfBrFpbzjfYr0Z4huT5zJk5VWwfzX45uNljyNDG05zZZKdtWi/ecFpYbIu+m0cllZZVlZHQPYDnh469YW+YyzGOs1S5DzmlyGLc62l2rDyDKuQ9jai4VknV0tK2tS0sZjDWVzstxY3VaP4afXf6L6++rhYwx42nFYXeLpTqVp3DHVMeFwJplHv2RyzDe41pTmr9dquVk5WI+7tqN7xMY51NrCwxYxWMMMUODVd94IflreuiuDS4Q8B+01PdWIeR7Z2PSboDmZBzTZbm0vlm+YamDe71cHDeLe3NLQx35tNze/Zd4OtZJ4H5jleDtaO1dLTc/Yf6Tit0uDFV/VZHc77k3njf+BwXQY7l3VpE6HQ1bjTVdzLwTiWkXW61jv/bfz2MftP2mn8hsbNmnfe1dlci/cpeBZe9O9Mb3ad5J4ZMoeVvXnvE3Vf8IfuPGfXMf/K309N6h2v/Rji5HmfdXovWedzucx3N7e+s7GzZvK6n3jxvabNmzc+0/BWl3nefl8G5cr7RzMaaP8jFsvuXMcFe0713O7ZfWn3pPaq4rdKejPByrzPUN6cBH6LZsTzcXrHkrrdfMONXA5hhyybH4Ru4LktC3T4dvvdXM7ThXPdPp7m9yTlH6VdKvbzBd4ZjJll3nAi+2sX31iU4m52d+92rlq6g9cftu+HkO1bx/+LuSKcKEhVMDc/A="

    kernels = Kernel(
        bz2.decompress(base64.b64decode(quantization_code)),
        [
            "weightInt8_int4",
            "weightInt4_fp16",
            "weightInt4_bf16"
        ],
    )
except Exception as exception:
    kernels = None
    logger.warning("Failed to load cpm_kernels:" + str(exception))


def quantize_int8(weight: torch.Tensor, bit_length: int):
    weight_scale = weight.abs().max(dim=-1).values / ((2 ** (bit_length - 1)) - 1)
    weight_scale = weight_scale.to(torch.float32)

    weight = torch.round(weight.to(weight_scale.dtype) / weight_scale[:, None]).to(torch.int8)
    return weight, weight_scale


def compress_int4_weight(weight: torch.Tensor):
    with torch.cuda.device(weight.device):
        num_row, num_chan = weight.size(0), weight.size(1)
        num_chan = num_chan // 2

        int8_weight = torch.empty(num_row, num_chan, dtype=torch.int8, device="cuda")
        stream = torch.cuda.current_stream()
        dim_grid = (num_row, 1, 1)
        dim_block = (min(round_up(num_chan, 32), 1024), 1, 1)

        kernels.weightInt8_int4(
            dim_grid,
            dim_block,
            0,
            stream,
            [
                ctypes.c_void_p(weight.data_ptr()), 
                ctypes.c_void_p(int8_weight.data_ptr()), 
                ctypes.c_int32(num_row), 
                ctypes.c_int32(num_chan)
            ],
        )

        return int8_weight


def dequantize_float(weight: torch.Tensor, weight_scale: torch.Tensor, bit_length: int, input: torch.Tensor):
    if bit_length == 8:
        float_weight = weight.to(input.dtype) * weight_scale.to(input.dtype)[:, None]
        return float_weight
    
    assert bit_length == 4, f"unsupported bit length: {bit_length}"
    
    func = (
        kernels.weightInt4_fp16 if input.dtype == torch.half else kernels.weightInt4_bf16
    )
    with torch.cuda.device(weight.device):
        num_row, num_chan = weight.size(0), weight.size(1)

        float_weight = torch.empty(num_row, num_chan * 2, dtype=input.dtype, device="cuda")
        stream = torch.cuda.current_stream()
        dim_grid = (num_row, 1, 1)
        dim_block = (min(round_up(num_chan, 32), 1024), 1, 1)

        func(
            dim_grid,
            dim_block,
            0,
            stream,
            [
                ctypes.c_void_p(weight.data_ptr()),
                ctypes.c_void_p(weight_scale.data_ptr()),
                ctypes.c_void_p(float_weight.data_ptr()),
                ctypes.c_int32(num_row),
                ctypes.c_int32(num_chan),
            ],
        )
        return float_weight

class QuantizationLinear(torch.nn.Module):
    def __init__(self, bit_length: int, weight: torch.Tensor, device="cuda"):
        super().__init__()

        self.bit_length = bit_length

        weight, weight_scale = quantize_int8(weight=weight, bit_length=bit_length)
        if bit_length == 4:
            weight = compress_int4_weight(weight)

        self.weight = torch.nn.Parameter(weight.to(device), requires_grad=False)
        self.weight_scale = torch.nn.Parameter(weight_scale.to(device), requires_grad=False)

    def forward(self, input: torch.Tensor):
        input_size = input.size()

        input = input.contiguous().view(-1, input.size(-1))
        original_weight = dequantize_float(self.weight, self.weight_scale, self.bit_length, input)

        output = torch.matmul(input, original_weight.t())
        return output.view(*(input_size[:-1] + (self.weight.size(0),)))