Project Name: Lightning-AI+lit-gpt
Language: Python
Statement Type: API

Source Code:
def test_base_model_can_be_adapter_v2_loaded(name):
    from lit_gpt.adapter_v2 import GPT as AdapterV2GPT
    from lit_gpt.adapter_v2 import adapter_filter
    from lit_gpt.model import GPT as BaseGPT

    kwargs = {"n_layer": 2, "n_head": 8, "n_embd": 16, "padded_vocab_size": 32}
    base_model = BaseGPT.from_name(name, **kwargs)
    base_model_state_dict = base_model.state_dict()
    lora_model = AdapterV2GPT.from_name(name, **kwargs, adapter_start_layer=0)
    keys = lora_model.load_state_dict(base_model_state_dict, strict=False)
    assert not keys.unexpected_keys
    for k in keys.missing_keys:
        assert adapter_filter(k, None)

test_base_model_can_be_adapter_v2_loaded(name='stablelm-base-alpha-3b')

Selected Statement: base_model = BaseGPT.from_name(name, **kwargs)

Function Input:
name: 'stablelm-base-alpha-3b'

Variable Values Before Statement:
name: 'stablelm-base-alpha-3b'

Value After Statement Execution: GPT(  (lm_head): Linear(in_features

Variable States During Runtime (line, value):
name: 1
'stablelm-base-alpha-3b'
AdapterV2GPT: 2
<class 'lit_gpt.adapter_v2.GPT'>
adapter_filter: 3
<function adapter_filter at 0x7f801db68af0>
BaseGPT: 4
<class 'lit_gpt.model.GPT'>
kwargs: 6
n_layer: 2
n_head: 8
n_embd: 16
padded_vocab_size: 32
base_model: 7
GPT(  (lm_head): Linear(in_features=16, out_features=32, bias=False)  (transformer): ModuleDict(    (wte): Embedding(32, 16)    (h): ModuleList(      (0-1): 2 x Block(        (norm_1): LayerNorm((16,), eps=1e-05, elementwise_affine=True)        (attn): CausalSelfAttention(          (attn): Linear(in_features=16, out_features=48, bias=True)          (proj): Linear(in_features=16, out_features=16, bias=True)        )        (norm_2): LayerNorm((16,), eps=1e-05, elementwise_affine=True)        (mlp): GptNeoxMLP(          (fc): Linear(in_features=16, out_features=64, bias=True)          (proj): Linear(in_features=64, out_features=16, bias=True)        )      )    )    (ln_f): LayerNorm((16,), eps=1e-05, elementwise_affine=True)  ))
base_model_state_dict: 8
OrderedDict([('lm_head.weight', tensor([[-0.2040,  0.1113,  0.0584, -0.1935, -0.0368, -0.2365,  0.0272,  0.1651,          0.1920,  0.2497, -0.1360, -0.2344, -0.0012,  0.1705,  0.1699,  0.2030],        [-0.2263, -0.1364, -0.1686, -0.0148,  0.1299,  0.1057, -0.1175,  0.2214,          0.0702,  0.0628, -0.0087, -0.1382,  0.1014,  0.0977,  0.0380,  0.1590],        [-0.0566, -0.1740, -0.1575,  0.0049,  0.1824, -0.2248,  0.1395,  0.0325,          0.2379, -0.0782,  0.1699, -0.0943,  0.2191, -0.0986,  0.0821, -0.2279],        [-0.2114,  0.1223,  0.0569, -0.2201,  0.1737, -0.0730,  0.2211,  0.0469,         -0.0359, -0.1367, -0.1522, -0.1705, -0.2474,  0.0640,  0.1010, -0.2316],        [ 0.2279,  0.2448, -0.0281, -0.0656,  0.0848,  0.0019, -0.1033,  0.2023,         -0.0742, -0.1102, -0.2262, -0.1674, -0.2286, -0.1058, -0.0161,  0.0969],        [ 0.1002, -0.2468, -0.0489,  0.2212, -0.1703,  0.2316, -0.1648,  0.1787,          0.2121,  0.0849,  0.2258, -0.2450, -0.1595,  0.1691,  0.0878, -0.1187],        [ 0.0137, -0.1362, -0.1799, -0.1539,  0.0538, -0.0110,  0.1377, -0.1469,         -0.2303, -0.0714,  0.0875, -0.2432,  0.1248, -0.1095,  0.0290, -0.1726],        [-0.1370,  0.0523,  0.1150, -0.2129,  0.1642, -0.0408, -0.1308, -0.0780,          0.0291, -0.0083, -0.1428,  0.1091,  0.1643,  0.0100,  0.2389,  0.0719],        [-0.2246, -0.1863, -0.1718, -0.1688, -0.1824, -0.0768,  0.0202,  0.1226,         -0.1975,  0.2080,  0.0941,  0.0397,  0.2238, -0.1715,  0.0790, -0.0336],        [-0.0374,  0.1743,  0.1776, -0.0401,  0.0524, -0.2052,  0.1173,  0.0335,         -0.2399,  0.2152,  0.0909, -0.0933,  0.1838, -0.0556,  0.0652,  0.2024],        [ 0.2485,  0.0462,  0.1087, -0.2251, -0.1969, -0.0321,  0.2268,  0.1194,         -0.0749,  0.0085,  0.0455,  0.2372, -0.0372,  0.2139, -0.0159, -0.1402],        [-0.2278,  0.1227, -0.0303, -0.1931,  0.2433, -0.2397, -0.0908,  0.0450,          0.0401, -0.1654,  0.1077, -0.1347, -0.1677, -0.0515,  0.1379, -0.0590],        [ 0.2161,  0.2441, -0.2048,  0.0042, -0.2058,  0.1390, -0.2005, -0.0724,         -0.0006, -0.0823, -0.1921,  0.0568, -0.1141, -0.1868, -0.0980,  0.1916],        [-0.2162, -0.0590,  0.1730,  0.0203, -0.1542, -0.0287, -0.1238,  0.2366,         -0.1960,  0.0638,  0.2467,  0.0968, -0.0297, -0.2187, -0.1270, -0.1592],        [-0.1953,  0.0800, -0.2453, -0.2434, -0.2289,  0.1761,  0.0080, -0.2330,         -0.1634,  0.0117,  0.1099,  0.1184,  0.0833,  0.1710,  0.0734,  0.0825],        [-0.0449,  0.0028, -0.1980, -0.1582, -0.0300, -0.2378,  0.1776, -0.0695,          0.1542, -0.0839, -0.0305, -0.1438, -0.1355,  0.1401,  0.1814,  0.0663],        [-0.1543,  0.2484, -0.1478,  0.1234, -0.1865,  0.1914,  0.0307,  0.1875,         -0.0973,  0.0588,  0.2018, -0.0548,  0.1702, -0.1610, -0.2060, -0.1724],        [ 0.1537, -0.0495, -0.1406,  0.0114,  0.0301, -0.1971,  0.0294,  0.0739,          0.0160,  0.1448, -0.2331, -0.0077, -0.1525, -0.0146,  0.1653, -0.0413],        [-0.2186, -0.0141, -0.1605, -0.0941,  0.2489, -0.0499, -0.0589, -0.0887,          0.1524, -0.1399,  0.2012, -0.0109, -0.0090,  0.0946, -0.1322, -0.0652],        [-0.1617,  0.1239,  0.0779, -0.1597,  0.0285, -0.0280, -0.2459,  0.1879,         -0.1888,  0.0874, -0.2031, -0.1358, -0.1345,  0.1417,  0.1186,  0.0337],        [-0.2315,  0.0632,  0.1275,  0.0153,  0.0495, -0.0769, -0.0769,  0.0444,         -0.0225,  0.1375, -0.1902,  0.1155, -0.2222,  0.0365, -0.0030,  0.1707],        [-0.1867,  0.0813,  0.2142,  0.1787,  0.0732, -0.1879, -0.2255, -0.2374,          0.1491,  0.1437, -0.0771, -0.1960,  0.1335,  0.0227,  0.2434, -0.0845],        [-0.1916, -0.1467,  0.0975, -0.0115, -0.1319,  0.0445,  0.0236, -0.1961,          0.0639, -0.1922,  0.0300,  0.0432, -0.0061, -0.1202,  0.0846, -0.0664],        [-0.2105,  0.0031, -0.1161, -0.0683,  0.2353,  0.1651, -0.2034,  0.1467,          0.0378, -0.0989,  0.0239,  0.2026,  0.2267,  0.2138, -0.2073,  0.0165],        [ 0.1156,  0.2149, -0.0286, -0.1842, -0.1246,  0.2320, -0.0424, -0.1798,         -0.0945, -0.2007,  0.0248,  0.1019,  0.1329, -0.1646,  0.0107,  0.1050],        [-0.1296, -0.1141,  0.2485,  ....1062, -0.1109, -0.1927,  0.0626,  0.2419,  0.1540,  0.1249,  0.2342],        [ 0.2244, -0.1377, -0.2170,  0.0662, -0.1891,  0.1060, -0.2274, -0.2134,          0.2055, -0.1398,  0.1706,  0.0286, -0.1660, -0.1758, -0.0727,  0.0104],        [-0.1086,  0.2059, -0.1085,  0.0878, -0.2465, -0.1247, -0.0222,  0.1380,          0.1035, -0.2425,  0.0100,  0.1510, -0.0806,  0.0448,  0.0790,  0.0523],        [ 0.1252,  0.0400,  0.0261, -0.2488, -0.2045, -0.1933,  0.1192,  0.1677,          0.0642,  0.1778,  0.2086,  0.1216, -0.0441, -0.2306,  0.2251,  0.1947],        [-0.0092,  0.0686,  0.0206,  0.0507,  0.0820,  0.1262,  0.0621,  0.2165,          0.2090, -0.1457,  0.1741,  0.1685, -0.2353, -0.0548,  0.1855, -0.2016],        [ 0.1959,  0.0742, -0.2326, -0.1294,  0.0701, -0.0846,  0.0796,  0.1885,          0.2356,  0.1602,  0.0801, -0.0599, -0.0415,  0.1231, -0.0243,  0.0458],        [-0.2164,  0.0750, -0.0714, -0.0557, -0.1265, -0.0025, -0.0520, -0.2037,         -0.2366, -0.0198, -0.0369, -0.1668,  0.1378, -0.2271, -0.0582,  0.1369],        [ 0.0529, -0.2322,  0.1400,  0.0548,  0.1427,  0.0732, -0.2172,  0.0945,          0.0295, -0.0840,  0.1653, -0.1925, -0.0347, -0.0753,  0.0523,  0.1021],        [-0.2317, -0.1887, -0.1400, -0.0594,  0.1515,  0.0425, -0.0596,  0.0958,         -0.1809, -0.0933,  0.0679,  0.0599, -0.0747,  0.1119, -0.0284,  0.0506],        [-0.1945, -0.1917, -0.1075, -0.1584, -0.2365, -0.2396, -0.2490, -0.0487,          0.1456,  0.1571,  0.0480,  0.2459,  0.2245, -0.0147,  0.0579,  0.0433],        [-0.1347, -0.1925, -0.2312,  0.1519, -0.1227,  0.1162,  0.1610, -0.1877,          0.2061, -0.2271,  0.1379, -0.2204,  0.2442,  0.1041,  0.0929, -0.1878]])), ('transformer.h.1.attn.proj.bias', tensor([-0.0892, -0.2182, -0.1580,  0.0412,  0.0140,  0.2101,  0.1820, -0.2064,        -0.1241, -0.0571,  0.1290,  0.0343, -0.2440, -0.1654,  0.0235, -0.1155])), ('transformer.h.1.norm_2.weight', tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])), ('transformer.h.1.norm_2.bias', tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])), ('transformer.h.1.mlp.fc.weight', tensor([[ 0.0924,  0.1510, -0.0735,  ...,  0.1847, -0.1331, -0.1429],        [-0.2016,  0.2156,  0.0506,  ..., -0.0418, -0.1739, -0.2487],        [ 0.2222,  0.1940,  0.0379,  ...,  0.1357,  0.2448, -0.2166],        ...,        [ 0.1076, -0.1423,  0.0219,  ..., -0.0825,  0.1934,  0.1640],        [ 0.1174,  0.0894, -0.0815,  ..., -0.1510,  0.0219, -0.0885],        [-0.1409,  0.0148,  0.2021,  ..., -0.2060, -0.0150, -0.1007]])), ('transformer.h.1.mlp.fc.bias', tensor([ 0.1542,  0.1957,  0.0429, -0.0221,  0.0788,  0.2306,  0.2165,  0.1671,         0.0664, -0.1140, -0.0531, -0.0085,  0.0917, -0.1900, -0.1731,  0.2154,        -0.1378, -0.0411, -0.2255,  0.1157, -0.1700,  0.1329,  0.1946,  0.0830,         0.0852,  0.1996,  0.2274,  0.0734,  0.1994, -0.2326,  0.2143,  0.1984,        -0.0805,  0.1104,  0.1824, -0.0666,  0.1265,  0.1228,  0.2238,  0.2137,         0.1964, -0.0859, -0.2379, -0.0537,  0.1860,  0.0125,  0.0383, -0.2439,        -0.2233, -0.1594,  0.0032,  0.1765, -0.0252,  0.2003,  0.0800,  0.0508,         0.0850,  0.0321,  0.0886, -0.1280, -0.0688, -0.0091,  0.1421, -0.2377])), ('transformer.h.1.mlp.proj.weight', tensor([[ 0.1238, -0.0415, -0.0093,  ...,  0.0712,  0.0379,  0.1029],        [ 0.0671, -0.0787, -0.0885,  ..., -0.0070,  0.0109, -0.0624],        [-0.1076, -0.0217, -0.0052,  ...,  0.0668, -0.0339,  0.1202],        ...,        [-0.0757, -0.0012,  0.0383,  ..., -0.0417, -0.0944, -0.0468],        [ 0.0752, -0.0184,  0.0511,  ..., -0.0576, -0.0293,  0.0188],        [-0.0496, -0.0871, -0.0883,  ..., -0.1221,  0.0080,  0.0647]])), ('transformer.h.1.mlp.proj.bias', tensor([-0.0183,  0.0377,  0.1179, -0.1148, -0.0526,  0.0324,  0.0845,  0.0960,        -0.0208,  0.1116, -0.0654, -0.0011, -0.0743,  0.1182,  0.0757,  0.0495])), ('transformer.ln_f.weight', tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])), ('transformer.ln_f.bias', tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]))])
lora_model: 9
GPT(  (lm_head): AdapterV2Linear(    (linear): Linear(in_features=16, out_features=32, bias=False)  )  (transformer): ModuleDict(    (wte): Embedding(32, 16)    (h): ModuleList(      (0-1): 2 x Block(        (norm_1): LayerNorm((16,), eps=1e-05, elementwise_affine=True)        (attn): CausalSelfAttention(          (attn): AdapterV2Linear(            (linear): Linear(in_features=16, out_features=48, bias=True)          )          (proj): AdapterV2Linear(            (linear): Linear(in_features=16, out_features=16, bias=True)          )          (adapter_wte): Embedding(10, 16)        )        (norm_2): LayerNorm((16,), eps=1e-05, elementwise_affine=True)        (mlp): GptNeoxMLP(          (fc): AdapterV2Linear(            (linear): Linear(in_features=16, out_features=64, bias=True)          )          (proj): AdapterV2Linear(            (linear): Linear(in_features=64, out_features=16, bias=True)          )        )      )    )    (ln_f): LayerNorm((16,), eps=1e-05, elementwise_affine=True)  ))
keys: 10
_IncompatibleKeys(missing_keys=['lm_head.adapter_bias', 'lm_head.adapter_scale', 'transformer.h.0.attn.gating_factor', 'transformer.h.0.attn.attn.adapter_bias', 'transformer.h.0.attn.attn.adapter_scale', 'transformer.h.0.attn.proj.adapter_bias', 'transformer.h.0.attn.proj.adapter_scale', 'transformer.h.0.attn.adapter_wte.weight', 'transformer.h.0.mlp.fc.adapter_bias', 'transformer.h.0.mlp.fc.adapter_scale', 'transformer.h.0.mlp.proj.adapter_bias', 'transformer.h.0.mlp.proj.adapter_scale', 'transformer.h.1.attn.gating_factor', 'transformer.h.1.attn.attn.adapter_bias', 'transformer.h.1.attn.attn.adapter_scale', 'transformer.h.1.attn.proj.adapter_bias', 'transformer.h.1.attn.proj.adapter_scale', 'transformer.h.1.attn.adapter_wte.weight', 'transformer.h.1.mlp.fc.adapter_bias', 'transformer.h.1.mlp.fc.adapter_scale', 'transformer.h.1.mlp.proj.adapter_bias', 'transformer.h.1.mlp.proj.adapter_scale'], unexpected_keys=[])
@py_assert1: 11
None
@py_assert3: 11
None
k: 12
'lm_head.adapter_bias'
12
'lm_head.adapter_scale'
12
'transformer.h.0.attn.gating_factor'
12
'transformer.h.0.attn.attn.adapter_bias'
12
'transformer.h.0.attn.attn.adapter_scale'
12
'transformer.h.0.attn.proj.adapter_bias'
12
'transformer.h.0.attn.proj.adapter_scale'
12
'transformer.h.0.attn.adapter_wte.weight'
12
'transformer.h.0.mlp.fc.adapter_bias'
12
'transformer.h.0.mlp.fc.adapter_scale'
12
'transformer.h.0.mlp.proj.adapter_bias'
12
'transformer.h.0.mlp.proj.adapter_scale'
12
'transformer.h.1.attn.gating_factor'
12
'transformer.h.1.attn.attn.adapter_bias'
12
'transformer.h.1.attn.attn.adapter_scale'
12
'transformer.h.1.attn.proj.adapter_bias'
12
'transformer.h.1.attn.proj.adapter_scale'
12
'transformer.h.1.attn.adapter_wte.weight'
12
'transformer.h.1.mlp.fc.adapter_bias'
12
'transformer.h.1.mlp.fc.adapter_scale'
12
'transformer.h.1.mlp.proj.adapter_bias'
12
'transformer.h.1.mlp.proj.adapter_scale'
@py_assert2: 13
None
@py_assert4: 13
None