patrickvonplaten
commited on
Commit
β’
a85c2d0
1
Parent(s):
f0d385b
finish
Browse files- after_fix_log.txt +62 -0
- after_fix_pretrained_log.txt +212 -0
- before_fix_log.txt +15 -0
- before_fix_pretrained_log.txt +40 -0
- check_gradients_pt_flax.py +4 -8
- run_models.sh +4 -4
after_fix_log.txt
ADDED
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Check hf-internal-testing/tiny-random-roberta ...
|
2 |
+
--------------------------Checking logits match--------------------------
|
3 |
+
Flax logits shape: (2, 64, 1000), PyTorch logits shape: torch.Size([2, 64, 1000])
|
4 |
+
β
Difference between Flax and PyTorch is 1.7881393432617188e-07 (< 0.01)
|
5 |
+
--------------------------Checking losses match--------------------------
|
6 |
+
Flax loss: 6.887884140014648, PyTorch loss: 6.887884616851807
|
7 |
+
β
Difference between Flax and PyTorch is 4.76837158203125e-07 (< 0.01)
|
8 |
+
--------------------------Checking gradients match--------------------------
|
9 |
+
β
All grads pass
|
10 |
+
--------------------------Checking rel gradients match--------------------------
|
11 |
+
β Layer ('roberta', 'encoder', 'layer', '0', 'attention', 'self', 'key', 'bias') has PT grad norm 7.584575871001642e-13 and flax grad norm 6.388195094436666e-13.
|
12 |
+
β Layer ('roberta', 'encoder', 'layer', '1', 'attention', 'self', 'key', 'bias') has PT grad norm 7.811836030477415e-13 and flax grad norm 6.42668156105447e-13.
|
13 |
+
β Layer ('roberta', 'encoder', 'layer', '2', 'attention', 'self', 'key', 'bias') has PT grad norm 8.422985074175993e-13 and flax grad norm 6.414080963405844e-13.
|
14 |
+
β Layer ('roberta', 'encoder', 'layer', '3', 'attention', 'self', 'key', 'bias') has PT grad norm 8.625919531608794e-13 and flax grad norm 7.699825477734679e-13.
|
15 |
+
β Layer ('roberta', 'encoder', 'layer', '4', 'attention', 'self', 'key', 'bias') has PT grad norm 1.0383360837806777e-12 and flax grad norm 6.049140680551568e-13.
|
16 |
+
=========================================
|
17 |
+
Check hf-internal-testing/tiny-random-bert ...
|
18 |
+
--------------------------Checking logits match--------------------------
|
19 |
+
Flax logits shape: (2, 64, 1124), PyTorch logits shape: torch.Size([2, 64, 1124])
|
20 |
+
β
Difference between Flax and PyTorch is 1.7881393432617188e-07 (< 0.01)
|
21 |
+
--------------------------Checking losses match--------------------------
|
22 |
+
Flax loss: 7.036032199859619, PyTorch loss: 7.036032676696777
|
23 |
+
β
Difference between Flax and PyTorch is 4.76837158203125e-07 (< 0.01)
|
24 |
+
--------------------------Checking gradients match--------------------------
|
25 |
+
β
All grads pass
|
26 |
+
--------------------------Checking rel gradients match--------------------------
|
27 |
+
β Layer ('bert', 'encoder', 'layer', '0', 'attention', 'self', 'key', 'bias') has PT grad norm 5.234438642080785e-13 and flax grad norm 4.935363641205004e-13.
|
28 |
+
β Layer ('bert', 'encoder', 'layer', '1', 'attention', 'self', 'key', 'bias') has PT grad norm 9.028551018787356e-13 and flax grad norm 6.16219206737989e-13.
|
29 |
+
β Layer ('bert', 'encoder', 'layer', '2', 'attention', 'self', 'key', 'bias') has PT grad norm 8.728350616056535e-13 and flax grad norm 6.037235598596591e-13.
|
30 |
+
β Layer ('bert', 'encoder', 'layer', '3', 'attention', 'self', 'key', 'bias') has PT grad norm 8.327751465850297e-13 and flax grad norm 7.390156737431541e-13.
|
31 |
+
β Layer ('bert', 'encoder', 'layer', '4', 'attention', 'self', 'key', 'bias') has PT grad norm 7.404479048130075e-13 and flax grad norm 7.178592030705755e-13.
|
32 |
+
=========================================
|
33 |
+
Check hf-internal-testing/tiny-random-t5 ...
|
34 |
+
--------------------------Checking logits match--------------------------
|
35 |
+
Flax logits shape: (2, 64, 1103), PyTorch logits shape: torch.Size([2, 64, 1103])
|
36 |
+
β
Difference between Flax and PyTorch is 3.725290298461914e-09 (< 0.01)
|
37 |
+
--------------------------Checking losses match--------------------------
|
38 |
+
Flax loss: 7.006012916564941, PyTorch loss: 7.006012916564941
|
39 |
+
β
Difference between Flax and PyTorch is 0.0 (< 0.01)
|
40 |
+
--------------------------Checking gradients match--------------------------
|
41 |
+
β
All grads pass
|
42 |
+
--------------------------Checking rel gradients match--------------------------
|
43 |
+
β
All rel grads pass
|
44 |
+
=========================================
|
45 |
+
Check hf-internal-testing/tiny-random-bart ...
|
46 |
+
--------------------------Checking logits match--------------------------
|
47 |
+
Flax logits shape: (2, 64, 1000), PyTorch logits shape: torch.Size([2, 64, 1000])
|
48 |
+
β
Difference between Flax and PyTorch is 8.940696716308594e-08 (< 0.01)
|
49 |
+
--------------------------Checking losses match--------------------------
|
50 |
+
Flax loss: 6.919522285461426, PyTorch loss: 6.919522285461426
|
51 |
+
β
Difference between Flax and PyTorch is 0.0 (< 0.01)
|
52 |
+
--------------------------Checking gradients match--------------------------
|
53 |
+
β
All grads pass
|
54 |
+
--------------------------Checking rel gradients match--------------------------
|
55 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.0.
|
56 |
+
β Layer ('model', 'decoder', 'layers', '0', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 1.1293364247239035e-13 and flax grad norm 7.444291358479557e-14.
|
57 |
+
β Layer ('model', 'decoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.9028742882613858e-13 and flax grad norm 1.0847509820726894e-13.
|
58 |
+
β Layer ('model', 'decoder', 'layers', '1', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 1.0747876384459981e-13 and flax grad norm 1.1924105637346055e-13.
|
59 |
+
β Layer ('model', 'decoder', 'layers', '1', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.0553104032074165e-13 and flax grad norm 2.416926793251395e-13.
|
60 |
+
β Layer ('model', 'encoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.6505221806215232e-16 and flax grad norm 8.704786207109524e-17.
|
61 |
+
β Layer ('model', 'encoder', 'layers', '1', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.145616114665214e-16 and flax grad norm 1.6750639014770325e-16.
|
62 |
+
=========================================
|
after_fix_pretrained_log.txt
ADDED
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Check roberta-base ...
|
2 |
+
--------------------------Checking logits match--------------------------
|
3 |
+
Flax logits shape: (2, 64, 50265), PyTorch logits shape: torch.Size([2, 64, 50265])
|
4 |
+
β
Difference between Flax and PyTorch is 0.00013017654418945312 (< 0.01)
|
5 |
+
--------------------------Checking losses match--------------------------
|
6 |
+
Flax loss: 14.801228523254395, PyTorch loss: 14.801219940185547
|
7 |
+
β
Difference between Flax and PyTorch is 8.58306884765625e-06 (< 0.01)
|
8 |
+
--------------------------Checking gradients match--------------------------
|
9 |
+
β
All grads pass
|
10 |
+
--------------------------Checking rel gradients match--------------------------
|
11 |
+
β Layer ('roberta', 'encoder', 'layer', '0', 'attention', 'self', 'key', 'bias') has PT grad norm 6.889232651019483e-08 and flax grad norm 5.7956174970286156e-08.
|
12 |
+
β Layer ('roberta', 'encoder', 'layer', '1', 'attention', 'self', 'key', 'bias') has PT grad norm 7.026115156349988e-08 and flax grad norm 6.282134989987753e-08.
|
13 |
+
β Layer ('roberta', 'encoder', 'layer', '10', 'attention', 'self', 'key', 'bias') has PT grad norm 2.273949206710313e-08 and flax grad norm 1.8748883334751554e-08.
|
14 |
+
β Layer ('roberta', 'encoder', 'layer', '11', 'attention', 'self', 'key', 'bias') has PT grad norm 2.9379741306456708e-08 and flax grad norm 2.6026933497291793e-08.
|
15 |
+
β Layer ('roberta', 'encoder', 'layer', '2', 'attention', 'self', 'key', 'bias') has PT grad norm 6.197853963385569e-08 and flax grad norm 5.317058082709991e-08.
|
16 |
+
β Layer ('roberta', 'encoder', 'layer', '3', 'attention', 'self', 'key', 'bias') has PT grad norm 7.359258802352997e-08 and flax grad norm 8.573702814373974e-08.
|
17 |
+
β Layer ('roberta', 'encoder', 'layer', '4', 'attention', 'self', 'key', 'bias') has PT grad norm 5.1634213349416314e-08 and flax grad norm 5.744939457485998e-08.
|
18 |
+
β Layer ('roberta', 'encoder', 'layer', '5', 'attention', 'self', 'key', 'bias') has PT grad norm 4.652720519970899e-08 and flax grad norm 6.121346984855336e-08.
|
19 |
+
β Layer ('roberta', 'encoder', 'layer', '6', 'attention', 'self', 'key', 'bias') has PT grad norm 3.8810604507943935e-08 and flax grad norm 4.2490388096894094e-08.
|
20 |
+
β Layer ('roberta', 'encoder', 'layer', '7', 'attention', 'self', 'key', 'bias') has PT grad norm 3.7450202938771326e-08 and flax grad norm 3.219445687818734e-08.
|
21 |
+
β Layer ('roberta', 'encoder', 'layer', '8', 'attention', 'self', 'key', 'bias') has PT grad norm 3.3088259243641005e-08 and flax grad norm 2.6118801343955056e-08.
|
22 |
+
β Layer ('roberta', 'encoder', 'layer', '9', 'attention', 'self', 'key', 'bias') has PT grad norm 2.6417508180998084e-08 and flax grad norm 2.415968225477627e-08.
|
23 |
+
=========================================
|
24 |
+
Check bert-base-cased ...
|
25 |
+
--------------------------Checking logits match--------------------------
|
26 |
+
Flax logits shape: (2, 64, 28996), PyTorch logits shape: torch.Size([2, 64, 28996])
|
27 |
+
β
Difference between Flax and PyTorch is 5.4836273193359375e-05 (< 0.01)
|
28 |
+
--------------------------Checking losses match--------------------------
|
29 |
+
Flax loss: 13.967159271240234, PyTorch loss: 13.967162132263184
|
30 |
+
β
Difference between Flax and PyTorch is 2.86102294921875e-06 (< 0.01)
|
31 |
+
--------------------------Checking gradients match--------------------------
|
32 |
+
β
All grads pass
|
33 |
+
--------------------------Checking rel gradients match--------------------------
|
34 |
+
β Layer ('bert', 'encoder', 'layer', '0', 'attention', 'self', 'key', 'bias') has PT grad norm 8.025740783068613e-08 and flax grad norm 8.381563532111613e-08.
|
35 |
+
β Layer ('bert', 'encoder', 'layer', '1', 'attention', 'self', 'key', 'bias') has PT grad norm 7.262840284738559e-08 and flax grad norm 5.0372555904232286e-08.
|
36 |
+
β Layer ('bert', 'encoder', 'layer', '10', 'attention', 'self', 'key', 'bias') has PT grad norm 2.6523425233904163e-08 and flax grad norm 2.7082945663892133e-08.
|
37 |
+
β Layer ('bert', 'encoder', 'layer', '11', 'attention', 'self', 'key', 'bias') has PT grad norm 2.9038789151059063e-08 and flax grad norm 3.3138192634396546e-08.
|
38 |
+
β Layer ('bert', 'encoder', 'layer', '2', 'attention', 'self', 'key', 'bias') has PT grad norm 5.880680831182872e-08 and flax grad norm 5.04786008548308e-08.
|
39 |
+
β Layer ('bert', 'encoder', 'layer', '3', 'attention', 'self', 'key', 'bias') has PT grad norm 4.705585965325554e-08 and flax grad norm 4.983893475696277e-08.
|
40 |
+
β Layer ('bert', 'encoder', 'layer', '4', 'attention', 'self', 'key', 'bias') has PT grad norm 6.595875134962625e-08 and flax grad norm 5.823812543326312e-08.
|
41 |
+
β Layer ('bert', 'encoder', 'layer', '5', 'attention', 'self', 'key', 'bias') has PT grad norm 4.716540402682767e-08 and flax grad norm 6.053270595884896e-08.
|
42 |
+
β Layer ('bert', 'encoder', 'layer', '6', 'attention', 'self', 'key', 'bias') has PT grad norm 5.4432636176215965e-08 and flax grad norm 4.0700697923057305e-08.
|
43 |
+
β Layer ('bert', 'encoder', 'layer', '8', 'attention', 'self', 'key', 'bias') has PT grad norm 4.059621971919114e-08 and flax grad norm 4.575255374561493e-08.
|
44 |
+
β Layer ('bert', 'encoder', 'layer', '9', 'attention', 'self', 'key', 'bias') has PT grad norm 2.9032529269557017e-08 and flax grad norm 2.659336217902819e-08.
|
45 |
+
=========================================
|
46 |
+
Check t5-small ...
|
47 |
+
--------------------------Checking logits match--------------------------
|
48 |
+
Flax logits shape: (2, 64, 32128), PyTorch logits shape: torch.Size([2, 64, 32128])
|
49 |
+
β
Difference between Flax and PyTorch is 7.62939453125e-05 (< 0.01)
|
50 |
+
--------------------------Checking losses match--------------------------
|
51 |
+
Flax loss: 20.534835815429688, PyTorch loss: 20.534835815429688
|
52 |
+
β
Difference between Flax and PyTorch is 0.0 (< 0.01)
|
53 |
+
--------------------------Checking gradients match--------------------------
|
54 |
+
β
All grads pass
|
55 |
+
--------------------------Checking rel gradients match--------------------------
|
56 |
+
β
All rel grads pass
|
57 |
+
=========================================
|
58 |
+
Check facebook/bart-large ...
|
59 |
+
--------------------------Checking logits match--------------------------
|
60 |
+
Flax logits shape: (2, 64, 50265), PyTorch logits shape: torch.Size([2, 64, 50265])
|
61 |
+
β
Difference between Flax and PyTorch is 0.0004191398620605469 (< 0.01)
|
62 |
+
--------------------------Checking losses match--------------------------
|
63 |
+
Flax loss: 13.993148803710938, PyTorch loss: 13.993138313293457
|
64 |
+
β
Difference between Flax and PyTorch is 1.049041748046875e-05 (< 0.01)
|
65 |
+
--------------------------Checking gradients match--------------------------
|
66 |
+
β Layer ('model', 'decoder', 'layers', '0', 'fc1', 'kernel') has PT grad norm 11.655710220336914 and flax grad norm 11.6015625.
|
67 |
+
β Layer ('model', 'decoder', 'layers', '0', 'fc2', 'kernel') has PT grad norm 7.740886211395264 and flax grad norm 7.71484375.
|
68 |
+
β Layer ('model', 'decoder', 'layers', '1', 'fc1', 'kernel') has PT grad norm 6.798306465148926 and flax grad norm 6.765625.
|
69 |
+
β Layer ('model', 'decoder', 'layers', '1', 'fc2', 'kernel') has PT grad norm 7.071859836578369 and flax grad norm 7.05078125.
|
70 |
+
β Layer ('model', 'decoder', 'layers', '10', 'fc1', 'kernel') has PT grad norm 16.904926300048828 and flax grad norm 16.859375.
|
71 |
+
β Layer ('model', 'decoder', 'layers', '10', 'fc2', 'kernel') has PT grad norm 6.783661842346191 and flax grad norm 6.765625.
|
72 |
+
β Layer ('model', 'decoder', 'layers', '10', 'self_attn', 'v_proj', 'kernel') has PT grad norm 6.97633171081543 and flax grad norm 6.96484375.
|
73 |
+
β Layer ('model', 'decoder', 'layers', '11', 'fc1', 'kernel') has PT grad norm 13.733073234558105 and flax grad norm 13.6875.
|
74 |
+
β Layer ('model', 'decoder', 'layers', '11', 'fc2', 'kernel') has PT grad norm 6.311193466186523 and flax grad norm 6.29296875.
|
75 |
+
β Layer ('model', 'decoder', 'layers', '2', 'fc1', 'kernel') has PT grad norm 6.043461322784424 and flax grad norm 6.01171875.
|
76 |
+
β Layer ('model', 'decoder', 'layers', '2', 'fc2', 'kernel') has PT grad norm 8.091109275817871 and flax grad norm 8.0703125.
|
77 |
+
β Layer ('model', 'decoder', 'layers', '3', 'fc1', 'kernel') has PT grad norm 6.561250686645508 and flax grad norm 6.52734375.
|
78 |
+
β Layer ('model', 'decoder', 'layers', '3', 'fc2', 'kernel') has PT grad norm 8.535536766052246 and flax grad norm 8.5.
|
79 |
+
β Layer ('model', 'decoder', 'layers', '4', 'fc1', 'kernel') has PT grad norm 5.882389545440674 and flax grad norm 5.859375.
|
80 |
+
β Layer ('model', 'decoder', 'layers', '4', 'fc2', 'kernel') has PT grad norm 8.772762298583984 and flax grad norm 8.75.
|
81 |
+
β Layer ('model', 'decoder', 'layers', '5', 'fc1', 'kernel') has PT grad norm 4.559173107147217 and flax grad norm 4.5390625.
|
82 |
+
β Layer ('model', 'decoder', 'layers', '5', 'fc2', 'kernel') has PT grad norm 7.053295612335205 and flax grad norm 7.03515625.
|
83 |
+
β Layer ('model', 'decoder', 'layers', '6', 'fc1', 'kernel') has PT grad norm 4.724750995635986 and flax grad norm 4.703125.
|
84 |
+
β Layer ('model', 'decoder', 'layers', '6', 'fc2', 'kernel') has PT grad norm 6.6051740646362305 and flax grad norm 6.5859375.
|
85 |
+
β Layer ('model', 'decoder', 'layers', '7', 'fc1', 'kernel') has PT grad norm 3.9028773307800293 and flax grad norm 3.884765625.
|
86 |
+
β Layer ('model', 'decoder', 'layers', '7', 'fc2', 'kernel') has PT grad norm 6.16121244430542 and flax grad norm 6.14453125.
|
87 |
+
β Layer ('model', 'decoder', 'layers', '8', 'fc1', 'kernel') has PT grad norm 3.8737242221832275 and flax grad norm 3.85546875.
|
88 |
+
β Layer ('model', 'decoder', 'layers', '8', 'fc2', 'kernel') has PT grad norm 6.476221084594727 and flax grad norm 6.45703125.
|
89 |
+
β Layer ('model', 'decoder', 'layers', '9', 'fc1', 'kernel') has PT grad norm 6.240624904632568 and flax grad norm 6.203125.
|
90 |
+
β Layer ('model', 'decoder', 'layers', '9', 'fc2', 'kernel') has PT grad norm 6.872060775756836 and flax grad norm 6.8515625.
|
91 |
+
β Layer ('model', 'encoder', 'layers', '0', 'fc1', 'kernel') has PT grad norm 18.140913009643555 and flax grad norm 18.0625.
|
92 |
+
β Layer ('model', 'encoder', 'layers', '0', 'fc2', 'kernel') has PT grad norm 13.278300285339355 and flax grad norm 13.2265625.
|
93 |
+
β Layer ('model', 'encoder', 'layers', '0', 'self_attn', 'v_proj', 'kernel') has PT grad norm 13.971784591674805 and flax grad norm 13.9609375.
|
94 |
+
β Layer ('model', 'encoder', 'layers', '1', 'fc1', 'kernel') has PT grad norm 9.453910827636719 and flax grad norm 9.4140625.
|
95 |
+
β Layer ('model', 'encoder', 'layers', '1', 'fc2', 'kernel') has PT grad norm 5.271415710449219 and flax grad norm 5.25.
|
96 |
+
β Layer ('model', 'encoder', 'layers', '10', 'fc1', 'kernel') has PT grad norm 13.269391059875488 and flax grad norm 13.2109375.
|
97 |
+
β Layer ('model', 'encoder', 'layers', '10', 'fc2', 'kernel') has PT grad norm 15.780173301696777 and flax grad norm 15.734375.
|
98 |
+
β Layer ('model', 'encoder', 'layers', '10', 'self_attn', 'v_proj', 'kernel') has PT grad norm 21.07574462890625 and flax grad norm 21.0625.
|
99 |
+
β Layer ('model', 'encoder', 'layers', '11', 'fc1', 'kernel') has PT grad norm 16.847095489501953 and flax grad norm 16.765625.
|
100 |
+
β Layer ('model', 'encoder', 'layers', '11', 'fc2', 'kernel') has PT grad norm 17.480010986328125 and flax grad norm 17.421875.
|
101 |
+
β Layer ('model', 'encoder', 'layers', '11', 'self_attn', 'out_proj', 'kernel') has PT grad norm 26.91538429260254 and flax grad norm 26.890625.
|
102 |
+
β Layer ('model', 'encoder', 'layers', '11', 'self_attn', 'v_proj', 'kernel') has PT grad norm 26.706096649169922 and flax grad norm 26.6875.
|
103 |
+
β Layer ('model', 'encoder', 'layers', '2', 'fc1', 'kernel') has PT grad norm 6.756587505340576 and flax grad norm 6.7265625.
|
104 |
+
β Layer ('model', 'encoder', 'layers', '2', 'fc2', 'kernel') has PT grad norm 5.000077724456787 and flax grad norm 4.98046875.
|
105 |
+
β Layer ('model', 'encoder', 'layers', '2', 'self_attn', 'out_proj', 'kernel') has PT grad norm 18.72007942199707 and flax grad norm 18.703125.
|
106 |
+
β Layer ('model', 'encoder', 'layers', '3', 'fc1', 'kernel') has PT grad norm 9.560458183288574 and flax grad norm 9.515625.
|
107 |
+
β Layer ('model', 'encoder', 'layers', '3', 'fc2', 'kernel') has PT grad norm 8.892805099487305 and flax grad norm 8.859375.
|
108 |
+
β Layer ('model', 'encoder', 'layers', '4', 'fc1', 'kernel') has PT grad norm 8.845908164978027 and flax grad norm 8.8046875.
|
109 |
+
β Layer ('model', 'encoder', 'layers', '4', 'fc2', 'kernel') has PT grad norm 8.670329093933105 and flax grad norm 8.640625.
|
110 |
+
β Layer ('model', 'encoder', 'layers', '4', 'self_attn', 'out_proj', 'kernel') has PT grad norm 18.135663986206055 and flax grad norm 18.125.
|
111 |
+
β Layer ('model', 'encoder', 'layers', '5', 'fc1', 'kernel') has PT grad norm 10.071086883544922 and flax grad norm 10.03125.
|
112 |
+
β Layer ('model', 'encoder', 'layers', '5', 'fc2', 'kernel') has PT grad norm 9.528592109680176 and flax grad norm 9.5.
|
113 |
+
β Layer ('model', 'encoder', 'layers', '5', 'self_attn', 'out_proj', 'kernel') has PT grad norm 16.202266693115234 and flax grad norm 16.1875.
|
114 |
+
β Layer ('model', 'encoder', 'layers', '5', 'self_attn', 'v_proj', 'kernel') has PT grad norm 17.79180908203125 and flax grad norm 17.78125.
|
115 |
+
β Layer ('model', 'encoder', 'layers', '6', 'fc1', 'kernel') has PT grad norm 12.527167320251465 and flax grad norm 12.46875.
|
116 |
+
β Layer ('model', 'encoder', 'layers', '6', 'fc2', 'kernel') has PT grad norm 11.495430946350098 and flax grad norm 11.453125.
|
117 |
+
β Layer ('model', 'encoder', 'layers', '6', 'self_attn', 'v_proj', 'kernel') has PT grad norm 18.312782287597656 and flax grad norm 18.296875.
|
118 |
+
β Layer ('model', 'encoder', 'layers', '7', 'fc1', 'kernel') has PT grad norm 11.963201522827148 and flax grad norm 11.9140625.
|
119 |
+
β Layer ('model', 'encoder', 'layers', '7', 'fc2', 'kernel') has PT grad norm 13.052857398986816 and flax grad norm 13.0078125.
|
120 |
+
β Layer ('model', 'encoder', 'layers', '7', 'self_attn', 'out_proj', 'kernel') has PT grad norm 17.2364501953125 and flax grad norm 17.21875.
|
121 |
+
β Layer ('model', 'encoder', 'layers', '7', 'self_attn', 'v_proj', 'kernel') has PT grad norm 18.88938331604004 and flax grad norm 18.875.
|
122 |
+
β Layer ('model', 'encoder', 'layers', '8', 'fc1', 'kernel') has PT grad norm 11.773221969604492 and flax grad norm 11.71875.
|
123 |
+
β Layer ('model', 'encoder', 'layers', '8', 'fc2', 'kernel') has PT grad norm 14.441213607788086 and flax grad norm 14.3984375.
|
124 |
+
β Layer ('model', 'encoder', 'layers', '8', 'self_attn', 'out_proj', 'kernel') has PT grad norm 19.045316696166992 and flax grad norm 19.03125.
|
125 |
+
β Layer ('model', 'encoder', 'layers', '8', 'self_attn', 'v_proj', 'kernel') has PT grad norm 18.37466049194336 and flax grad norm 18.359375.
|
126 |
+
β Layer ('model', 'encoder', 'layers', '9', 'fc1', 'kernel') has PT grad norm 12.223063468933105 and flax grad norm 12.1640625.
|
127 |
+
β Layer ('model', 'encoder', 'layers', '9', 'fc2', 'kernel') has PT grad norm 15.896522521972656 and flax grad norm 15.8515625.
|
128 |
+
--------------------------Checking rel gradients match--------------------------
|
129 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.0.
|
130 |
+
β Layer ('model', 'decoder', 'layers', '0', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 8.274865592738934e-08 and flax grad norm 0.0.
|
131 |
+
β Layer ('model', 'decoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 3.530680814378684e-08 and flax grad norm 0.0.
|
132 |
+
β Layer ('model', 'decoder', 'layers', '1', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 9.156278935051887e-08 and flax grad norm 0.0.
|
133 |
+
β Layer ('model', 'decoder', 'layers', '1', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.7762926180230352e-08 and flax grad norm 0.0.
|
134 |
+
β Layer ('model', 'decoder', 'layers', '10', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.314587632668008e-08 and flax grad norm 0.0.
|
135 |
+
β Layer ('model', 'decoder', 'layers', '10', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.0350275303494527e-08 and flax grad norm 0.0.
|
136 |
+
β Layer ('model', 'decoder', 'layers', '11', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.4003222520718737e-08 and flax grad norm 0.0.
|
137 |
+
β Layer ('model', 'decoder', 'layers', '11', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.0850777165671843e-08 and flax grad norm 0.0.
|
138 |
+
β Layer ('model', 'decoder', 'layers', '2', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 8.753153935003866e-08 and flax grad norm 0.0.
|
139 |
+
β Layer ('model', 'decoder', 'layers', '2', 'self_attn', 'k_proj', 'bias') has PT grad norm 3.340263532436438e-08 and flax grad norm 0.0.
|
140 |
+
β Layer ('model', 'decoder', 'layers', '3', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 7.864576190286243e-08 and flax grad norm 0.0.
|
141 |
+
β Layer ('model', 'decoder', 'layers', '3', 'self_attn', 'k_proj', 'bias') has PT grad norm 3.161582284860742e-08 and flax grad norm 0.0.
|
142 |
+
β Layer ('model', 'decoder', 'layers', '4', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 3.1698593971896116e-08 and flax grad norm 0.0.
|
143 |
+
β Layer ('model', 'decoder', 'layers', '4', 'self_attn', 'k_proj', 'bias') has PT grad norm 3.397210690536667e-08 and flax grad norm 0.0.
|
144 |
+
β Layer ('model', 'decoder', 'layers', '5', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 4.269594100492213e-08 and flax grad norm 0.0.
|
145 |
+
β Layer ('model', 'decoder', 'layers', '5', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.3111518032692402e-08 and flax grad norm 0.0.
|
146 |
+
β Layer ('model', 'decoder', 'layers', '6', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 3.222339373110117e-08 and flax grad norm 0.0.
|
147 |
+
β Layer ('model', 'decoder', 'layers', '6', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.2391466458770992e-08 and flax grad norm 0.0.
|
148 |
+
β Layer ('model', 'decoder', 'layers', '7', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.4886496419185278e-08 and flax grad norm 0.0.
|
149 |
+
β Layer ('model', 'decoder', 'layers', '7', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.1538945798861278e-08 and flax grad norm 0.0.
|
150 |
+
β Layer ('model', 'decoder', 'layers', '8', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.3322632713984603e-08 and flax grad norm 0.0.
|
151 |
+
β Layer ('model', 'decoder', 'layers', '8', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.525707205018989e-08 and flax grad norm 0.0.
|
152 |
+
β Layer ('model', 'decoder', 'layers', '9', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.3752901867624132e-08 and flax grad norm 0.0.
|
153 |
+
β Layer ('model', 'decoder', 'layers', '9', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.4804857784156411e-08 and flax grad norm 0.0.
|
154 |
+
β Layer ('model', 'encoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 8.215602775862862e-08 and flax grad norm 0.0.
|
155 |
+
β Layer ('model', 'encoder', 'layers', '1', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.3884370275718538e-07 and flax grad norm 0.0.
|
156 |
+
β Layer ('model', 'encoder', 'layers', '10', 'self_attn', 'k_proj', 'bias') has PT grad norm 6.668184937552724e-08 and flax grad norm 0.0.
|
157 |
+
β Layer ('model', 'encoder', 'layers', '11', 'self_attn', 'k_proj', 'bias') has PT grad norm 8.486110658623147e-08 and flax grad norm 0.0.
|
158 |
+
β Layer ('model', 'encoder', 'layers', '2', 'self_attn', 'k_proj', 'bias') has PT grad norm 7.912614563565512e-08 and flax grad norm 0.0.
|
159 |
+
β Layer ('model', 'encoder', 'layers', '3', 'self_attn', 'k_proj', 'bias') has PT grad norm 7.959246062227976e-08 and flax grad norm 0.0.
|
160 |
+
β Layer ('model', 'encoder', 'layers', '4', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.1111747255654336e-07 and flax grad norm 0.0.
|
161 |
+
β Layer ('model', 'encoder', 'layers', '5', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.0121711824240265e-07 and flax grad norm 0.0.
|
162 |
+
β Layer ('model', 'encoder', 'layers', '6', 'self_attn', 'k_proj', 'bias') has PT grad norm 7.735735607639072e-08 and flax grad norm 0.0.
|
163 |
+
β Layer ('model', 'encoder', 'layers', '7', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.0352330548357713e-07 and flax grad norm 0.0.
|
164 |
+
β Layer ('model', 'encoder', 'layers', '8', 'self_attn', 'k_proj', 'bias') has PT grad norm 8.3155640595578e-08 and flax grad norm 0.0.
|
165 |
+
β Layer ('model', 'encoder', 'layers', '9', 'self_attn', 'k_proj', 'bias') has PT grad norm 6.08824493042448e-08 and flax grad norm 0.0.
|
166 |
+
=========================================
|
167 |
+
Check facebook/bart-large-cnn ...
|
168 |
+
--------------------------Checking logits match--------------------------
|
169 |
+
Flax logits shape: (2, 64, 50264), PyTorch logits shape: torch.Size([2, 64, 50264])
|
170 |
+
β
Difference between Flax and PyTorch is 0.0003502368927001953 (< 0.01)
|
171 |
+
--------------------------Checking losses match--------------------------
|
172 |
+
Flax loss: 13.418181419372559, PyTorch loss: 13.418176651000977
|
173 |
+
β
Difference between Flax and PyTorch is 4.76837158203125e-06 (< 0.01)
|
174 |
+
--------------------------Checking gradients match--------------------------
|
175 |
+
β
All grads pass
|
176 |
+
--------------------------Checking rel gradients match--------------------------
|
177 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.0.
|
178 |
+
β Layer ('model', 'decoder', 'layers', '0', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 3.5387660091146245e-07 and flax grad norm 4.874667069998395e-07.
|
179 |
+
β Layer ('model', 'decoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 6.254911966152576e-08 and flax grad norm 6.927437112835833e-08.
|
180 |
+
β Layer ('model', 'decoder', 'layers', '1', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.0864062832970376e-07 and flax grad norm 1.8239754240312323e-07.
|
181 |
+
β Layer ('model', 'decoder', 'layers', '1', 'self_attn', 'k_proj', 'bias') has PT grad norm 6.265895535761956e-08 and flax grad norm 6.79184637419894e-08.
|
182 |
+
β Layer ('model', 'decoder', 'layers', '10', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.3348372124587513e-08 and flax grad norm 1.9192864186834413e-08.
|
183 |
+
β Layer ('model', 'decoder', 'layers', '10', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.108701835368265e-08 and flax grad norm 1.938536442480654e-08.
|
184 |
+
β Layer ('model', 'decoder', 'layers', '11', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.4383044916476138e-08 and flax grad norm 1.890670375814807e-08.
|
185 |
+
β Layer ('model', 'decoder', 'layers', '11', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.4334801790027996e-08 and flax grad norm 1.3059753278810149e-08.
|
186 |
+
β Layer ('model', 'decoder', 'layers', '2', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.0973679681901558e-07 and flax grad norm 2.6336286396144715e-07.
|
187 |
+
β Layer ('model', 'decoder', 'layers', '2', 'self_attn', 'k_proj', 'bias') has PT grad norm 8.528008521579977e-08 and flax grad norm 1.286241939624233e-07.
|
188 |
+
β Layer ('model', 'decoder', 'layers', '3', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 1.4659757141544105e-07 and flax grad norm 1.4895935862568876e-07.
|
189 |
+
β Layer ('model', 'decoder', 'layers', '3', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.2112769809391466e-07 and flax grad norm 1.382354497536653e-07.
|
190 |
+
β Layer ('model', 'decoder', 'layers', '4', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 6.575358924010288e-08 and flax grad norm 7.240216604031957e-08.
|
191 |
+
β Layer ('model', 'decoder', 'layers', '4', 'self_attn', 'k_proj', 'bias') has PT grad norm 5.0567738441031906e-08 and flax grad norm 5.817578241362753e-08.
|
192 |
+
β Layer ('model', 'decoder', 'layers', '5', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.5854723162410664e-07 and flax grad norm 2.9657505251634575e-07.
|
193 |
+
β Layer ('model', 'decoder', 'layers', '5', 'self_attn', 'k_proj', 'bias') has PT grad norm 8.977868048987148e-08 and flax grad norm 8.695727160556999e-08.
|
194 |
+
β Layer ('model', 'decoder', 'layers', '6', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 1.271907592581556e-07 and flax grad norm 1.5420768306739774e-07.
|
195 |
+
β Layer ('model', 'decoder', 'layers', '6', 'self_attn', 'k_proj', 'bias') has PT grad norm 3.32223777377294e-08 and flax grad norm 2.9034252335691235e-08.
|
196 |
+
β Layer ('model', 'decoder', 'layers', '7', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.932817011469524e-08 and flax grad norm 2.990202219166349e-08.
|
197 |
+
β Layer ('model', 'decoder', 'layers', '7', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.2563149570942187e-08 and flax grad norm 1.903124946522894e-08.
|
198 |
+
β Layer ('model', 'decoder', 'layers', '8', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.1031839381180362e-08 and flax grad norm 1.922012415889185e-08.
|
199 |
+
β Layer ('model', 'decoder', 'layers', '8', 'self_attn', 'k_proj', 'bias') has PT grad norm 1.9617312219111227e-08 and flax grad norm 1.901776514046105e-08.
|
200 |
+
β Layer ('model', 'encoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 9.498415209918676e-08 and flax grad norm 1.0714285281210323e-07.
|
201 |
+
β Layer ('model', 'encoder', 'layers', '1', 'self_attn', 'k_proj', 'bias') has PT grad norm 4.2883741002697207e-08 and flax grad norm 3.708849050099161e-08.
|
202 |
+
β Layer ('model', 'encoder', 'layers', '10', 'self_attn', 'k_proj', 'bias') has PT grad norm 6.720534884152585e-08 and flax grad norm 5.363398614122161e-08.
|
203 |
+
β Layer ('model', 'encoder', 'layers', '11', 'self_attn', 'k_proj', 'bias') has PT grad norm 7.474503149751399e-08 and flax grad norm 7.271213320336756e-08.
|
204 |
+
β Layer ('model', 'encoder', 'layers', '2', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.8607770374833308e-08 and flax grad norm 2.428515344377047e-08.
|
205 |
+
β Layer ('model', 'encoder', 'layers', '3', 'self_attn', 'k_proj', 'bias') has PT grad norm 6.400713203902342e-08 and flax grad norm 5.387828849734433e-08.
|
206 |
+
β Layer ('model', 'encoder', 'layers', '4', 'self_attn', 'k_proj', 'bias') has PT grad norm 4.873627545975978e-08 and flax grad norm 4.757723104376055e-08.
|
207 |
+
β Layer ('model', 'encoder', 'layers', '5', 'self_attn', 'k_proj', 'bias') has PT grad norm 5.0619281211083944e-08 and flax grad norm 4.67279193117065e-08.
|
208 |
+
β Layer ('model', 'encoder', 'layers', '6', 'self_attn', 'k_proj', 'bias') has PT grad norm 5.6844903895125753e-08 and flax grad norm 6.739185920423552e-08.
|
209 |
+
β Layer ('model', 'encoder', 'layers', '7', 'self_attn', 'k_proj', 'bias') has PT grad norm 5.603576624935158e-08 and flax grad norm 5.457893337279529e-08.
|
210 |
+
β Layer ('model', 'encoder', 'layers', '8', 'self_attn', 'k_proj', 'bias') has PT grad norm 5.864935914701164e-08 and flax grad norm 6.345069891722233e-08.
|
211 |
+
β Layer ('model', 'encoder', 'layers', '9', 'self_attn', 'k_proj', 'bias') has PT grad norm 6.470781244161117e-08 and flax grad norm 6.696199505995537e-08.
|
212 |
+
=========================================
|
before_fix_log.txt
ADDED
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
=========================================
|
2 |
+
Check hf-internal-testing/tiny-random-bart ...
|
3 |
+
--------------------------Checking logits match--------------------------
|
4 |
+
Flax logits shape: (2, 64, 1000), PyTorch logits shape: torch.Size([2, 64, 1000])
|
5 |
+
β
Difference between Flax and PyTorch is 8.940696716308594e-08 (< 0.01)
|
6 |
+
--------------------------Checking losses match--------------------------
|
7 |
+
Flax loss: 6.923163414001465, PyTorch loss: 6.923163414001465
|
8 |
+
β
Difference between Flax and PyTorch is 0.0 (< 0.01)
|
9 |
+
--------------------------Checking gradients match--------------------------
|
10 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.09163407981395721.
|
11 |
+
--------------------------Checking rel gradients match--------------------------
|
12 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.09163407981395721.
|
13 |
+
β Layer ('model', 'decoder', 'layers', '0', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 9.3028212357852e-14 and flax grad norm 1.6552796459901042e-13.
|
14 |
+
...
|
15 |
+
=========================================
|
before_fix_pretrained_log.txt
ADDED
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
=========================================
|
2 |
+
Check facebook/bart-large ...
|
3 |
+
--------------------------Checking logits match--------------------------
|
4 |
+
Flax logits shape: (2, 64, 50265), PyTorch logits shape: torch.Size([2, 64, 50265])
|
5 |
+
β
Difference between Flax and PyTorch is 0.00039315223693847656 (< 0.01)
|
6 |
+
--------------------------Checking losses match--------------------------
|
7 |
+
Flax loss: 15.027304649353027, PyTorch loss: 15.027304649353027
|
8 |
+
β
Difference between Flax and PyTorch is 0.0 (< 0.01)
|
9 |
+
--------------------------Checking gradients match--------------------------
|
10 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.09944064915180206.
|
11 |
+
β Layer ('model', 'decoder', 'layers', '0', 'fc1', 'kernel') has PT grad norm 13.111018180847168 and flax grad norm 13.0546875.
|
12 |
+
β Layer ('model', 'decoder', 'layers', '0', 'fc2', 'kernel') has PT grad norm 8.751346588134766 and flax grad norm 8.71875.
|
13 |
+
...
|
14 |
+
β Layer ('model', 'encoder', 'layers', '0', 'self_attn', 'k_proj', 'kernel') has PT grad norm 18.60892105102539 and flax grad norm 18.59375.
|
15 |
+
...
|
16 |
+
β Layer ('model', 'encoder', 'layers', '0', 'self_attn', 'v_proj', 'kernel') has PT grad norm 96.85579681396484 and flax grad norm 96.8125.
|
17 |
+
...
|
18 |
+
β Layer ('model', 'encoder', 'layers', '1', 'self_attn', 'out_proj', 'kernel') has PT grad norm 199.41278076171875 and flax grad norm 199.25.
|
19 |
+
...
|
20 |
+
--------------------------Checking rel gradients match--------------------------
|
21 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.09944064915180206.
|
22 |
+
β Layer ('model', 'decoder', 'layers', '0', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 1.4212106691502413e-07 and flax grad norm 0.0.
|
23 |
+
β Layer ('model', 'decoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 2.0100719311244575e-08 and flax grad norm 0.0.
|
24 |
+
...
|
25 |
+
=========================================
|
26 |
+
Check facebook/bart-large-cnn ...
|
27 |
+
--------------------------Checking logits match--------------------------
|
28 |
+
Flax logits shape: (2, 64, 50264), PyTorch logits shape: torch.Size([2, 64, 50264])
|
29 |
+
β
Difference between Flax and PyTorch is 0.0001919269561767578 (< 0.01)
|
30 |
+
--------------------------Checking losses match--------------------------
|
31 |
+
Flax loss: 13.262251853942871, PyTorch loss: 13.262249946594238
|
32 |
+
β
Difference between Flax and PyTorch is 1.9073486328125e-06 (< 0.01)
|
33 |
+
--------------------------Checking gradients match--------------------------
|
34 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.09764379262924194.
|
35 |
+
--------------------------Checking rel gradients match--------------------------
|
36 |
+
β Layer ('final_logits_bias',) has PT grad norm 0.0 and flax grad norm 0.09764379262924194.
|
37 |
+
β Layer ('model', 'decoder', 'layers', '0', 'encoder_attn', 'k_proj', 'bias') has PT grad norm 2.1513474735002092e-07 and flax grad norm 1.5481474235912174e-07.
|
38 |
+
β Layer ('model', 'decoder', 'layers', '0', 'self_attn', 'k_proj', 'bias') has PT grad norm 3.8047311079481005e-08 and flax grad norm 3.508952062247772e-08.
|
39 |
+
...
|
40 |
+
=========================================
|
check_gradients_pt_flax.py
CHANGED
@@ -163,16 +163,12 @@ def compare_grads(model_id, pt_architecture):
|
|
163 |
|
164 |
pt_loss.backward()
|
165 |
|
166 |
-
pt_grad_dict = {k: v.grad
|
|
|
167 |
|
168 |
-
|
169 |
-
if k not in pt_grad_dict:
|
170 |
-
# set any unused parameters to zero in the grad-dict
|
171 |
-
# these won't be compared to the Flax model, but required for loading the PT model from state-dict
|
172 |
-
pt_grad_dict[k] = torch.zeros_like(pt_model.state_dict()[k])
|
173 |
-
pt_model.state_dict()[k] = pt_grad_dict[k]
|
174 |
|
175 |
-
|
176 |
|
177 |
with tempfile.TemporaryDirectory() as tmpdirname:
|
178 |
pt_model.save_pretrained(tmpdirname)
|
163 |
|
164 |
pt_loss.backward()
|
165 |
|
166 |
+
pt_grad_dict = {k: v.grad for k, v in pt_model.named_parameters()}
|
167 |
+
missing_grads = [k for k in pt_model.state_dict().keys() if k not in pt_grad_dict]
|
168 |
|
169 |
+
missing_keys, unexpected_keys = pt_model.load_state_dict(pt_grad_dict, strict=False)
|
|
|
|
|
|
|
|
|
|
|
170 |
|
171 |
+
assert missing_grads == missing_keys, f"Error with either grads {missing_keys} or keys {unexpected_keys}"
|
172 |
|
173 |
with tempfile.TemporaryDirectory() as tmpdirname:
|
174 |
pt_model.save_pretrained(tmpdirname)
|
run_models.sh
CHANGED
@@ -1,8 +1,8 @@
|
|
1 |
#!/usr/bin/env bash
|
2 |
-
|
3 |
-
|
4 |
-
model_ids=("hf-internal-testing/tiny-random-roberta")
|
5 |
-
model_architectures=("RobertaForMaskedLM")
|
6 |
|
7 |
rm -rf log.txt
|
8 |
touch log.txt
|
1 |
#!/usr/bin/env bash
|
2 |
+
model_ids=("roberta-base" "bert-base-cased" "t5-small" "facebook/bart-large" "facebook/bart-large-cnn")
|
3 |
+
model_architectures=("RobertaForMaskedLM" "BertForMaskedLM" "T5ForConditionalGeneration" "BartForConditionalGeneration" "BartForConditionalGeneration" )
|
4 |
+
#model_ids=("hf-internal-testing/tiny-random-roberta" "hf-internal-testing/tiny-random-bert" "hf-internal-testing/tiny-random-t5" "hf-internal-testing/tiny-random-bart")
|
5 |
+
#model_architectures=("RobertaForMaskedLM" "BertForMaskedLM" "T5ForConditionalGeneration" "BartForConditionalGeneration")
|
6 |
|
7 |
rm -rf log.txt
|
8 |
touch log.txt
|