test

generate_stats.py

@@ -8,7 +8,8 @@ import sys
 client = storage.Client()
 
 # Get the bucket
-bucket = client.bucket("nb-t5x-us-central2")
+bucket_name = "nb-t5x-us-central2"
+bucket = client.bucket(bucket_name)
 
 
 checkpoints=["exp1-t5-base-ul2-engvoc","exp2-t5-base-ul2-scandvoc","exp3-t5-base-span-engvoc","exp4-t5-base-span-scandvoc","exp5-t5-base-ul2-scandvoc-full","exp6-t5-base-span-scandvoc-full","exp7-t5-base-ul2-511-scandvoc","exp8-t5-base-span-511-scandvoc","exp9-t5-base-ul2-mt5voc","exp10-t5-base-span-mt5voc","exp11-t5-base-ul2-511-scandvoc-full","exp12-t5-base-span-511-scandvoc-full","exp13-t5-base-ul2-mt5voc-full","exp14-t5-base-span-mt5voc-full","exp15-t5-base-ul2-511-scandvoc-full-scratch","exp16-t5-base-span-511-scandvoc-full-scratch","exp17-t5-small-ul2-mt5voc-scratch","exp18-t5-small-span-mt5voc-scratch","exp19-t5-small-ul2-mt5voc","exp20-t5-small-span-mt5voc","exp21-t5-small-ul2-mt5voc-full","exp22-t5-small-span-mt5voc-full"] 
@@ -45,7 +46,8 @@ not_downloaded = 0
 for file_name in file_names:
     # Get the file
     blob = bucket.get_blob(file_name)
-    
+    print(f'gs://{bucket_name}/{file_name}')
+
     if not blob:
         #print(f"Unable to download {file_name}")
         not_downloaded+=1

stats/average_at_5000.csv

@@ -5,7 +5,7 @@ experiment,experiment_name,pretraining_steps,accuracy,f1_macro,num_experiments
 1,t5-base-ul2-engvoc,1300000,80.56666666666666,80.54410574835832,5
 1,t5-base-ul2-engvoc,1400000,75.36666666666666,73.14064173847524,5
 1,t5-base-ul2-engvoc,1500000,63.96666666666666,56.65949975003067,5
-10,t5-base-span-mt5voc,1000000,72.6875,72.64078351515401,4
+10,t5-base-span-mt5voc,1000000,72.8,72.75950043762255,5
 10,t5-base-span-mt5voc,1100000,83.96666666666667,83.9503840051814,5
 10,t5-base-span-mt5voc,1200000,83.35,83.34139996834791,5
 10,t5-base-span-mt5voc,1300000,83.93333333333334,83.92100424084556,5
@@ -16,7 +16,7 @@ experiment,experiment_name,pretraining_steps,accuracy,f1_macro,num_experiments
 11,t5-base-ul2-511-scandvoc-full,1200000,83.53333333333333,83.5149918633688,5
 11,t5-base-ul2-511-scandvoc-full,1300000,83.61666666666666,83.60361062580016,5
 11,t5-base-ul2-511-scandvoc-full,1400000,83.0,82.9732431830119,5
-11,t5-base-ul2-511-scandvoc-full,1500000,83.54166666666667,83.48306522079274,2
+11,t5-base-ul2-511-scandvoc-full,1500000,84.28333333333333,84.24425188831961,5
 12,t5-base-span-511-scandvoc-full,1000000,76.0,75.98198131672166,5
 12,t5-base-span-511-scandvoc-full,1100000,79.1,78.84782778864289,5
 12,t5-base-span-511-scandvoc-full,1200000,82.46666666666667,82.43126640302947,5
@@ -64,7 +64,7 @@ experiment,experiment_name,pretraining_steps,accuracy,f1_macro,num_experiments
 19,t5-small-ul2-mt5voc,1400000,78.13333333333333,78.07000440858576,5
 19,t5-small-ul2-mt5voc,1500000,77.66666666666666,77.60123555974505,5
 2,t5-base-ul2-scandvoc,1184000,77.6,77.58923655371049,5
-2,t5-base-ul2-scandvoc,1204000,81.27083333333333,81.26140841936191,4
+2,t5-base-ul2-scandvoc,1204000,81.3,81.29084623179868,5
 2,t5-base-ul2-scandvoc,1284000,83.51666666666667,83.45695672244898,5
 2,t5-base-ul2-scandvoc,1300000,83.35,83.32806554148515,5
 2,t5-base-ul2-scandvoc,1400000,84.11666666666667,84.07798589917346,5
@@ -78,9 +78,15 @@ experiment,experiment_name,pretraining_steps,accuracy,f1_macro,num_experiments
 21,t5-small-ul2-mt5voc-full,1000000,68.28333333333333,68.23077615099666,5
 21,t5-small-ul2-mt5voc-full,1100000,77.51666666666668,77.32287928690386,5
 21,t5-small-ul2-mt5voc-full,1200000,76.88333333333334,76.78568232856061,5
-21,t5-small-ul2-mt5voc-full,1300000,77.5625,77.44014671469886,4
-21,t5-small-ul2-mt5voc-full,1400000,78.77083333333333,78.74873465182691,4
-21,t5-small-ul2-mt5voc-full,1500000,79.47916666666666,79.46198316868436,4
+21,t5-small-ul2-mt5voc-full,1300000,77.6,77.50142195059952,5
+21,t5-small-ul2-mt5voc-full,1400000,78.75,78.72162765462778,5
+21,t5-small-ul2-mt5voc-full,1500000,79.3,79.27656707646095,5
+22,t5-small-span-mt5voc-full,1000000,68.05,67.97619798144095,5
+22,t5-small-span-mt5voc-full,1100000,78.03333333333333,78.01703014179279,5
+22,t5-small-span-mt5voc-full,1200000,77.8,77.71168905429123,5
+22,t5-small-span-mt5voc-full,1300000,78.6,78.51600126484362,5
+22,t5-small-span-mt5voc-full,1400000,76.36666666666666,76.21138167505276,5
+22,t5-small-span-mt5voc-full,1500000,79.05,79.02453592993025,5
 3,t5-base-span-engvoc,1184000,74.63333333333333,74.62000410946226,5
 3,t5-base-span-engvoc,1204000,80.98333333333333,80.9740844273007,5
 3,t5-base-span-engvoc,1284000,79.4,79.33323889498027,5

stats/average_at_5000.jsonl

@@ -4,7 +4,7 @@
 {"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1300000,"accuracy":80.5666666667,"f1_macro":80.5441057484,"num_experiments":5}
 {"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1400000,"accuracy":75.3666666667,"f1_macro":73.1406417385,"num_experiments":5}
 {"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1500000,"accuracy":63.9666666667,"f1_macro":56.65949975,"num_experiments":5}
-{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1000000,"accuracy":72.6875,"f1_macro":72.6407835152,"num_experiments":4}
+{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1000000,"accuracy":72.8,"f1_macro":72.7595004376,"num_experiments":5}
 {"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1100000,"accuracy":83.9666666667,"f1_macro":83.9503840052,"num_experiments":5}
 {"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1200000,"accuracy":83.35,"f1_macro":83.3413999683,"num_experiments":5}
 {"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1300000,"accuracy":83.9333333333,"f1_macro":83.9210042408,"num_experiments":5}
@@ -15,7 +15,7 @@
 {"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1200000,"accuracy":83.5333333333,"f1_macro":83.5149918634,"num_experiments":5}
 {"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1300000,"accuracy":83.6166666667,"f1_macro":83.6036106258,"num_experiments":5}
 {"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1400000,"accuracy":83.0,"f1_macro":82.973243183,"num_experiments":5}
-{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1500000,"accuracy":83.5416666667,"f1_macro":83.4830652208,"num_experiments":2}
+{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1500000,"accuracy":84.2833333333,"f1_macro":84.2442518883,"num_experiments":5}
 {"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1000000,"accuracy":76.0,"f1_macro":75.9819813167,"num_experiments":5}
 {"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1100000,"accuracy":79.1,"f1_macro":78.8478277886,"num_experiments":5}
 {"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1200000,"accuracy":82.4666666667,"f1_macro":82.431266403,"num_experiments":5}
@@ -63,7 +63,7 @@
 {"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1400000,"accuracy":78.1333333333,"f1_macro":78.0700044086,"num_experiments":5}
 {"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1500000,"accuracy":77.6666666667,"f1_macro":77.6012355597,"num_experiments":5}
 {"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1184000,"accuracy":77.6,"f1_macro":77.5892365537,"num_experiments":5}
-{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1204000,"accuracy":81.2708333333,"f1_macro":81.2614084194,"num_experiments":4}
+{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1204000,"accuracy":81.3,"f1_macro":81.2908462318,"num_experiments":5}
 {"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1284000,"accuracy":83.5166666667,"f1_macro":83.4569567224,"num_experiments":5}
 {"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1300000,"accuracy":83.35,"f1_macro":83.3280655415,"num_experiments":5}
 {"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1400000,"accuracy":84.1166666667,"f1_macro":84.0779858992,"num_experiments":5}
@@ -77,9 +77,15 @@
 {"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1000000,"accuracy":68.2833333333,"f1_macro":68.230776151,"num_experiments":5}
 {"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1100000,"accuracy":77.5166666667,"f1_macro":77.3228792869,"num_experiments":5}
 {"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1200000,"accuracy":76.8833333333,"f1_macro":76.7856823286,"num_experiments":5}
-{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1300000,"accuracy":77.5625,"f1_macro":77.4401467147,"num_experiments":4}
-{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1400000,"accuracy":78.7708333333,"f1_macro":78.7487346518,"num_experiments":4}
-{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1500000,"accuracy":79.4791666667,"f1_macro":79.4619831687,"num_experiments":4}
+{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1300000,"accuracy":77.6,"f1_macro":77.5014219506,"num_experiments":5}
+{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1400000,"accuracy":78.75,"f1_macro":78.7216276546,"num_experiments":5}
+{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1500000,"accuracy":79.3,"f1_macro":79.2765670765,"num_experiments":5}
+{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1000000,"accuracy":68.05,"f1_macro":67.9761979814,"num_experiments":5}
+{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1100000,"accuracy":78.0333333333,"f1_macro":78.0170301418,"num_experiments":5}
+{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1200000,"accuracy":77.8,"f1_macro":77.7116890543,"num_experiments":5}
+{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1300000,"accuracy":78.6,"f1_macro":78.5160012648,"num_experiments":5}
+{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1400000,"accuracy":76.3666666667,"f1_macro":76.2113816751,"num_experiments":5}
+{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1500000,"accuracy":79.05,"f1_macro":79.0245359299,"num_experiments":5}
 {"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1184000,"accuracy":74.6333333333,"f1_macro":74.6200041095,"num_experiments":5}
 {"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1204000,"accuracy":80.9833333333,"f1_macro":80.9740844273,"num_experiments":5}
 {"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1284000,"accuracy":79.4,"f1_macro":79.333238895,"num_experiments":5}