t5-political-evaluator / stats /average_at_5000.jsonl
pere's picture
final
3bfac5f
{"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1184000,"accuracy":74.2333333333,"f1_macro":74.2301492959,"num_experiments":5}
{"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1204000,"accuracy":80.2666666667,"f1_macro":80.2282359353,"num_experiments":5}
{"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1284000,"accuracy":81.5666666667,"f1_macro":81.562633361,"num_experiments":5}
{"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1300000,"accuracy":80.5666666667,"f1_macro":80.5441057484,"num_experiments":5}
{"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1400000,"accuracy":75.3666666667,"f1_macro":73.1406417385,"num_experiments":5}
{"experiment":"1","experiment_name":"t5-base-ul2-engvoc","pretraining_steps":1500000,"accuracy":63.9666666667,"f1_macro":56.65949975,"num_experiments":5}
{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1000000,"accuracy":72.8,"f1_macro":72.7595004376,"num_experiments":5}
{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1100000,"accuracy":83.9666666667,"f1_macro":83.9503840052,"num_experiments":5}
{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1200000,"accuracy":83.35,"f1_macro":83.3413999683,"num_experiments":5}
{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1300000,"accuracy":83.9333333333,"f1_macro":83.9210042408,"num_experiments":5}
{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1400000,"accuracy":82.6333333333,"f1_macro":82.602222236,"num_experiments":5}
{"experiment":"10","experiment_name":"t5-base-span-mt5voc","pretraining_steps":1500000,"accuracy":76.65,"f1_macro":73.2875255298,"num_experiments":5}
{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1000000,"accuracy":77.95,"f1_macro":77.9365395658,"num_experiments":5}
{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1100000,"accuracy":81.5,"f1_macro":81.48782102,"num_experiments":5}
{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1200000,"accuracy":83.5333333333,"f1_macro":83.5149918634,"num_experiments":5}
{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1300000,"accuracy":83.6166666667,"f1_macro":83.6036106258,"num_experiments":5}
{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1400000,"accuracy":83.0,"f1_macro":82.973243183,"num_experiments":5}
{"experiment":"11","experiment_name":"t5-base-ul2-511-scandvoc-full","pretraining_steps":1500000,"accuracy":84.2833333333,"f1_macro":84.2442518883,"num_experiments":5}
{"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1000000,"accuracy":76.0,"f1_macro":75.9819813167,"num_experiments":5}
{"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1100000,"accuracy":79.1,"f1_macro":78.8478277886,"num_experiments":5}
{"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1200000,"accuracy":82.4666666667,"f1_macro":82.431266403,"num_experiments":5}
{"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1300000,"accuracy":81.9833333333,"f1_macro":81.919477877,"num_experiments":5}
{"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1400000,"accuracy":81.9166666667,"f1_macro":81.835559089,"num_experiments":5}
{"experiment":"12","experiment_name":"t5-base-span-511-scandvoc-full","pretraining_steps":1500000,"accuracy":83.5166666667,"f1_macro":83.4802655372,"num_experiments":5}
{"experiment":"13","experiment_name":"t5-base-ul2-mt5voc-full","pretraining_steps":1000000,"accuracy":72.5,"f1_macro":72.4951550014,"num_experiments":5}
{"experiment":"13","experiment_name":"t5-base-ul2-mt5voc-full","pretraining_steps":1100000,"accuracy":81.3166666667,"f1_macro":81.2497422408,"num_experiments":5}
{"experiment":"13","experiment_name":"t5-base-ul2-mt5voc-full","pretraining_steps":1200000,"accuracy":83.6833333333,"f1_macro":83.6623497558,"num_experiments":5}
{"experiment":"13","experiment_name":"t5-base-ul2-mt5voc-full","pretraining_steps":1300000,"accuracy":83.9166666667,"f1_macro":83.9020937336,"num_experiments":5}
{"experiment":"13","experiment_name":"t5-base-ul2-mt5voc-full","pretraining_steps":1400000,"accuracy":84.3166666667,"f1_macro":84.2794957077,"num_experiments":5}
{"experiment":"13","experiment_name":"t5-base-ul2-mt5voc-full","pretraining_steps":1500000,"accuracy":84.5666666667,"f1_macro":84.5349324831,"num_experiments":5}
{"experiment":"14","experiment_name":"t5-base-span-mt5voc-full","pretraining_steps":1000000,"accuracy":73.1666666667,"f1_macro":73.1562481232,"num_experiments":5}
{"experiment":"14","experiment_name":"t5-base-span-mt5voc-full","pretraining_steps":1100000,"accuracy":84.8,"f1_macro":84.7789487112,"num_experiments":5}
{"experiment":"14","experiment_name":"t5-base-span-mt5voc-full","pretraining_steps":1200000,"accuracy":84.3166666667,"f1_macro":84.2741693196,"num_experiments":5}
{"experiment":"14","experiment_name":"t5-base-span-mt5voc-full","pretraining_steps":1300000,"accuracy":84.8333333333,"f1_macro":84.7973823259,"num_experiments":5}
{"experiment":"14","experiment_name":"t5-base-span-mt5voc-full","pretraining_steps":1400000,"accuracy":85.3166666667,"f1_macro":85.30452274,"num_experiments":5}
{"experiment":"14","experiment_name":"t5-base-span-mt5voc-full","pretraining_steps":1500000,"accuracy":84.8,"f1_macro":84.7518324993,"num_experiments":5}
{"experiment":"15","experiment_name":"t5-base-ul2-511-scandvoc-full-scratch","pretraining_steps":1184000,"accuracy":77.8833333333,"f1_macro":77.8517336748,"num_experiments":5}
{"experiment":"15","experiment_name":"t5-base-ul2-511-scandvoc-full-scratch","pretraining_steps":1284000,"accuracy":83.2666666667,"f1_macro":83.2260572034,"num_experiments":5}
{"experiment":"15","experiment_name":"t5-base-ul2-511-scandvoc-full-scratch","pretraining_steps":1384000,"accuracy":84.6,"f1_macro":84.5542192117,"num_experiments":5}
{"experiment":"15","experiment_name":"t5-base-ul2-511-scandvoc-full-scratch","pretraining_steps":1484000,"accuracy":81.7,"f1_macro":81.6719329395,"num_experiments":5}
{"experiment":"15","experiment_name":"t5-base-ul2-511-scandvoc-full-scratch","pretraining_steps":1500000,"accuracy":85.0666666667,"f1_macro":85.0437769014,"num_experiments":5}
{"experiment":"16","experiment_name":"t5-base-span-511-scandvoc-full-scratch","pretraining_steps":1184000,"accuracy":77.2166666667,"f1_macro":77.1353081977,"num_experiments":5}
{"experiment":"16","experiment_name":"t5-base-span-511-scandvoc-full-scratch","pretraining_steps":1284000,"accuracy":80.1333333333,"f1_macro":80.0082321848,"num_experiments":5}
{"experiment":"16","experiment_name":"t5-base-span-511-scandvoc-full-scratch","pretraining_steps":1384000,"accuracy":74.3333333333,"f1_macro":74.0095205556,"num_experiments":5}
{"experiment":"16","experiment_name":"t5-base-span-511-scandvoc-full-scratch","pretraining_steps":1484000,"accuracy":76.8166666667,"f1_macro":73.4697446568,"num_experiments":5}
{"experiment":"16","experiment_name":"t5-base-span-511-scandvoc-full-scratch","pretraining_steps":1500000,"accuracy":83.5166666667,"f1_macro":83.4714515547,"num_experiments":5}
{"experiment":"17","experiment_name":"t5-small-ul2-mt5voc-scratch","pretraining_steps":100000,"accuracy":82.35,"f1_macro":82.3060608438,"num_experiments":5}
{"experiment":"17","experiment_name":"t5-small-ul2-mt5voc-scratch","pretraining_steps":200000,"accuracy":81.75,"f1_macro":81.6847187053,"num_experiments":5}
{"experiment":"17","experiment_name":"t5-small-ul2-mt5voc-scratch","pretraining_steps":300000,"accuracy":82.8666666667,"f1_macro":82.7826873834,"num_experiments":5}
{"experiment":"17","experiment_name":"t5-small-ul2-mt5voc-scratch","pretraining_steps":400000,"accuracy":82.3166666667,"f1_macro":82.2304443323,"num_experiments":5}
{"experiment":"17","experiment_name":"t5-small-ul2-mt5voc-scratch","pretraining_steps":500000,"accuracy":81.45,"f1_macro":81.3952814501,"num_experiments":5}
{"experiment":"17","experiment_name":"t5-small-ul2-mt5voc-scratch","pretraining_steps":1000000,"accuracy":81.7,"f1_macro":81.6785215558,"num_experiments":5}
{"experiment":"17","experiment_name":"t5-small-ul2-mt5voc-scratch","pretraining_steps":1100000,"accuracy":81.5666666667,"f1_macro":81.5227132021,"num_experiments":5}
{"experiment":"18","experiment_name":"t5-small-span-mt5voc-scratch","pretraining_steps":100000,"accuracy":82.4166666667,"f1_macro":82.3927034099,"num_experiments":5}
{"experiment":"18","experiment_name":"t5-small-span-mt5voc-scratch","pretraining_steps":200000,"accuracy":82.1166666667,"f1_macro":82.0609888058,"num_experiments":5}
{"experiment":"18","experiment_name":"t5-small-span-mt5voc-scratch","pretraining_steps":300000,"accuracy":82.5333333333,"f1_macro":82.4903861292,"num_experiments":5}
{"experiment":"18","experiment_name":"t5-small-span-mt5voc-scratch","pretraining_steps":400000,"accuracy":82.4333333333,"f1_macro":82.4271891298,"num_experiments":5}
{"experiment":"18","experiment_name":"t5-small-span-mt5voc-scratch","pretraining_steps":500000,"accuracy":81.6166666667,"f1_macro":81.5626397569,"num_experiments":5}
{"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1000000,"accuracy":69.3833333333,"f1_macro":69.3460800226,"num_experiments":5}
{"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1100000,"accuracy":76.9,"f1_macro":76.7488158445,"num_experiments":5}
{"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1200000,"accuracy":78.1,"f1_macro":78.0438002105,"num_experiments":5}
{"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1300000,"accuracy":76.2833333333,"f1_macro":76.1019592008,"num_experiments":5}
{"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1400000,"accuracy":78.1333333333,"f1_macro":78.0700044086,"num_experiments":5}
{"experiment":"19","experiment_name":"t5-small-ul2-mt5voc","pretraining_steps":1500000,"accuracy":77.6666666667,"f1_macro":77.6012355597,"num_experiments":5}
{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1184000,"accuracy":77.6,"f1_macro":77.5892365537,"num_experiments":5}
{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1204000,"accuracy":81.3,"f1_macro":81.2908462318,"num_experiments":5}
{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1284000,"accuracy":83.5166666667,"f1_macro":83.4569567224,"num_experiments":5}
{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1300000,"accuracy":83.35,"f1_macro":83.3280655415,"num_experiments":5}
{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1400000,"accuracy":84.1166666667,"f1_macro":84.0779858992,"num_experiments":5}
{"experiment":"2","experiment_name":"t5-base-ul2-scandvoc","pretraining_steps":1500000,"accuracy":84.75,"f1_macro":84.7253606622,"num_experiments":5}
{"experiment":"20","experiment_name":"t5-small-span-mt5voc","pretraining_steps":1000000,"accuracy":67.6666666667,"f1_macro":67.5851653159,"num_experiments":5}
{"experiment":"20","experiment_name":"t5-small-span-mt5voc","pretraining_steps":1100000,"accuracy":77.6833333333,"f1_macro":77.5748501484,"num_experiments":5}
{"experiment":"20","experiment_name":"t5-small-span-mt5voc","pretraining_steps":1200000,"accuracy":71.4166666667,"f1_macro":71.3046914963,"num_experiments":5}
{"experiment":"20","experiment_name":"t5-small-span-mt5voc","pretraining_steps":1300000,"accuracy":76.6666666667,"f1_macro":76.5862722579,"num_experiments":5}
{"experiment":"20","experiment_name":"t5-small-span-mt5voc","pretraining_steps":1400000,"accuracy":75.7833333333,"f1_macro":75.5534321168,"num_experiments":5}
{"experiment":"20","experiment_name":"t5-small-span-mt5voc","pretraining_steps":1500000,"accuracy":72.2333333333,"f1_macro":69.4998772891,"num_experiments":5}
{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1000000,"accuracy":68.2833333333,"f1_macro":68.230776151,"num_experiments":5}
{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1100000,"accuracy":77.5166666667,"f1_macro":77.3228792869,"num_experiments":5}
{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1200000,"accuracy":76.8833333333,"f1_macro":76.7856823286,"num_experiments":5}
{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1300000,"accuracy":77.6,"f1_macro":77.5014219506,"num_experiments":5}
{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1400000,"accuracy":78.75,"f1_macro":78.7216276546,"num_experiments":5}
{"experiment":"21","experiment_name":"t5-small-ul2-mt5voc-full","pretraining_steps":1500000,"accuracy":79.3,"f1_macro":79.2765670765,"num_experiments":5}
{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1000000,"accuracy":68.05,"f1_macro":67.9761979814,"num_experiments":5}
{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1100000,"accuracy":78.0333333333,"f1_macro":78.0170301418,"num_experiments":5}
{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1200000,"accuracy":77.8,"f1_macro":77.7116890543,"num_experiments":5}
{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1300000,"accuracy":78.6,"f1_macro":78.5160012648,"num_experiments":5}
{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1400000,"accuracy":76.3666666667,"f1_macro":76.2113816751,"num_experiments":5}
{"experiment":"22","experiment_name":"t5-small-span-mt5voc-full","pretraining_steps":1500000,"accuracy":79.05,"f1_macro":79.0245359299,"num_experiments":5}
{"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1184000,"accuracy":74.6333333333,"f1_macro":74.6200041095,"num_experiments":5}
{"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1204000,"accuracy":80.9833333333,"f1_macro":80.9740844273,"num_experiments":5}
{"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1284000,"accuracy":79.4,"f1_macro":79.333238895,"num_experiments":5}
{"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1300000,"accuracy":77.7666666667,"f1_macro":77.6803642122,"num_experiments":5}
{"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1400000,"accuracy":78.2,"f1_macro":78.1951653886,"num_experiments":5}
{"experiment":"3","experiment_name":"t5-base-span-engvoc","pretraining_steps":1500000,"accuracy":76.3166666667,"f1_macro":76.2973105322,"num_experiments":5}
{"experiment":"4","experiment_name":"t5-base-span-scandvoc","pretraining_steps":1184000,"accuracy":77.35,"f1_macro":77.3336095988,"num_experiments":5}
{"experiment":"4","experiment_name":"t5-base-span-scandvoc","pretraining_steps":1204000,"accuracy":82.6333333333,"f1_macro":82.6293487369,"num_experiments":5}
{"experiment":"4","experiment_name":"t5-base-span-scandvoc","pretraining_steps":1284000,"accuracy":83.5,"f1_macro":83.4834082879,"num_experiments":5}
{"experiment":"4","experiment_name":"t5-base-span-scandvoc","pretraining_steps":1300000,"accuracy":83.4333333333,"f1_macro":83.4195232466,"num_experiments":5}
{"experiment":"4","experiment_name":"t5-base-span-scandvoc","pretraining_steps":1400000,"accuracy":76.8333333333,"f1_macro":73.4840684203,"num_experiments":5}
{"experiment":"4","experiment_name":"t5-base-span-scandvoc","pretraining_steps":1500000,"accuracy":75.7666666667,"f1_macro":72.4110729975,"num_experiments":5}
{"experiment":"5","experiment_name":"t5-base-ul2-scandvoc-full","pretraining_steps":1184000,"accuracy":77.1333333333,"f1_macro":77.1222860108,"num_experiments":5}
{"experiment":"5","experiment_name":"t5-base-ul2-scandvoc-full","pretraining_steps":1284000,"accuracy":84.9333333333,"f1_macro":84.9195472261,"num_experiments":5}
{"experiment":"5","experiment_name":"t5-base-ul2-scandvoc-full","pretraining_steps":1384000,"accuracy":83.3,"f1_macro":83.2918726525,"num_experiments":5}
{"experiment":"5","experiment_name":"t5-base-ul2-scandvoc-full","pretraining_steps":1484000,"accuracy":86.75,"f1_macro":86.7497325508,"num_experiments":5}
{"experiment":"5","experiment_name":"t5-base-ul2-scandvoc-full","pretraining_steps":1500000,"accuracy":86.05,"f1_macro":86.0381232786,"num_experiments":5}
{"experiment":"6","experiment_name":"t5-base-span-scandvoc-full","pretraining_steps":1184000,"accuracy":78.9333333333,"f1_macro":78.9271489614,"num_experiments":5}
{"experiment":"6","experiment_name":"t5-base-span-scandvoc-full","pretraining_steps":1284000,"accuracy":85.35,"f1_macro":85.335785239,"num_experiments":5}
{"experiment":"6","experiment_name":"t5-base-span-scandvoc-full","pretraining_steps":1384000,"accuracy":85.8166666667,"f1_macro":85.8080171241,"num_experiments":5}
{"experiment":"6","experiment_name":"t5-base-span-scandvoc-full","pretraining_steps":1484000,"accuracy":85.25,"f1_macro":85.2243496772,"num_experiments":5}
{"experiment":"6","experiment_name":"t5-base-span-scandvoc-full","pretraining_steps":1500000,"accuracy":85.2166666667,"f1_macro":85.1992014678,"num_experiments":5}
{"experiment":"7","experiment_name":"t5-base-ul2-511-scandvoc","pretraining_steps":1000000,"accuracy":73.8333333333,"f1_macro":73.8061730252,"num_experiments":5}
{"experiment":"7","experiment_name":"t5-base-ul2-511-scandvoc","pretraining_steps":1100000,"accuracy":76.0166666667,"f1_macro":75.9942316465,"num_experiments":5}
{"experiment":"7","experiment_name":"t5-base-ul2-511-scandvoc","pretraining_steps":1200000,"accuracy":77.7666666667,"f1_macro":77.7030816078,"num_experiments":5}
{"experiment":"7","experiment_name":"t5-base-ul2-511-scandvoc","pretraining_steps":1300000,"accuracy":76.65,"f1_macro":76.6085376539,"num_experiments":5}
{"experiment":"7","experiment_name":"t5-base-ul2-511-scandvoc","pretraining_steps":1400000,"accuracy":78.55,"f1_macro":78.5337238563,"num_experiments":5}
{"experiment":"7","experiment_name":"t5-base-ul2-511-scandvoc","pretraining_steps":1500000,"accuracy":78.7833333333,"f1_macro":78.755274257,"num_experiments":5}
{"experiment":"8","experiment_name":"t5-base-span-511-scandvoc","pretraining_steps":1000000,"accuracy":75.45,"f1_macro":75.4303591458,"num_experiments":5}
{"experiment":"8","experiment_name":"t5-base-span-511-scandvoc","pretraining_steps":1100000,"accuracy":76.75,"f1_macro":76.4553139363,"num_experiments":5}
{"experiment":"8","experiment_name":"t5-base-span-511-scandvoc","pretraining_steps":1200000,"accuracy":72.2833333333,"f1_macro":68.9063221254,"num_experiments":5}
{"experiment":"8","experiment_name":"t5-base-span-511-scandvoc","pretraining_steps":1300000,"accuracy":56.45,"f1_macro":45.5065610464,"num_experiments":5}
{"experiment":"8","experiment_name":"t5-base-span-511-scandvoc","pretraining_steps":1400000,"accuracy":60.9666666667,"f1_macro":56.1138360815,"num_experiments":5}
{"experiment":"8","experiment_name":"t5-base-span-511-scandvoc","pretraining_steps":1500000,"accuracy":64.35,"f1_macro":58.429161656,"num_experiments":5}
{"experiment":"9","experiment_name":"t5-base-ul2-mt5voc","pretraining_steps":1000000,"accuracy":71.1,"f1_macro":71.0952242998,"num_experiments":5}
{"experiment":"9","experiment_name":"t5-base-ul2-mt5voc","pretraining_steps":1100000,"accuracy":81.0333333333,"f1_macro":80.9959212654,"num_experiments":5}
{"experiment":"9","experiment_name":"t5-base-ul2-mt5voc","pretraining_steps":1200000,"accuracy":80.5,"f1_macro":80.3834883051,"num_experiments":5}
{"experiment":"9","experiment_name":"t5-base-ul2-mt5voc","pretraining_steps":1300000,"accuracy":82.7666666667,"f1_macro":82.7304446175,"num_experiments":5}
{"experiment":"9","experiment_name":"t5-base-ul2-mt5voc","pretraining_steps":1400000,"accuracy":82.65,"f1_macro":82.6014266748,"num_experiments":5}
{"experiment":"9","experiment_name":"t5-base-ul2-mt5voc","pretraining_steps":1500000,"accuracy":83.6666666667,"f1_macro":83.6466045157,"num_experiments":5}