nvan15 commited on Jan 15

Commit

dad14e4

verified ·

1 Parent(s): b816a2c

Batch upload part 20

Browse files

Files changed (50) hide show

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/RTE.tsv +3001 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/all_results.json +9 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/eval_results.json +9 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/adapter_config.json +26 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/added_tokens.json +3 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/special_tokens_map.json +15 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/spm.model +3 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/tokenizer.json +0 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/tokenizer_config.json +60 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft2/adapter_config.json +26 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft2/adapter_model.bin +3 -0
nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/trainer_state.json +411 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/RTE.tsv +3001 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/all_results.json +9 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/eval_results.json +9 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/added_tokens.json +3 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/special_tokens_map.json +15 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/spm.model +3 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/tokenizer.json +0 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/tokenizer_config.json +60 -0
nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/trainer_state.json +411 -0
nlu/test.sh +81 -0
nlu/training_metrics_bs8.json +26 -0
omini/train_flux/train_spatial_alignment.py +211 -0
omini/train_flux/train_spatial_alignment_rotation.py +211 -0
omini/train_flux/train_subject.py +205 -0
omini/train_flux/train_subject_rotation.py +205 -0
omini/train_flux/train_token_integration.py +136 -0
omini/train_flux/trainer.py +384 -0
omini/train_flux/trainer_rotation.py +449 -0
train/README.md +253 -0
train/config/compact_token_representation.yaml +65 -0
train/config/feature_reuse.yaml +64 -0
train/config/multi_condition.yaml +68 -0
train/config/spatial_alignment.yaml +64 -0
train/config/spatial_alignment_rotation.yaml +63 -0
train/config/subject.yaml +54 -0
train/config/subject_rotation.yaml +53 -0
train/config/token_integration.yaml +63 -0
train/requirements.txt +15 -0
train/script/data_download/data_download1.sh +1 -0
train/script/data_download/data_download2.sh +3 -0
train/script/train_compact_token_representation.sh +13 -0
train/script/train_feature_reuse.sh +13 -0
train/script/train_multi_condition.sh +13 -0
train/script/train_spatial_alignment.sh +13 -0
train/script/train_spatial_alignment_rotation.sh +13 -0
train/script/train_subject.sh +13 -0
train/script/train_subject_rotation.sh +13 -0
train/script/train_token_intergration.sh +13 -0

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/RTE.tsv ADDED Viewed

	@@ -0,0 +1,3001 @@

+index	prediction
+0	not_entailment
+1	not_entailment
+2	entailment
+3	not_entailment
+4	entailment
+5	entailment
+6	entailment
+7	not_entailment
+8	not_entailment
+9	entailment
+10	not_entailment
+11	entailment
+12	not_entailment
+13	entailment
+14	not_entailment
+15	not_entailment
+16	not_entailment
+17	entailment
+18	entailment
+19	not_entailment
+20	entailment
+21	not_entailment
+22	not_entailment
+23	not_entailment
+24	not_entailment
+25	not_entailment
+26	entailment
+27	entailment
+28	entailment
+29	entailment
+30	not_entailment
+31	entailment
+32	not_entailment
+33	not_entailment
+34	not_entailment
+35	entailment
+36	not_entailment
+37	entailment
+38	not_entailment
+39	entailment
+40	not_entailment
+41	not_entailment
+42	entailment
+43	not_entailment
+44	not_entailment
+45	entailment
+46	entailment
+47	not_entailment
+48	entailment
+49	not_entailment
+50	not_entailment
+51	entailment
+52	not_entailment
+53	entailment
+54	entailment
+55	not_entailment
+56	entailment
+57	not_entailment
+58	not_entailment
+59	entailment
+60	entailment
+61	not_entailment
+62	not_entailment
+63	entailment
+64	entailment
+65	not_entailment
+66	entailment
+67	entailment
+68	not_entailment
+69	entailment
+70	entailment
+71	entailment
+72	entailment
+73	entailment
+74	entailment
+75	entailment
+76	not_entailment
+77	entailment
+78	entailment
+79	not_entailment
+80	not_entailment
+81	entailment
+82	not_entailment
+83	not_entailment
+84	not_entailment
+85	not_entailment
+86	entailment
+87	not_entailment
+88	entailment
+89	entailment
+90	not_entailment
+91	entailment
+92	not_entailment
+93	entailment
+94	entailment
+95	not_entailment
+96	entailment
+97	entailment
+98	not_entailment
+99	entailment
+100	entailment
+101	entailment
+102	not_entailment
+103	entailment
+104	entailment
+105	not_entailment
+106	entailment
+107	not_entailment
+108	entailment
+109	not_entailment
+110	entailment
+111	not_entailment
+112	not_entailment
+113	not_entailment
+114	entailment
+115	not_entailment
+116	entailment
+117	not_entailment
+118	not_entailment
+119	not_entailment
+120	not_entailment
+121	not_entailment
+122	not_entailment
+123	not_entailment
+124	entailment
+125	entailment
+126	entailment
+127	entailment
+128	entailment
+129	entailment
+130	entailment
+131	entailment
+132	not_entailment
+133	not_entailment
+134	entailment
+135	not_entailment
+136	not_entailment
+137	not_entailment
+138	entailment
+139	not_entailment
+140	entailment
+141	not_entailment
+142	not_entailment
+143	entailment
+144	entailment
+145	not_entailment
+146	not_entailment
+147	entailment
+148	not_entailment
+149	entailment
+150	entailment
+151	entailment
+152	not_entailment
+153	entailment
+154	entailment
+155	not_entailment
+156	not_entailment
+157	entailment
+158	not_entailment
+159	not_entailment
+160	entailment
+161	not_entailment
+162	entailment
+163	not_entailment
+164	not_entailment
+165	entailment
+166	entailment
+167	entailment
+168	entailment
+169	not_entailment
+170	entailment
+171	entailment
+172	not_entailment
+173	entailment
+174	entailment
+175	not_entailment
+176	not_entailment
+177	entailment
+178	entailment
+179	not_entailment
+180	not_entailment
+181	not_entailment
+182	entailment
+183	entailment
+184	entailment
+185	not_entailment
+186	entailment
+187	entailment
+188	not_entailment
+189	not_entailment
+190	entailment
+191	entailment
+192	entailment
+193	not_entailment
+194	entailment
+195	entailment
+196	entailment
+197	not_entailment
+198	not_entailment
+199	not_entailment
+200	entailment
+201	entailment
+202	entailment
+203	not_entailment
+204	not_entailment
+205	entailment
+206	not_entailment
+207	not_entailment
+208	not_entailment
+209	entailment
+210	not_entailment
+211	not_entailment
+212	entailment
+213	entailment
+214	entailment
+215	entailment
+216	not_entailment
+217	not_entailment
+218	not_entailment
+219	not_entailment
+220	entailment
+221	entailment
+222	entailment
+223	not_entailment
+224	entailment
+225	not_entailment
+226	entailment
+227	entailment
+228	not_entailment
+229	not_entailment
+230	entailment
+231	entailment
+232	entailment
+233	not_entailment
+234	entailment
+235	not_entailment
+236	entailment
+237	entailment
+238	not_entailment
+239	entailment
+240	entailment
+241	entailment
+242	entailment
+243	entailment
+244	not_entailment
+245	not_entailment
+246	entailment
+247	not_entailment
+248	entailment
+249	entailment
+250	not_entailment
+251	not_entailment
+252	entailment
+253	not_entailment
+254	entailment
+255	entailment
+256	not_entailment
+257	not_entailment
+258	entailment
+259	entailment
+260	entailment
+261	not_entailment
+262	entailment
+263	entailment
+264	entailment
+265	entailment
+266	not_entailment
+267	not_entailment
+268	entailment
+269	not_entailment
+270	not_entailment
+271	entailment
+272	not_entailment
+273	entailment
+274	entailment
+275	not_entailment
+276	not_entailment
+277	entailment
+278	not_entailment
+279	not_entailment
+280	entailment
+281	entailment
+282	entailment
+283	not_entailment
+284	not_entailment
+285	not_entailment
+286	entailment
+287	not_entailment
+288	not_entailment
+289	entailment
+290	not_entailment
+291	not_entailment
+292	not_entailment
+293	entailment
+294	entailment
+295	not_entailment
+296	not_entailment
+297	not_entailment
+298	entailment
+299	entailment
+300	not_entailment
+301	entailment
+302	entailment
+303	entailment
+304	entailment
+305	not_entailment
+306	entailment
+307	not_entailment
+308	not_entailment
+309	entailment
+310	not_entailment
+311	entailment
+312	not_entailment
+313	not_entailment
+314	entailment
+315	not_entailment
+316	entailment
+317	not_entailment
+318	entailment
+319	entailment
+320	not_entailment
+321	not_entailment
+322	entailment
+323	entailment
+324	entailment
+325	not_entailment
+326	not_entailment
+327	entailment
+328	not_entailment
+329	entailment
+330	entailment
+331	entailment
+332	not_entailment
+333	not_entailment
+334	not_entailment
+335	entailment
+336	not_entailment
+337	entailment
+338	entailment
+339	entailment
+340	entailment
+341	entailment
+342	entailment
+343	not_entailment
+344	not_entailment
+345	entailment
+346	entailment
+347	entailment
+348	not_entailment
+349	not_entailment
+350	entailment
+351	entailment
+352	not_entailment
+353	entailment
+354	entailment
+355	not_entailment
+356	entailment
+357	not_entailment
+358	entailment
+359	entailment
+360	entailment
+361	entailment
+362	entailment
+363	not_entailment
+364	entailment
+365	entailment
+366	entailment
+367	entailment
+368	entailment
+369	not_entailment
+370	not_entailment
+371	entailment
+372	not_entailment
+373	entailment
+374	not_entailment
+375	entailment
+376	not_entailment
+377	entailment
+378	not_entailment
+379	not_entailment
+380	not_entailment
+381	not_entailment
+382	entailment
+383	entailment
+384	not_entailment
+385	entailment
+386	not_entailment
+387	entailment
+388	entailment
+389	not_entailment
+390	not_entailment
+391	not_entailment
+392	entailment
+393	entailment
+394	entailment
+395	entailment
+396	entailment
+397	not_entailment
+398	entailment
+399	not_entailment
+400	entailment
+401	entailment
+402	entailment
+403	entailment
+404	not_entailment
+405	entailment
+406	not_entailment
+407	entailment
+408	not_entailment
+409	entailment
+410	not_entailment
+411	entailment
+412	entailment
+413	entailment
+414	not_entailment
+415	not_entailment
+416	not_entailment
+417	not_entailment
+418	not_entailment
+419	not_entailment
+420	entailment
+421	entailment
+422	not_entailment
+423	not_entailment
+424	entailment
+425	entailment
+426	entailment
+427	not_entailment
+428	entailment
+429	entailment
+430	not_entailment
+431	not_entailment
+432	entailment
+433	entailment
+434	not_entailment
+435	not_entailment
+436	not_entailment
+437	entailment
+438	entailment
+439	not_entailment
+440	not_entailment
+441	entailment
+442	entailment
+443	entailment
+444	not_entailment
+445	entailment
+446	entailment
+447	not_entailment
+448	entailment
+449	entailment
+450	entailment
+451	entailment
+452	not_entailment
+453	entailment
+454	not_entailment
+455	not_entailment
+456	entailment
+457	entailment
+458	entailment
+459	entailment
+460	entailment
+461	entailment
+462	entailment
+463	entailment
+464	entailment
+465	entailment
+466	entailment
+467	not_entailment
+468	entailment
+469	entailment
+470	not_entailment
+471	entailment
+472	not_entailment
+473	entailment
+474	entailment
+475	not_entailment
+476	not_entailment
+477	not_entailment
+478	not_entailment
+479	entailment
+480	not_entailment
+481	entailment
+482	entailment
+483	entailment
+484	entailment
+485	entailment
+486	not_entailment
+487	entailment
+488	not_entailment
+489	entailment
+490	entailment
+491	entailment
+492	not_entailment
+493	entailment
+494	entailment
+495	not_entailment
+496	not_entailment
+497	not_entailment
+498	entailment
+499	entailment
+500	not_entailment
+501	entailment
+502	entailment
+503	entailment
+504	not_entailment
+505	entailment
+506	entailment
+507	not_entailment
+508	entailment
+509	not_entailment
+510	not_entailment
+511	entailment
+512	not_entailment
+513	entailment
+514	entailment
+515	not_entailment
+516	not_entailment
+517	entailment
+518	not_entailment
+519	entailment
+520	not_entailment
+521	not_entailment
+522	not_entailment
+523	entailment
+524	entailment
+525	not_entailment
+526	not_entailment
+527	entailment
+528	entailment
+529	not_entailment
+530	entailment
+531	not_entailment
+532	not_entailment
+533	entailment
+534	entailment
+535	entailment
+536	not_entailment
+537	entailment
+538	not_entailment
+539	not_entailment
+540	entailment
+541	not_entailment
+542	not_entailment
+543	entailment
+544	entailment
+545	not_entailment
+546	entailment
+547	entailment
+548	entailment
+549	entailment
+550	not_entailment
+551	entailment
+552	entailment
+553	not_entailment
+554	entailment
+555	not_entailment
+556	not_entailment
+557	not_entailment
+558	not_entailment
+559	not_entailment
+560	entailment
+561	entailment
+562	not_entailment
+563	entailment
+564	entailment
+565	entailment
+566	entailment
+567	entailment
+568	not_entailment
+569	entailment
+570	not_entailment
+571	entailment
+572	entailment
+573	not_entailment
+574	entailment
+575	entailment
+576	entailment
+577	entailment
+578	not_entailment
+579	entailment
+580	entailment
+581	not_entailment
+582	not_entailment
+583	entailment
+584	entailment
+585	entailment
+586	entailment
+587	not_entailment
+588	entailment
+589	not_entailment
+590	entailment
+591	not_entailment
+592	not_entailment
+593	not_entailment
+594	entailment
+595	entailment
+596	entailment
+597	not_entailment
+598	entailment
+599	not_entailment
+600	entailment
+601	entailment
+602	entailment
+603	not_entailment
+604	not_entailment
+605	not_entailment
+606	entailment
+607	entailment
+608	not_entailment
+609	entailment
+610	entailment
+611	not_entailment
+612	entailment
+613	not_entailment
+614	not_entailment
+615	entailment
+616	not_entailment
+617	entailment
+618	not_entailment
+619	entailment
+620	entailment
+621	entailment
+622	entailment
+623	not_entailment
+624	entailment
+625	entailment
+626	not_entailment
+627	entailment
+628	not_entailment
+629	not_entailment
+630	entailment
+631	entailment
+632	not_entailment
+633	not_entailment
+634	not_entailment
+635	not_entailment
+636	entailment
+637	not_entailment
+638	not_entailment
+639	entailment
+640	entailment
+641	not_entailment
+642	entailment
+643	not_entailment
+644	entailment
+645	not_entailment
+646	entailment
+647	entailment
+648	entailment
+649	not_entailment
+650	not_entailment
+651	not_entailment
+652	not_entailment
+653	not_entailment
+654	entailment
+655	entailment
+656	entailment
+657	entailment
+658	not_entailment
+659	not_entailment
+660	not_entailment
+661	entailment
+662	entailment
+663	entailment
+664	not_entailment
+665	not_entailment
+666	entailment
+667	not_entailment
+668	not_entailment
+669	not_entailment
+670	entailment
+671	entailment
+672	not_entailment
+673	entailment
+674	not_entailment
+675	entailment
+676	entailment
+677	entailment
+678	entailment
+679	entailment
+680	entailment
+681	entailment
+682	not_entailment
+683	not_entailment
+684	not_entailment
+685	entailment
+686	entailment
+687	entailment
+688	not_entailment
+689	entailment
+690	entailment
+691	not_entailment
+692	entailment
+693	entailment
+694	entailment
+695	not_entailment
+696	not_entailment
+697	entailment
+698	entailment
+699	entailment
+700	entailment
+701	entailment
+702	entailment
+703	not_entailment
+704	not_entailment
+705	not_entailment
+706	not_entailment
+707	entailment
+708	entailment
+709	entailment
+710	not_entailment
+711	entailment
+712	not_entailment
+713	entailment
+714	not_entailment
+715	not_entailment
+716	not_entailment
+717	entailment
+718	not_entailment
+719	not_entailment
+720	entailment
+721	entailment
+722	entailment
+723	not_entailment
+724	entailment
+725	not_entailment
+726	not_entailment
+727	entailment
+728	entailment
+729	entailment
+730	entailment
+731	entailment
+732	not_entailment
+733	entailment
+734	entailment
+735	entailment
+736	not_entailment
+737	not_entailment
+738	entailment
+739	entailment
+740	not_entailment
+741	entailment
+742	entailment
+743	entailment
+744	not_entailment
+745	entailment
+746	not_entailment
+747	not_entailment
+748	not_entailment
+749	not_entailment
+750	entailment
+751	entailment
+752	not_entailment
+753	not_entailment
+754	entailment
+755	not_entailment
+756	entailment
+757	not_entailment
+758	entailment
+759	entailment
+760	entailment
+761	entailment
+762	entailment
+763	entailment
+764	not_entailment
+765	not_entailment
+766	entailment
+767	not_entailment
+768	entailment
+769	not_entailment
+770	entailment
+771	not_entailment
+772	not_entailment
+773	not_entailment
+774	not_entailment
+775	entailment
+776	not_entailment
+777	entailment
+778	entailment
+779	not_entailment
+780	entailment
+781	entailment
+782	entailment
+783	entailment
+784	entailment
+785	entailment
+786	entailment
+787	entailment
+788	entailment
+789	entailment
+790	not_entailment
+791	entailment
+792	not_entailment
+793	entailment
+794	not_entailment
+795	entailment
+796	entailment
+797	entailment
+798	not_entailment
+799	not_entailment
+800	entailment
+801	entailment
+802	entailment
+803	entailment
+804	not_entailment
+805	entailment
+806	not_entailment
+807	entailment
+808	entailment
+809	not_entailment
+810	not_entailment
+811	not_entailment
+812	not_entailment
+813	entailment
+814	entailment
+815	entailment
+816	not_entailment
+817	not_entailment
+818	entailment
+819	not_entailment
+820	entailment
+821	not_entailment
+822	entailment
+823	entailment
+824	entailment
+825	entailment
+826	entailment
+827	entailment
+828	entailment
+829	entailment
+830	entailment
+831	entailment
+832	entailment
+833	not_entailment
+834	entailment
+835	entailment
+836	entailment
+837	not_entailment
+838	not_entailment
+839	entailment
+840	entailment
+841	entailment
+842	entailment
+843	not_entailment
+844	entailment
+845	entailment
+846	entailment
+847	entailment
+848	entailment
+849	not_entailment
+850	entailment
+851	entailment
+852	entailment
+853	entailment
+854	not_entailment
+855	not_entailment
+856	entailment
+857	not_entailment
+858	entailment
+859	entailment
+860	entailment
+861	not_entailment
+862	not_entailment
+863	not_entailment
+864	entailment
+865	entailment
+866	entailment
+867	entailment
+868	entailment
+869	entailment
+870	entailment
+871	entailment
+872	entailment
+873	entailment
+874	entailment
+875	entailment
+876	not_entailment
+877	entailment
+878	entailment
+879	not_entailment
+880	entailment
+881	entailment
+882	entailment
+883	entailment
+884	entailment
+885	entailment
+886	entailment
+887	entailment
+888	not_entailment
+889	entailment
+890	entailment
+891	entailment
+892	entailment
+893	entailment
+894	entailment
+895	entailment
+896	entailment
+897	not_entailment
+898	not_entailment
+899	entailment
+900	not_entailment
+901	not_entailment
+902	entailment
+903	not_entailment
+904	entailment
+905	entailment
+906	entailment
+907	entailment
+908	entailment
+909	entailment
+910	not_entailment
+911	entailment
+912	entailment
+913	entailment
+914	entailment
+915	entailment
+916	entailment
+917	entailment
+918	not_entailment
+919	not_entailment
+920	entailment
+921	entailment
+922	entailment
+923	not_entailment
+924	not_entailment
+925	entailment
+926	entailment
+927	entailment
+928	entailment
+929	not_entailment
+930	not_entailment
+931	entailment
+932	not_entailment
+933	not_entailment
+934	entailment
+935	entailment
+936	entailment
+937	entailment
+938	not_entailment
+939	not_entailment
+940	not_entailment
+941	entailment
+942	entailment
+943	entailment
+944	entailment
+945	not_entailment
+946	not_entailment
+947	not_entailment
+948	not_entailment
+949	not_entailment
+950	entailment
+951	entailment
+952	entailment
+953	not_entailment
+954	entailment
+955	not_entailment
+956	entailment
+957	entailment
+958	not_entailment
+959	entailment
+960	entailment
+961	entailment
+962	not_entailment
+963	entailment
+964	entailment
+965	entailment
+966	entailment
+967	entailment
+968	entailment
+969	not_entailment
+970	entailment
+971	entailment
+972	entailment
+973	entailment
+974	entailment
+975	entailment
+976	entailment
+977	entailment
+978	not_entailment
+979	not_entailment
+980	not_entailment
+981	not_entailment
+982	entailment
+983	entailment
+984	not_entailment
+985	entailment
+986	entailment
+987	entailment
+988	not_entailment
+989	entailment
+990	entailment
+991	not_entailment
+992	entailment
+993	not_entailment
+994	entailment
+995	entailment
+996	entailment
+997	not_entailment
+998	entailment
+999	not_entailment
+1000	entailment
+1001	not_entailment
+1002	entailment
+1003	not_entailment
+1004	not_entailment
+1005	entailment
+1006	entailment
+1007	entailment
+1008	entailment
+1009	entailment
+1010	entailment
+1011	not_entailment
+1012	not_entailment
+1013	not_entailment
+1014	not_entailment
+1015	entailment
+1016	entailment
+1017	entailment
+1018	entailment
+1019	entailment
+1020	entailment
+1021	entailment
+1022	not_entailment
+1023	entailment
+1024	entailment
+1025	not_entailment
+1026	not_entailment
+1027	entailment
+1028	entailment
+1029	not_entailment
+1030	not_entailment
+1031	not_entailment
+1032	entailment
+1033	entailment
+1034	entailment
+1035	entailment
+1036	not_entailment
+1037	not_entailment
+1038	not_entailment
+1039	entailment
+1040	not_entailment
+1041	not_entailment
+1042	not_entailment
+1043	not_entailment
+1044	not_entailment
+1045	entailment
+1046	not_entailment
+1047	entailment
+1048	entailment
+1049	entailment
+1050	not_entailment
+1051	entailment
+1052	entailment
+1053	not_entailment
+1054	entailment
+1055	entailment
+1056	not_entailment
+1057	entailment
+1058	entailment
+1059	entailment
+1060	entailment
+1061	not_entailment
+1062	not_entailment
+1063	entailment
+1064	not_entailment
+1065	entailment
+1066	entailment
+1067	entailment
+1068	entailment
+1069	not_entailment
+1070	entailment
+1071	entailment
+1072	entailment
+1073	entailment
+1074	not_entailment
+1075	not_entailment
+1076	not_entailment
+1077	entailment
+1078	not_entailment
+1079	not_entailment
+1080	entailment
+1081	not_entailment
+1082	not_entailment
+1083	entailment
+1084	entailment
+1085	entailment
+1086	entailment
+1087	not_entailment
+1088	entailment
+1089	not_entailment
+1090	not_entailment
+1091	entailment
+1092	not_entailment
+1093	not_entailment
+1094	not_entailment
+1095	not_entailment
+1096	entailment
+1097	entailment
+1098	not_entailment
+1099	not_entailment
+1100	not_entailment
+1101	entailment
+1102	entailment
+1103	not_entailment
+1104	not_entailment
+1105	not_entailment
+1106	not_entailment
+1107	not_entailment
+1108	not_entailment
+1109	not_entailment
+1110	not_entailment
+1111	not_entailment
+1112	not_entailment
+1113	entailment
+1114	entailment
+1115	not_entailment
+1116	not_entailment
+1117	not_entailment
+1118	not_entailment
+1119	entailment
+1120	not_entailment
+1121	not_entailment
+1122	not_entailment
+1123	entailment
+1124	entailment
+1125	entailment
+1126	entailment
+1127	entailment
+1128	not_entailment
+1129	not_entailment
+1130	not_entailment
+1131	not_entailment
+1132	not_entailment
+1133	not_entailment
+1134	not_entailment
+1135	not_entailment
+1136	not_entailment
+1137	not_entailment
+1138	not_entailment
+1139	entailment
+1140	not_entailment
+1141	not_entailment
+1142	entailment
+1143	not_entailment
+1144	not_entailment
+1145	entailment
+1146	not_entailment
+1147	entailment
+1148	not_entailment
+1149	not_entailment
+1150	not_entailment
+1151	not_entailment
+1152	not_entailment
+1153	entailment
+1154	not_entailment
+1155	entailment
+1156	entailment
+1157	entailment
+1158	not_entailment
+1159	not_entailment
+1160	entailment
+1161	not_entailment
+1162	not_entailment
+1163	not_entailment
+1164	not_entailment
+1165	entailment
+1166	not_entailment
+1167	entailment
+1168	entailment
+1169	not_entailment
+1170	entailment
+1171	entailment
+1172	entailment
+1173	entailment
+1174	not_entailment
+1175	entailment
+1176	entailment
+1177	entailment
+1178	not_entailment
+1179	not_entailment
+1180	not_entailment
+1181	entailment
+1182	not_entailment
+1183	not_entailment
+1184	not_entailment
+1185	not_entailment
+1186	not_entailment
+1187	not_entailment
+1188	entailment
+1189	not_entailment
+1190	not_entailment
+1191	not_entailment
+1192	not_entailment
+1193	not_entailment
+1194	not_entailment
+1195	entailment
+1196	not_entailment
+1197	not_entailment
+1198	not_entailment
+1199	not_entailment
+1200	entailment
+1201	entailment
+1202	not_entailment
+1203	not_entailment
+1204	entailment
+1205	entailment
+1206	entailment
+1207	entailment
+1208	not_entailment
+1209	not_entailment
+1210	entailment
+1211	entailment
+1212	not_entailment
+1213	entailment
+1214	entailment
+1215	not_entailment
+1216	entailment
+1217	not_entailment
+1218	entailment
+1219	entailment
+1220	entailment
+1221	entailment
+1222	entailment
+1223	entailment
+1224	not_entailment
+1225	entailment
+1226	entailment
+1227	not_entailment
+1228	entailment
+1229	entailment
+1230	entailment
+1231	entailment
+1232	entailment
+1233	not_entailment
+1234	entailment
+1235	entailment
+1236	entailment
+1237	entailment
+1238	entailment
+1239	not_entailment
+1240	not_entailment
+1241	entailment
+1242	entailment
+1243	entailment
+1244	entailment
+1245	entailment
+1246	entailment
+1247	not_entailment
+1248	entailment
+1249	entailment
+1250	not_entailment
+1251	entailment
+1252	entailment
+1253	entailment
+1254	not_entailment
+1255	entailment
+1256	not_entailment
+1257	not_entailment
+1258	entailment
+1259	not_entailment
+1260	not_entailment
+1261	entailment
+1262	not_entailment
+1263	not_entailment
+1264	not_entailment
+1265	entailment
+1266	entailment
+1267	entailment
+1268	not_entailment
+1269	not_entailment
+1270	entailment
+1271	entailment
+1272	entailment
+1273	entailment
+1274	entailment
+1275	entailment
+1276	not_entailment
+1277	entailment
+1278	entailment
+1279	entailment
+1280	entailment
+1281	not_entailment
+1282	not_entailment
+1283	not_entailment
+1284	not_entailment
+1285	not_entailment
+1286	not_entailment
+1287	entailment
+1288	entailment
+1289	not_entailment
+1290	entailment
+1291	entailment
+1292	entailment
+1293	not_entailment
+1294	not_entailment
+1295	not_entailment
+1296	not_entailment
+1297	not_entailment
+1298	entailment
+1299	not_entailment
+1300	entailment
+1301	not_entailment
+1302	entailment
+1303	not_entailment
+1304	entailment
+1305	entailment
+1306	not_entailment
+1307	not_entailment
+1308	entailment
+1309	not_entailment
+1310	not_entailment
+1311	not_entailment
+1312	not_entailment
+1313	entailment
+1314	entailment
+1315	not_entailment
+1316	not_entailment
+1317	entailment
+1318	not_entailment
+1319	not_entailment
+1320	entailment
+1321	entailment
+1322	not_entailment
+1323	not_entailment
+1324	not_entailment
+1325	not_entailment
+1326	entailment
+1327	entailment
+1328	not_entailment
+1329	entailment
+1330	entailment
+1331	not_entailment
+1332	not_entailment
+1333	not_entailment
+1334	entailment
+1335	entailment
+1336	not_entailment
+1337	entailment
+1338	entailment
+1339	entailment
+1340	entailment
+1341	entailment
+1342	entailment
+1343	entailment
+1344	not_entailment
+1345	not_entailment
+1346	entailment
+1347	entailment
+1348	entailment
+1349	entailment
+1350	not_entailment
+1351	entailment
+1352	entailment
+1353	not_entailment
+1354	entailment
+1355	entailment
+1356	entailment
+1357	entailment
+1358	entailment
+1359	entailment
+1360	not_entailment
+1361	not_entailment
+1362	not_entailment
+1363	not_entailment
+1364	not_entailment
+1365	entailment
+1366	entailment
+1367	not_entailment
+1368	entailment
+1369	not_entailment
+1370	not_entailment
+1371	entailment
+1372	not_entailment
+1373	entailment
+1374	entailment
+1375	entailment
+1376	not_entailment
+1377	not_entailment
+1378	not_entailment
+1379	not_entailment
+1380	not_entailment
+1381	entailment
+1382	entailment
+1383	not_entailment
+1384	not_entailment
+1385	not_entailment
+1386	entailment
+1387	entailment
+1388	not_entailment
+1389	not_entailment
+1390	entailment
+1391	not_entailment
+1392	not_entailment
+1393	entailment
+1394	entailment
+1395	not_entailment
+1396	not_entailment
+1397	entailment
+1398	not_entailment
+1399	entailment
+1400	not_entailment
+1401	not_entailment
+1402	entailment
+1403	entailment
+1404	entailment
+1405	entailment
+1406	entailment
+1407	entailment
+1408	not_entailment
+1409	not_entailment
+1410	entailment
+1411	entailment
+1412	not_entailment
+1413	entailment
+1414	not_entailment
+1415	entailment
+1416	entailment
+1417	entailment
+1418	entailment
+1419	entailment
+1420	not_entailment
+1421	entailment
+1422	entailment
+1423	entailment
+1424	entailment
+1425	entailment
+1426	entailment
+1427	entailment
+1428	entailment
+1429	not_entailment
+1430	entailment
+1431	entailment
+1432	not_entailment
+1433	entailment
+1434	entailment
+1435	not_entailment
+1436	entailment
+1437	not_entailment
+1438	entailment
+1439	entailment
+1440	entailment
+1441	entailment
+1442	entailment
+1443	not_entailment
+1444	entailment
+1445	entailment
+1446	not_entailment
+1447	entailment
+1448	not_entailment
+1449	entailment
+1450	entailment
+1451	not_entailment
+1452	not_entailment
+1453	not_entailment
+1454	entailment
+1455	entailment
+1456	entailment
+1457	entailment
+1458	entailment
+1459	entailment
+1460	not_entailment
+1461	entailment
+1462	entailment
+1463	entailment
+1464	entailment
+1465	entailment
+1466	entailment
+1467	entailment
+1468	entailment
+1469	not_entailment
+1470	entailment
+1471	entailment
+1472	not_entailment
+1473	not_entailment
+1474	entailment
+1475	entailment
+1476	entailment
+1477	entailment
+1478	entailment
+1479	not_entailment
+1480	not_entailment
+1481	not_entailment
+1482	entailment
+1483	entailment
+1484	entailment
+1485	entailment
+1486	entailment
+1487	entailment
+1488	entailment
+1489	entailment
+1490	entailment
+1491	entailment
+1492	entailment
+1493	entailment
+1494	entailment
+1495	entailment
+1496	entailment
+1497	not_entailment
+1498	not_entailment
+1499	not_entailment
+1500	not_entailment
+1501	entailment
+1502	not_entailment
+1503	not_entailment
+1504	entailment
+1505	entailment
+1506	not_entailment
+1507	entailment
+1508	not_entailment
+1509	not_entailment
+1510	entailment
+1511	entailment
+1512	entailment
+1513	not_entailment
+1514	entailment
+1515	entailment
+1516	entailment
+1517	entailment
+1518	entailment
+1519	entailment
+1520	entailment
+1521	entailment
+1522	entailment
+1523	entailment
+1524	entailment
+1525	entailment
+1526	not_entailment
+1527	entailment
+1528	not_entailment
+1529	entailment
+1530	entailment
+1531	entailment
+1532	entailment
+1533	entailment
+1534	entailment
+1535	entailment
+1536	entailment
+1537	not_entailment
+1538	entailment
+1539	not_entailment
+1540	not_entailment
+1541	not_entailment
+1542	not_entailment
+1543	entailment
+1544	entailment
+1545	entailment
+1546	not_entailment
+1547	entailment
+1548	entailment
+1549	entailment
+1550	entailment
+1551	entailment
+1552	entailment
+1553	entailment
+1554	not_entailment
+1555	entailment
+1556	entailment
+1557	entailment
+1558	not_entailment
+1559	not_entailment
+1560	entailment
+1561	not_entailment
+1562	entailment
+1563	entailment
+1564	not_entailment
+1565	entailment
+1566	not_entailment
+1567	entailment
+1568	not_entailment
+1569	not_entailment
+1570	entailment
+1571	not_entailment
+1572	entailment
+1573	entailment
+1574	not_entailment
+1575	not_entailment
+1576	entailment
+1577	entailment
+1578	entailment
+1579	not_entailment
+1580	entailment
+1581	entailment
+1582	not_entailment
+1583	not_entailment
+1584	not_entailment
+1585	entailment
+1586	entailment
+1587	not_entailment
+1588	not_entailment
+1589	entailment
+1590	entailment
+1591	not_entailment
+1592	entailment
+1593	not_entailment
+1594	entailment
+1595	entailment
+1596	not_entailment
+1597	entailment
+1598	entailment
+1599	not_entailment
+1600	entailment
+1601	entailment
+1602	entailment
+1603	not_entailment
+1604	not_entailment
+1605	entailment
+1606	entailment
+1607	not_entailment
+1608	entailment
+1609	entailment
+1610	entailment
+1611	not_entailment
+1612	entailment
+1613	entailment
+1614	not_entailment
+1615	not_entailment
+1616	not_entailment
+1617	entailment
+1618	not_entailment
+1619	not_entailment
+1620	not_entailment
+1621	entailment
+1622	not_entailment
+1623	entailment
+1624	entailment
+1625	not_entailment
+1626	not_entailment
+1627	entailment
+1628	not_entailment
+1629	entailment
+1630	not_entailment
+1631	entailment
+1632	entailment
+1633	not_entailment
+1634	entailment
+1635	entailment
+1636	not_entailment
+1637	not_entailment
+1638	not_entailment
+1639	not_entailment
+1640	entailment
+1641	not_entailment
+1642	not_entailment
+1643	entailment
+1644	not_entailment
+1645	not_entailment
+1646	not_entailment
+1647	not_entailment
+1648	not_entailment
+1649	not_entailment
+1650	entailment
+1651	entailment
+1652	not_entailment
+1653	not_entailment
+1654	entailment
+1655	entailment
+1656	entailment
+1657	entailment
+1658	entailment
+1659	entailment
+1660	not_entailment
+1661	entailment
+1662	entailment
+1663	not_entailment
+1664	entailment
+1665	not_entailment
+1666	entailment
+1667	not_entailment
+1668	entailment
+1669	entailment
+1670	entailment
+1671	entailment
+1672	not_entailment
+1673	not_entailment
+1674	entailment
+1675	not_entailment
+1676	entailment
+1677	entailment
+1678	not_entailment
+1679	entailment
+1680	entailment
+1681	entailment
+1682	not_entailment
+1683	entailment
+1684	entailment
+1685	not_entailment
+1686	entailment
+1687	entailment
+1688	not_entailment
+1689	not_entailment
+1690	entailment
+1691	entailment
+1692	not_entailment
+1693	not_entailment
+1694	not_entailment
+1695	entailment
+1696	entailment
+1697	not_entailment
+1698	entailment
+1699	not_entailment
+1700	entailment
+1701	entailment
+1702	not_entailment
+1703	entailment
+1704	entailment
+1705	not_entailment
+1706	not_entailment
+1707	entailment
+1708	not_entailment
+1709	not_entailment
+1710	not_entailment
+1711	not_entailment
+1712	entailment
+1713	entailment
+1714	not_entailment
+1715	entailment
+1716	entailment
+1717	entailment
+1718	not_entailment
+1719	not_entailment
+1720	not_entailment
+1721	entailment
+1722	entailment
+1723	not_entailment
+1724	entailment
+1725	entailment
+1726	entailment
+1727	entailment
+1728	entailment
+1729	entailment
+1730	not_entailment
+1731	entailment
+1732	not_entailment
+1733	entailment
+1734	entailment
+1735	not_entailment
+1736	entailment
+1737	entailment
+1738	not_entailment
+1739	not_entailment
+1740	entailment
+1741	entailment
+1742	not_entailment
+1743	not_entailment
+1744	entailment
+1745	not_entailment
+1746	entailment
+1747	not_entailment
+1748	entailment
+1749	not_entailment
+1750	not_entailment
+1751	not_entailment
+1752	not_entailment
+1753	entailment
+1754	not_entailment
+1755	not_entailment
+1756	not_entailment
+1757	not_entailment
+1758	entailment
+1759	entailment
+1760	entailment
+1761	entailment
+1762	entailment
+1763	not_entailment
+1764	entailment
+1765	entailment
+1766	not_entailment
+1767	not_entailment
+1768	entailment
+1769	not_entailment
+1770	entailment
+1771	not_entailment
+1772	not_entailment
+1773	entailment
+1774	not_entailment
+1775	entailment
+1776	not_entailment
+1777	entailment
+1778	not_entailment
+1779	entailment
+1780	not_entailment
+1781	not_entailment
+1782	not_entailment
+1783	entailment
+1784	not_entailment
+1785	entailment
+1786	entailment
+1787	not_entailment
+1788	entailment
+1789	entailment
+1790	entailment
+1791	entailment
+1792	entailment
+1793	entailment
+1794	entailment
+1795	entailment
+1796	not_entailment
+1797	not_entailment
+1798	entailment
+1799	not_entailment
+1800	entailment
+1801	not_entailment
+1802	entailment
+1803	entailment
+1804	not_entailment
+1805	entailment
+1806	not_entailment
+1807	not_entailment
+1808	not_entailment
+1809	entailment
+1810	entailment
+1811	entailment
+1812	not_entailment
+1813	entailment
+1814	entailment
+1815	entailment
+1816	entailment
+1817	not_entailment
+1818	entailment
+1819	entailment
+1820	not_entailment
+1821	not_entailment
+1822	entailment
+1823	not_entailment
+1824	entailment
+1825	entailment
+1826	entailment
+1827	not_entailment
+1828	entailment
+1829	entailment
+1830	entailment
+1831	not_entailment
+1832	not_entailment
+1833	not_entailment
+1834	entailment
+1835	entailment
+1836	entailment
+1837	entailment
+1838	not_entailment
+1839	not_entailment
+1840	entailment
+1841	entailment
+1842	not_entailment
+1843	entailment
+1844	not_entailment
+1845	not_entailment
+1846	not_entailment
+1847	not_entailment
+1848	not_entailment
+1849	not_entailment
+1850	not_entailment
+1851	not_entailment
+1852	entailment
+1853	not_entailment
+1854	entailment
+1855	not_entailment
+1856	not_entailment
+1857	entailment
+1858	entailment
+1859	not_entailment
+1860	not_entailment
+1861	entailment
+1862	not_entailment
+1863	entailment
+1864	entailment
+1865	not_entailment
+1866	entailment
+1867	not_entailment
+1868	entailment
+1869	entailment
+1870	not_entailment
+1871	entailment
+1872	entailment
+1873	not_entailment
+1874	entailment
+1875	entailment
+1876	not_entailment
+1877	not_entailment
+1878	not_entailment
+1879	not_entailment
+1880	not_entailment
+1881	entailment
+1882	not_entailment
+1883	not_entailment
+1884	entailment
+1885	entailment
+1886	entailment
+1887	entailment
+1888	entailment
+1889	entailment
+1890	not_entailment
+1891	not_entailment
+1892	entailment
+1893	entailment
+1894	entailment
+1895	not_entailment
+1896	entailment
+1897	entailment
+1898	not_entailment
+1899	entailment
+1900	not_entailment
+1901	entailment
+1902	not_entailment
+1903	not_entailment
+1904	not_entailment
+1905	not_entailment
+1906	entailment
+1907	not_entailment
+1908	not_entailment
+1909	entailment
+1910	entailment
+1911	entailment
+1912	not_entailment
+1913	not_entailment
+1914	entailment
+1915	not_entailment
+1916	not_entailment
+1917	not_entailment
+1918	not_entailment
+1919	entailment
+1920	not_entailment
+1921	not_entailment
+1922	entailment
+1923	entailment
+1924	entailment
+1925	entailment
+1926	not_entailment
+1927	not_entailment
+1928	not_entailment
+1929	entailment
+1930	not_entailment
+1931	entailment
+1932	entailment
+1933	not_entailment
+1934	not_entailment
+1935	entailment
+1936	not_entailment
+1937	not_entailment
+1938	not_entailment
+1939	entailment
+1940	entailment
+1941	not_entailment
+1942	not_entailment
+1943	entailment
+1944	entailment
+1945	not_entailment
+1946	entailment
+1947	entailment
+1948	entailment
+1949	not_entailment
+1950	not_entailment
+1951	not_entailment
+1952	not_entailment
+1953	not_entailment
+1954	not_entailment
+1955	not_entailment
+1956	entailment
+1957	entailment
+1958	not_entailment
+1959	entailment
+1960	entailment
+1961	not_entailment
+1962	entailment
+1963	not_entailment
+1964	not_entailment
+1965	entailment
+1966	not_entailment
+1967	entailment
+1968	not_entailment
+1969	not_entailment
+1970	entailment
+1971	entailment
+1972	not_entailment
+1973	entailment
+1974	entailment
+1975	not_entailment
+1976	entailment
+1977	not_entailment
+1978	entailment
+1979	entailment
+1980	entailment
+1981	entailment
+1982	not_entailment
+1983	entailment
+1984	entailment
+1985	entailment
+1986	entailment
+1987	not_entailment
+1988	not_entailment
+1989	entailment
+1990	entailment
+1991	entailment
+1992	not_entailment
+1993	not_entailment
+1994	entailment
+1995	entailment
+1996	entailment
+1997	not_entailment
+1998	entailment
+1999	entailment
+2000	not_entailment
+2001	not_entailment
+2002	entailment
+2003	entailment
+2004	not_entailment
+2005	entailment
+2006	not_entailment
+2007	not_entailment
+2008	not_entailment
+2009	not_entailment
+2010	entailment
+2011	entailment
+2012	not_entailment
+2013	entailment
+2014	entailment
+2015	entailment
+2016	entailment
+2017	entailment
+2018	not_entailment
+2019	entailment
+2020	not_entailment
+2021	not_entailment
+2022	entailment
+2023	entailment
+2024	entailment
+2025	entailment
+2026	entailment
+2027	not_entailment
+2028	not_entailment
+2029	not_entailment
+2030	not_entailment
+2031	entailment
+2032	entailment
+2033	entailment
+2034	entailment
+2035	not_entailment
+2036	entailment
+2037	entailment
+2038	entailment
+2039	not_entailment
+2040	entailment
+2041	entailment
+2042	entailment
+2043	not_entailment
+2044	entailment
+2045	entailment
+2046	entailment
+2047	not_entailment
+2048	not_entailment
+2049	entailment
+2050	entailment
+2051	entailment
+2052	not_entailment
+2053	not_entailment
+2054	not_entailment
+2055	not_entailment
+2056	not_entailment
+2057	entailment
+2058	not_entailment
+2059	not_entailment
+2060	not_entailment
+2061	not_entailment
+2062	not_entailment
+2063	entailment
+2064	entailment
+2065	entailment
+2066	not_entailment
+2067	entailment
+2068	not_entailment
+2069	entailment
+2070	entailment
+2071	not_entailment
+2072	entailment
+2073	entailment
+2074	not_entailment
+2075	entailment
+2076	not_entailment
+2077	entailment
+2078	entailment
+2079	entailment
+2080	entailment
+2081	entailment
+2082	not_entailment
+2083	entailment
+2084	entailment
+2085	not_entailment
+2086	entailment
+2087	entailment
+2088	entailment
+2089	entailment
+2090	not_entailment
+2091	entailment
+2092	not_entailment
+2093	not_entailment
+2094	entailment
+2095	entailment
+2096	not_entailment
+2097	entailment
+2098	not_entailment
+2099	not_entailment
+2100	entailment
+2101	not_entailment
+2102	entailment
+2103	not_entailment
+2104	entailment
+2105	not_entailment
+2106	entailment
+2107	entailment
+2108	entailment
+2109	not_entailment
+2110	entailment
+2111	entailment
+2112	entailment
+2113	not_entailment
+2114	not_entailment
+2115	entailment
+2116	not_entailment
+2117	entailment
+2118	not_entailment
+2119	entailment
+2120	entailment
+2121	not_entailment
+2122	entailment
+2123	entailment
+2124	not_entailment
+2125	not_entailment
+2126	entailment
+2127	entailment
+2128	entailment
+2129	not_entailment
+2130	entailment
+2131	not_entailment
+2132	not_entailment
+2133	not_entailment
+2134	entailment
+2135	entailment
+2136	not_entailment
+2137	not_entailment
+2138	not_entailment
+2139	not_entailment
+2140	entailment
+2141	not_entailment
+2142	entailment
+2143	not_entailment
+2144	not_entailment
+2145	entailment
+2146	entailment
+2147	not_entailment
+2148	not_entailment
+2149	entailment
+2150	not_entailment
+2151	not_entailment
+2152	entailment
+2153	entailment
+2154	entailment
+2155	entailment
+2156	entailment
+2157	entailment
+2158	not_entailment
+2159	entailment
+2160	not_entailment
+2161	entailment
+2162	entailment
+2163	not_entailment
+2164	not_entailment
+2165	not_entailment
+2166	not_entailment
+2167	entailment
+2168	entailment
+2169	not_entailment
+2170	not_entailment
+2171	entailment
+2172	entailment
+2173	not_entailment
+2174	entailment
+2175	not_entailment
+2176	not_entailment
+2177	entailment
+2178	not_entailment
+2179	entailment
+2180	not_entailment
+2181	not_entailment
+2182	entailment
+2183	not_entailment
+2184	not_entailment
+2185	entailment
+2186	entailment
+2187	entailment
+2188	entailment
+2189	not_entailment
+2190	not_entailment
+2191	entailment
+2192	entailment
+2193	not_entailment
+2194	not_entailment
+2195	not_entailment
+2196	entailment
+2197	entailment
+2198	not_entailment
+2199	entailment
+2200	entailment
+2201	entailment
+2202	not_entailment
+2203	not_entailment
+2204	not_entailment
+2205	entailment
+2206	not_entailment
+2207	entailment
+2208	not_entailment
+2209	entailment
+2210	not_entailment
+2211	entailment
+2212	entailment
+2213	entailment
+2214	entailment
+2215	entailment
+2216	not_entailment
+2217	not_entailment
+2218	not_entailment
+2219	not_entailment
+2220	not_entailment
+2221	entailment
+2222	not_entailment
+2223	not_entailment
+2224	not_entailment
+2225	not_entailment
+2226	not_entailment
+2227	not_entailment
+2228	not_entailment
+2229	not_entailment
+2230	not_entailment
+2231	not_entailment
+2232	not_entailment
+2233	not_entailment
+2234	entailment
+2235	not_entailment
+2236	entailment
+2237	not_entailment
+2238	entailment
+2239	not_entailment
+2240	not_entailment
+2241	not_entailment
+2242	entailment
+2243	not_entailment
+2244	entailment
+2245	entailment
+2246	entailment
+2247	not_entailment
+2248	entailment
+2249	not_entailment
+2250	entailment
+2251	not_entailment
+2252	entailment
+2253	entailment
+2254	entailment
+2255	not_entailment
+2256	not_entailment
+2257	not_entailment
+2258	not_entailment
+2259	not_entailment
+2260	entailment
+2261	entailment
+2262	not_entailment
+2263	not_entailment
+2264	not_entailment
+2265	entailment
+2266	not_entailment
+2267	entailment
+2268	not_entailment
+2269	not_entailment
+2270	not_entailment
+2271	entailment
+2272	entailment
+2273	entailment
+2274	entailment
+2275	entailment
+2276	entailment
+2277	entailment
+2278	not_entailment
+2279	entailment
+2280	not_entailment
+2281	not_entailment
+2282	entailment
+2283	not_entailment
+2284	not_entailment
+2285	entailment
+2286	not_entailment
+2287	not_entailment
+2288	entailment
+2289	entailment
+2290	entailment
+2291	entailment
+2292	entailment
+2293	entailment
+2294	entailment
+2295	entailment
+2296	entailment
+2297	entailment
+2298	entailment
+2299	entailment
+2300	not_entailment
+2301	entailment
+2302	not_entailment
+2303	entailment
+2304	entailment
+2305	entailment
+2306	entailment
+2307	not_entailment
+2308	not_entailment
+2309	not_entailment
+2310	entailment
+2311	not_entailment
+2312	not_entailment
+2313	entailment
+2314	not_entailment
+2315	entailment
+2316	entailment
+2317	not_entailment
+2318	entailment
+2319	not_entailment
+2320	not_entailment
+2321	entailment
+2322	entailment
+2323	not_entailment
+2324	entailment
+2325	not_entailment
+2326	not_entailment
+2327	not_entailment
+2328	not_entailment
+2329	entailment
+2330	entailment
+2331	entailment
+2332	entailment
+2333	entailment
+2334	not_entailment
+2335	not_entailment
+2336	entailment
+2337	entailment
+2338	entailment
+2339	not_entailment
+2340	not_entailment
+2341	entailment
+2342	not_entailment
+2343	entailment
+2344	not_entailment
+2345	entailment
+2346	entailment
+2347	entailment
+2348	not_entailment
+2349	entailment
+2350	entailment
+2351	not_entailment
+2352	entailment
+2353	entailment
+2354	not_entailment
+2355	entailment
+2356	entailment
+2357	not_entailment
+2358	entailment
+2359	entailment
+2360	entailment
+2361	not_entailment
+2362	not_entailment
+2363	not_entailment
+2364	not_entailment
+2365	not_entailment
+2366	not_entailment
+2367	entailment
+2368	not_entailment
+2369	entailment
+2370	entailment
+2371	entailment
+2372	entailment
+2373	entailment
+2374	entailment
+2375	entailment
+2376	not_entailment
+2377	entailment
+2378	not_entailment
+2379	not_entailment
+2380	not_entailment
+2381	entailment
+2382	entailment
+2383	entailment
+2384	entailment
+2385	not_entailment
+2386	entailment
+2387	entailment
+2388	entailment
+2389	entailment
+2390	entailment
+2391	entailment
+2392	entailment
+2393	entailment
+2394	entailment
+2395	entailment
+2396	not_entailment
+2397	not_entailment
+2398	not_entailment
+2399	entailment
+2400	not_entailment
+2401	not_entailment
+2402	not_entailment
+2403	not_entailment
+2404	not_entailment
+2405	entailment
+2406	entailment
+2407	entailment
+2408	entailment
+2409	not_entailment
+2410	not_entailment
+2411	not_entailment
+2412	not_entailment
+2413	entailment
+2414	entailment
+2415	entailment
+2416	not_entailment
+2417	not_entailment
+2418	not_entailment
+2419	entailment
+2420	entailment
+2421	entailment
+2422	entailment
+2423	not_entailment
+2424	not_entailment
+2425	entailment
+2426	entailment
+2427	entailment
+2428	not_entailment
+2429	not_entailment
+2430	entailment
+2431	not_entailment
+2432	entailment
+2433	not_entailment
+2434	not_entailment
+2435	entailment
+2436	entailment
+2437	entailment
+2438	not_entailment
+2439	not_entailment
+2440	entailment
+2441	entailment
+2442	not_entailment
+2443	not_entailment
+2444	entailment
+2445	entailment
+2446	not_entailment
+2447	entailment
+2448	not_entailment
+2449	entailment
+2450	entailment
+2451	entailment
+2452	entailment
+2453	entailment
+2454	not_entailment
+2455	not_entailment
+2456	entailment
+2457	entailment
+2458	not_entailment
+2459	entailment
+2460	entailment
+2461	not_entailment
+2462	entailment
+2463	not_entailment
+2464	entailment
+2465	entailment
+2466	not_entailment
+2467	entailment
+2468	entailment
+2469	entailment
+2470	entailment
+2471	not_entailment
+2472	not_entailment
+2473	not_entailment
+2474	not_entailment
+2475	not_entailment
+2476	not_entailment
+2477	entailment
+2478	entailment
+2479	entailment
+2480	entailment
+2481	entailment
+2482	entailment
+2483	not_entailment
+2484	not_entailment
+2485	entailment
+2486	not_entailment
+2487	not_entailment
+2488	entailment
+2489	not_entailment
+2490	entailment
+2491	not_entailment
+2492	not_entailment
+2493	entailment
+2494	not_entailment
+2495	entailment
+2496	not_entailment
+2497	entailment
+2498	not_entailment
+2499	not_entailment
+2500	entailment
+2501	not_entailment
+2502	entailment
+2503	entailment
+2504	entailment
+2505	entailment
+2506	entailment
+2507	entailment
+2508	not_entailment
+2509	not_entailment
+2510	not_entailment
+2511	entailment
+2512	entailment
+2513	not_entailment
+2514	not_entailment
+2515	entailment
+2516	not_entailment
+2517	not_entailment
+2518	not_entailment
+2519	entailment
+2520	not_entailment
+2521	not_entailment
+2522	entailment
+2523	entailment
+2524	not_entailment
+2525	entailment
+2526	not_entailment
+2527	not_entailment
+2528	not_entailment
+2529	entailment
+2530	not_entailment
+2531	not_entailment
+2532	not_entailment
+2533	entailment
+2534	not_entailment
+2535	not_entailment
+2536	entailment
+2537	not_entailment
+2538	not_entailment
+2539	entailment
+2540	not_entailment
+2541	entailment
+2542	entailment
+2543	entailment
+2544	entailment
+2545	entailment
+2546	not_entailment
+2547	entailment
+2548	not_entailment
+2549	not_entailment
+2550	entailment
+2551	entailment
+2552	entailment
+2553	entailment
+2554	entailment
+2555	entailment
+2556	entailment
+2557	entailment
+2558	entailment
+2559	entailment
+2560	not_entailment
+2561	not_entailment
+2562	entailment
+2563	entailment
+2564	entailment
+2565	entailment
+2566	not_entailment
+2567	entailment
+2568	entailment
+2569	entailment
+2570	entailment
+2571	not_entailment
+2572	not_entailment
+2573	entailment
+2574	not_entailment
+2575	entailment
+2576	not_entailment
+2577	entailment
+2578	not_entailment
+2579	not_entailment
+2580	entailment
+2581	not_entailment
+2582	not_entailment
+2583	entailment
+2584	entailment
+2585	not_entailment
+2586	entailment
+2587	entailment
+2588	not_entailment
+2589	entailment
+2590	entailment
+2591	not_entailment
+2592	not_entailment
+2593	entailment
+2594	not_entailment
+2595	not_entailment
+2596	entailment
+2597	not_entailment
+2598	entailment
+2599	not_entailment
+2600	entailment
+2601	entailment
+2602	not_entailment
+2603	entailment
+2604	entailment
+2605	not_entailment
+2606	entailment
+2607	not_entailment
+2608	not_entailment
+2609	not_entailment
+2610	not_entailment
+2611	entailment
+2612	not_entailment
+2613	entailment
+2614	not_entailment
+2615	entailment
+2616	not_entailment
+2617	not_entailment
+2618	entailment
+2619	entailment
+2620	not_entailment
+2621	entailment
+2622	entailment
+2623	not_entailment
+2624	entailment
+2625	entailment
+2626	entailment
+2627	entailment
+2628	not_entailment
+2629	not_entailment
+2630	entailment
+2631	entailment
+2632	entailment
+2633	not_entailment
+2634	not_entailment
+2635	entailment
+2636	not_entailment
+2637	not_entailment
+2638	not_entailment
+2639	not_entailment
+2640	not_entailment
+2641	entailment
+2642	not_entailment
+2643	not_entailment
+2644	entailment
+2645	entailment
+2646	entailment
+2647	not_entailment
+2648	entailment
+2649	entailment
+2650	not_entailment
+2651	not_entailment
+2652	not_entailment
+2653	entailment
+2654	entailment
+2655	not_entailment
+2656	entailment
+2657	not_entailment
+2658	not_entailment
+2659	entailment
+2660	entailment
+2661	not_entailment
+2662	not_entailment
+2663	entailment
+2664	not_entailment
+2665	not_entailment
+2666	not_entailment
+2667	not_entailment
+2668	entailment
+2669	entailment
+2670	not_entailment
+2671	entailment
+2672	entailment
+2673	not_entailment
+2674	not_entailment
+2675	not_entailment
+2676	not_entailment
+2677	entailment
+2678	not_entailment
+2679	not_entailment
+2680	not_entailment
+2681	not_entailment
+2682	not_entailment
+2683	not_entailment
+2684	entailment
+2685	entailment
+2686	not_entailment
+2687	entailment
+2688	not_entailment
+2689	not_entailment
+2690	not_entailment
+2691	entailment
+2692	entailment
+2693	not_entailment
+2694	not_entailment
+2695	entailment
+2696	entailment
+2697	entailment
+2698	entailment
+2699	entailment
+2700	entailment
+2701	not_entailment
+2702	entailment
+2703	not_entailment
+2704	entailment
+2705	entailment
+2706	not_entailment
+2707	not_entailment
+2708	entailment
+2709	not_entailment
+2710	not_entailment
+2711	entailment
+2712	entailment
+2713	entailment
+2714	entailment
+2715	entailment
+2716	entailment
+2717	entailment
+2718	entailment
+2719	not_entailment
+2720	not_entailment
+2721	entailment
+2722	not_entailment
+2723	not_entailment
+2724	not_entailment
+2725	entailment
+2726	not_entailment
+2727	entailment
+2728	not_entailment
+2729	entailment
+2730	entailment
+2731	not_entailment
+2732	entailment
+2733	not_entailment
+2734	not_entailment
+2735	not_entailment
+2736	entailment
+2737	entailment
+2738	not_entailment
+2739	entailment
+2740	not_entailment
+2741	not_entailment
+2742	entailment
+2743	not_entailment
+2744	entailment
+2745	entailment
+2746	entailment
+2747	entailment
+2748	entailment
+2749	not_entailment
+2750	not_entailment
+2751	entailment
+2752	entailment
+2753	entailment
+2754	not_entailment
+2755	not_entailment
+2756	not_entailment
+2757	not_entailment
+2758	not_entailment
+2759	not_entailment
+2760	entailment
+2761	not_entailment
+2762	entailment
+2763	entailment
+2764	not_entailment
+2765	entailment
+2766	not_entailment
+2767	not_entailment
+2768	not_entailment
+2769	entailment
+2770	entailment
+2771	not_entailment
+2772	entailment
+2773	entailment
+2774	not_entailment
+2775	not_entailment
+2776	entailment
+2777	entailment
+2778	not_entailment
+2779	not_entailment
+2780	entailment
+2781	not_entailment
+2782	entailment
+2783	entailment
+2784	not_entailment
+2785	not_entailment
+2786	not_entailment
+2787	entailment
+2788	not_entailment
+2789	entailment
+2790	not_entailment
+2791	not_entailment
+2792	not_entailment
+2793	entailment
+2794	entailment
+2795	not_entailment
+2796	entailment
+2797	entailment
+2798	not_entailment
+2799	not_entailment
+2800	entailment
+2801	not_entailment
+2802	not_entailment
+2803	entailment
+2804	not_entailment
+2805	entailment
+2806	not_entailment
+2807	entailment
+2808	not_entailment
+2809	entailment
+2810	entailment
+2811	entailment
+2812	entailment
+2813	entailment
+2814	entailment
+2815	entailment
+2816	not_entailment
+2817	not_entailment
+2818	entailment
+2819	not_entailment
+2820	not_entailment
+2821	entailment
+2822	not_entailment
+2823	entailment
+2824	entailment
+2825	entailment
+2826	entailment
+2827	entailment
+2828	entailment
+2829	not_entailment
+2830	not_entailment
+2831	not_entailment
+2832	not_entailment
+2833	not_entailment
+2834	not_entailment
+2835	entailment
+2836	not_entailment
+2837	not_entailment
+2838	not_entailment
+2839	not_entailment
+2840	not_entailment
+2841	entailment
+2842	entailment
+2843	entailment
+2844	not_entailment
+2845	entailment
+2846	entailment
+2847	not_entailment
+2848	not_entailment
+2849	not_entailment
+2850	not_entailment
+2851	not_entailment
+2852	entailment
+2853	entailment
+2854	not_entailment
+2855	entailment
+2856	not_entailment
+2857	entailment
+2858	entailment
+2859	not_entailment
+2860	entailment
+2861	not_entailment
+2862	entailment
+2863	not_entailment
+2864	not_entailment
+2865	entailment
+2866	entailment
+2867	not_entailment
+2868	entailment
+2869	entailment
+2870	not_entailment
+2871	entailment
+2872	entailment
+2873	entailment
+2874	entailment
+2875	not_entailment
+2876	entailment
+2877	entailment
+2878	not_entailment
+2879	entailment
+2880	entailment
+2881	not_entailment
+2882	not_entailment
+2883	not_entailment
+2884	entailment
+2885	entailment
+2886	not_entailment
+2887	entailment
+2888	not_entailment
+2889	entailment
+2890	entailment
+2891	entailment
+2892	not_entailment
+2893	entailment
+2894	not_entailment
+2895	entailment
+2896	not_entailment
+2897	not_entailment
+2898	not_entailment
+2899	entailment
+2900	not_entailment
+2901	entailment
+2902	entailment
+2903	entailment
+2904	not_entailment
+2905	not_entailment
+2906	not_entailment
+2907	not_entailment
+2908	entailment
+2909	entailment
+2910	entailment
+2911	entailment
+2912	not_entailment
+2913	entailment
+2914	entailment
+2915	not_entailment
+2916	not_entailment
+2917	not_entailment
+2918	entailment
+2919	entailment
+2920	not_entailment
+2921	not_entailment
+2922	entailment
+2923	entailment
+2924	not_entailment
+2925	not_entailment
+2926	not_entailment
+2927	not_entailment
+2928	not_entailment
+2929	not_entailment
+2930	not_entailment
+2931	entailment
+2932	not_entailment
+2933	entailment
+2934	entailment
+2935	not_entailment
+2936	entailment
+2937	not_entailment
+2938	not_entailment
+2939	entailment
+2940	entailment
+2941	not_entailment
+2942	not_entailment
+2943	entailment
+2944	entailment
+2945	entailment
+2946	not_entailment
+2947	entailment
+2948	entailment
+2949	entailment
+2950	not_entailment
+2951	entailment
+2952	not_entailment
+2953	not_entailment
+2954	not_entailment
+2955	entailment
+2956	not_entailment
+2957	not_entailment
+2958	not_entailment
+2959	not_entailment
+2960	entailment
+2961	not_entailment
+2962	entailment
+2963	not_entailment
+2964	not_entailment
+2965	not_entailment
+2966	not_entailment
+2967	entailment
+2968	not_entailment
+2969	entailment
+2970	entailment
+2971	entailment
+2972	not_entailment
+2973	entailment
+2974	entailment
+2975	not_entailment
+2976	entailment
+2977	entailment
+2978	entailment
+2979	not_entailment
+2980	entailment
+2981	entailment
+2982	not_entailment
+2983	entailment
+2984	entailment
+2985	entailment
+2986	not_entailment
+2987	not_entailment
+2988	entailment
+2989	entailment
+2990	not_entailment
+2991	entailment
+2992	entailment
+2993	not_entailment
+2994	entailment
+2995	entailment
+2996	not_entailment
+2997	entailment
+2998	not_entailment
+2999	not_entailment

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/all_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 30.0,
+    "eval_accuracy": 0.9025270758122743,
+    "eval_loss": 0.8071204423904419,
+    "eval_runtime": 0.4563,
+    "eval_samples": 277,
+    "eval_samples_per_second": 607.117,
+    "eval_steps_per_second": 2.192
+}

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/eval_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 30.0,
+    "eval_accuracy": 0.9025270758122743,
+    "eval_loss": 0.8071204423904419,
+    "eval_runtime": 0.4563,
+    "eval_samples": 277,
+    "eval_samples_per_second": 607.117,
+    "eval_steps_per_second": 2.192
+}

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/adapter_config.json ADDED Viewed

	@@ -0,0 +1,26 @@

+{
+  "T": 1.0,
+  "base_model_name_or_path": "microsoft/deberta-v3-base",
+  "bias": "none",
+  "drop_out": 0.15,
+  "inference_mode": false,
+  "layers_to_transform": null,
+  "modules_to_save": [
+    "classifier",
+    "pooler"
+  ],
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 4,
+  "revision": null,
+  "target_modules": [
+    "key_proj",
+    "output.dense",
+    "intermediate.dense",
+    "query_proj",
+    "attention.output.dense",
+    "value_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "SEQ_CLS"
+}

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/added_tokens.json ADDED Viewed

	@@ -0,0 +1,3 @@

+{
+  "[MASK]": 128000
+}

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,15 @@

+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/spm.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,60 @@

+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft2/adapter_config.json ADDED Viewed

	@@ -0,0 +1,26 @@

+{
+  "T": 1.0,
+  "base_model_name_or_path": "microsoft/deberta-v3-base",
+  "bias": "none",
+  "drop_out": 0.15,
+  "inference_mode": true,
+  "layers_to_transform": null,
+  "modules_to_save": [
+    "classifier",
+    "pooler"
+  ],
+  "num_rotations": 1,
+  "peft_type": "ROTATION",
+  "r": 4,
+  "revision": null,
+  "target_modules": [
+    "key_proj",
+    "output.dense",
+    "intermediate.dense",
+    "query_proj",
+    "attention.output.dense",
+    "value_proj"
+  ],
+  "target_modules_to_skip": null,
+  "task_type": "SEQ_CLS"
+}

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/ft2/adapter_model.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:38f0ec60401ea6883447e8300f883d980e31d46eb9ee6edfc58bee5349bb3c6d
+size 7449859

nlu/glue_exp/rte/1dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/trainer_state.json ADDED Viewed

	@@ -0,0 +1,411 @@

+{
+  "best_global_step": 1600,
+  "best_metric": 0.9025270758122743,
+  "best_model_checkpoint": "./glue_exp/rte/dr0.15,mlr5e-04,clr2e-03,ep=30.0t=18d21h28m30/checkpoint-1600",
+  "epoch": 30.0,
+  "eval_steps": 100,
+  "global_step": 2340,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.282051282051282,
+      "grad_norm": 1.9696931838989258,
+      "learning_rate": 0.00198,
+      "loss": 0.6596,
+      "step": 100
+    },
+    {
+      "epoch": 1.282051282051282,
+      "eval_accuracy": 0.7833935018050542,
+      "eval_loss": 0.5189762115478516,
+      "eval_runtime": 0.5489,
+      "eval_samples_per_second": 504.682,
+      "eval_steps_per_second": 1.822,
+      "step": 100
+    },
+    {
+      "epoch": 2.564102564102564,
+      "grad_norm": 4.750543117523193,
+      "learning_rate": 0.0019901135883728755,
+      "loss": 0.4184,
+      "step": 200
+    },
+    {
+      "epoch": 2.564102564102564,
+      "eval_accuracy": 0.8050541516245487,
+      "eval_loss": 0.486526221036911,
+      "eval_runtime": 0.4628,
+      "eval_samples_per_second": 598.492,
+      "eval_steps_per_second": 2.161,
+      "step": 200
+    },
+    {
+      "epoch": 3.8461538461538463,
+      "grad_norm": 6.372544765472412,
+      "learning_rate": 0.001960254016408426,
+      "loss": 0.3039,
+      "step": 300
+    },
+    {
+      "epoch": 3.8461538461538463,
+      "eval_accuracy": 0.8158844765342961,
+      "eval_loss": 0.45899271965026855,
+      "eval_runtime": 0.4548,
+      "eval_samples_per_second": 609.084,
+      "eval_steps_per_second": 2.199,
+      "step": 300
+    },
+    {
+      "epoch": 5.128205128205128,
+      "grad_norm": 4.390738010406494,
+      "learning_rate": 0.0019110226492460884,
+      "loss": 0.1707,
+      "step": 400
+    },
+    {
+      "epoch": 5.128205128205128,
+      "eval_accuracy": 0.855595667870036,
+      "eval_loss": 0.46550631523132324,
+      "eval_runtime": 0.4618,
+      "eval_samples_per_second": 599.885,
+      "eval_steps_per_second": 2.166,
+      "step": 400
+    },
+    {
+      "epoch": 6.410256410256411,
+      "grad_norm": 4.9435834884643555,
+      "learning_rate": 0.0018434126615518196,
+      "loss": 0.1205,
+      "step": 500
+    },
+    {
+      "epoch": 6.410256410256411,
+      "eval_accuracy": 0.8664259927797834,
+      "eval_loss": 0.580657422542572,
+      "eval_runtime": 0.4664,
+      "eval_samples_per_second": 593.97,
+      "eval_steps_per_second": 2.144,
+      "step": 500
+    },
+    {
+      "epoch": 7.6923076923076925,
+      "grad_norm": 10.373971939086914,
+      "learning_rate": 0.0017587879912134756,
+      "loss": 0.0955,
+      "step": 600
+    },
+    {
+      "epoch": 7.6923076923076925,
+      "eval_accuracy": 0.8447653429602888,
+      "eval_loss": 0.9098978638648987,
+      "eval_runtime": 0.4594,
+      "eval_samples_per_second": 602.925,
+      "eval_steps_per_second": 2.177,
+      "step": 600
+    },
+    {
+      "epoch": 8.974358974358974,
+      "grad_norm": 0.25669166445732117,
+      "learning_rate": 0.0016588558237824534,
+      "loss": 0.0601,
+      "step": 700
+    },
+    {
+      "epoch": 8.974358974358974,
+      "eval_accuracy": 0.8592057761732852,
+      "eval_loss": 0.7891349792480469,
+      "eval_runtime": 0.46,
+      "eval_samples_per_second": 602.238,
+      "eval_steps_per_second": 2.174,
+      "step": 700
+    },
+    {
+      "epoch": 10.256410256410255,
+      "grad_norm": 0.056794606149196625,
+      "learning_rate": 0.0015456321523689876,
+      "loss": 0.0391,
+      "step": 800
+    },
+    {
+      "epoch": 10.256410256410255,
+      "eval_accuracy": 0.8700361010830325,
+      "eval_loss": 0.8846717476844788,
+      "eval_runtime": 0.4637,
+      "eval_samples_per_second": 597.326,
+      "eval_steps_per_second": 2.156,
+      "step": 800
+    },
+    {
+      "epoch": 11.538461538461538,
+      "grad_norm": 0.22149260342121124,
+      "learning_rate": 0.0014214011077725293,
+      "loss": 0.0627,
+      "step": 900
+    },
+    {
+      "epoch": 11.538461538461538,
+      "eval_accuracy": 0.8736462093862816,
+      "eval_loss": 0.5859602093696594,
+      "eval_runtime": 0.4545,
+      "eval_samples_per_second": 609.489,
+      "eval_steps_per_second": 2.2,
+      "step": 900
+    },
+    {
+      "epoch": 12.820512820512821,
+      "grad_norm": 14.073284149169922,
+      "learning_rate": 0.0012886688793055097,
+      "loss": 0.0429,
+      "step": 1000
+    },
+    {
+      "epoch": 12.820512820512821,
+      "eval_accuracy": 0.8592057761732852,
+      "eval_loss": 0.7581544518470764,
+      "eval_runtime": 0.4614,
+      "eval_samples_per_second": 600.386,
+      "eval_steps_per_second": 2.167,
+      "step": 1000
+    },
+    {
+      "epoch": 14.102564102564102,
+      "grad_norm": 0.21496029198169708,
+      "learning_rate": 0.0011501131558940736,
+      "loss": 0.0353,
+      "step": 1100
+    },
+    {
+      "epoch": 14.102564102564102,
+      "eval_accuracy": 0.8700361010830325,
+      "eval_loss": 0.6726717352867126,
+      "eval_runtime": 0.4603,
+      "eval_samples_per_second": 601.72,
+      "eval_steps_per_second": 2.172,
+      "step": 1100
+    },
+    {
+      "epoch": 15.384615384615385,
+      "grad_norm": 0.010951673611998558,
+      "learning_rate": 0.0010085291074115638,
+      "loss": 0.0226,
+      "step": 1200
+    },
+    {
+      "epoch": 15.384615384615385,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 0.6311701536178589,
+      "eval_runtime": 0.458,
+      "eval_samples_per_second": 604.781,
+      "eval_steps_per_second": 2.183,
+      "step": 1200
+    },
+    {
+      "epoch": 16.666666666666668,
+      "grad_norm": 0.48148414492607117,
+      "learning_rate": 0.0008667729959965649,
+      "loss": 0.011,
+      "step": 1300
+    },
+    {
+      "epoch": 16.666666666666668,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 0.8177604675292969,
+      "eval_runtime": 0.4634,
+      "eval_samples_per_second": 597.743,
+      "eval_steps_per_second": 2.158,
+      "step": 1300
+    },
+    {
+      "epoch": 17.94871794871795,
+      "grad_norm": 0.05897713080048561,
+      "learning_rate": 0.0007277045549190833,
+      "loss": 0.0239,
+      "step": 1400
+    },
+    {
+      "epoch": 17.94871794871795,
+      "eval_accuracy": 0.8664259927797834,
+      "eval_loss": 0.9213047623634338,
+      "eval_runtime": 0.4514,
+      "eval_samples_per_second": 613.595,
+      "eval_steps_per_second": 2.215,
+      "step": 1400
+    },
+    {
+      "epoch": 19.23076923076923,
+      "grad_norm": 0.000974228314589709,
+      "learning_rate": 0.0005941292974214578,
+      "loss": 0.0046,
+      "step": 1500
+    },
+    {
+      "epoch": 19.23076923076923,
+      "eval_accuracy": 0.8664259927797834,
+      "eval_loss": 0.9644713401794434,
+      "eval_runtime": 0.4582,
+      "eval_samples_per_second": 604.586,
+      "eval_steps_per_second": 2.183,
+      "step": 1500
+    },
+    {
+      "epoch": 20.51282051282051,
+      "grad_norm": 0.0010532273445278406,
+      "learning_rate": 0.00046874191937325337,
+      "loss": 0.0045,
+      "step": 1600
+    },
+    {
+      "epoch": 20.51282051282051,
+      "eval_accuracy": 0.9025270758122743,
+      "eval_loss": 0.8071204423904419,
+      "eval_runtime": 0.451,
+      "eval_samples_per_second": 614.137,
+      "eval_steps_per_second": 2.217,
+      "step": 1600
+    },
+    {
+      "epoch": 21.794871794871796,
+      "grad_norm": 0.0004142747784499079,
+      "learning_rate": 0.00035407193751321286,
+      "loss": 0.0032,
+      "step": 1700
+    },
+    {
+      "epoch": 21.794871794871796,
+      "eval_accuracy": 0.8916967509025271,
+      "eval_loss": 0.8693183064460754,
+      "eval_runtime": 0.464,
+      "eval_samples_per_second": 596.937,
+      "eval_steps_per_second": 2.155,
+      "step": 1700
+    },
+    {
+      "epoch": 23.076923076923077,
+      "grad_norm": 0.0005826250999234617,
+      "learning_rate": 0.00025243265995146156,
+      "loss": 0.0043,
+      "step": 1800
+    },
+    {
+      "epoch": 23.076923076923077,
+      "eval_accuracy": 0.8736462093862816,
+      "eval_loss": 1.0998587608337402,
+      "eval_runtime": 0.452,
+      "eval_samples_per_second": 612.852,
+      "eval_steps_per_second": 2.212,
+      "step": 1800
+    },
+    {
+      "epoch": 24.358974358974358,
+      "grad_norm": 0.0007387926452793181,
+      "learning_rate": 0.0001658745183814303,
+      "loss": 0.0034,
+      "step": 1900
+    },
+    {
+      "epoch": 24.358974358974358,
+      "eval_accuracy": 0.8916967509025271,
+      "eval_loss": 0.9102649092674255,
+      "eval_runtime": 0.4611,
+      "eval_samples_per_second": 600.729,
+      "eval_steps_per_second": 2.169,
+      "step": 1900
+    },
+    {
+      "epoch": 25.641025641025642,
+      "grad_norm": 0.0003036344132851809,
+      "learning_rate": 9.614370345949952e-05,
+      "loss": 0.0046,
+      "step": 2000
+    },
+    {
+      "epoch": 25.641025641025642,
+      "eval_accuracy": 0.8844765342960289,
+      "eval_loss": 0.9732220768928528,
+      "eval_runtime": 0.4493,
+      "eval_samples_per_second": 616.515,
+      "eval_steps_per_second": 2.226,
+      "step": 2000
+    },
+    {
+      "epoch": 26.923076923076923,
+      "grad_norm": 0.000459953211247921,
+      "learning_rate": 4.4646937826311684e-05,
+      "loss": 0.0059,
+      "step": 2100
+    },
+    {
+      "epoch": 26.923076923076923,
+      "eval_accuracy": 0.8916967509025271,
+      "eval_loss": 0.9090432524681091,
+      "eval_runtime": 0.4561,
+      "eval_samples_per_second": 607.359,
+      "eval_steps_per_second": 2.193,
+      "step": 2100
+    },
+    {
+      "epoch": 28.205128205128204,
+      "grad_norm": 0.0002905978763010353,
+      "learning_rate": 1.2423097425266594e-05,
+      "loss": 0.0009,
+      "step": 2200
+    },
+    {
+      "epoch": 28.205128205128204,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 0.8938511610031128,
+      "eval_runtime": 0.4591,
+      "eval_samples_per_second": 603.397,
+      "eval_steps_per_second": 2.178,
+      "step": 2200
+    },
+    {
+      "epoch": 29.487179487179485,
+      "grad_norm": 0.054287414997816086,
+      "learning_rate": 1.2225361880779938e-07,
+      "loss": 0.0009,
+      "step": 2300
+    },
+    {
+      "epoch": 29.487179487179485,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 0.8920240998268127,
+      "eval_runtime": 0.4639,
+      "eval_samples_per_second": 597.164,
+      "eval_steps_per_second": 2.156,
+      "step": 2300
+    },
+    {
+      "epoch": 30.0,
+      "step": 2340,
+      "total_flos": 1.2464312631552e+16,
+      "train_loss": 0.08967419704850206,
+      "train_runtime": 576.0074,
+      "train_samples_per_second": 129.686,
+      "train_steps_per_second": 4.062
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 2340,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 30,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.2464312631552e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/RTE.tsv ADDED Viewed

	@@ -0,0 +1,3001 @@

+index	prediction
+0	not_entailment
+1	not_entailment
+2	entailment
+3	not_entailment
+4	entailment
+5	entailment
+6	entailment
+7	not_entailment
+8	not_entailment
+9	entailment
+10	not_entailment
+11	entailment
+12	not_entailment
+13	entailment
+14	not_entailment
+15	not_entailment
+16	not_entailment
+17	entailment
+18	entailment
+19	not_entailment
+20	entailment
+21	entailment
+22	not_entailment
+23	not_entailment
+24	not_entailment
+25	not_entailment
+26	entailment
+27	entailment
+28	entailment
+29	entailment
+30	not_entailment
+31	not_entailment
+32	not_entailment
+33	not_entailment
+34	not_entailment
+35	entailment
+36	not_entailment
+37	entailment
+38	entailment
+39	entailment
+40	not_entailment
+41	not_entailment
+42	entailment
+43	not_entailment
+44	not_entailment
+45	entailment
+46	entailment
+47	not_entailment
+48	entailment
+49	not_entailment
+50	not_entailment
+51	entailment
+52	not_entailment
+53	entailment
+54	entailment
+55	not_entailment
+56	entailment
+57	not_entailment
+58	not_entailment
+59	entailment
+60	entailment
+61	entailment
+62	not_entailment
+63	entailment
+64	entailment
+65	not_entailment
+66	not_entailment
+67	entailment
+68	not_entailment
+69	entailment
+70	entailment
+71	entailment
+72	entailment
+73	entailment
+74	entailment
+75	entailment
+76	not_entailment
+77	entailment
+78	entailment
+79	not_entailment
+80	not_entailment
+81	entailment
+82	not_entailment
+83	not_entailment
+84	entailment
+85	not_entailment
+86	entailment
+87	not_entailment
+88	entailment
+89	entailment
+90	not_entailment
+91	entailment
+92	not_entailment
+93	entailment
+94	entailment
+95	not_entailment
+96	entailment
+97	not_entailment
+98	not_entailment
+99	entailment
+100	entailment
+101	entailment
+102	not_entailment
+103	entailment
+104	entailment
+105	not_entailment
+106	entailment
+107	not_entailment
+108	not_entailment
+109	entailment
+110	entailment
+111	not_entailment
+112	not_entailment
+113	not_entailment
+114	entailment
+115	not_entailment
+116	entailment
+117	not_entailment
+118	not_entailment
+119	not_entailment
+120	not_entailment
+121	not_entailment
+122	not_entailment
+123	not_entailment
+124	entailment
+125	entailment
+126	entailment
+127	entailment
+128	entailment
+129	entailment
+130	not_entailment
+131	entailment
+132	not_entailment
+133	not_entailment
+134	entailment
+135	not_entailment
+136	not_entailment
+137	not_entailment
+138	entailment
+139	not_entailment
+140	entailment
+141	not_entailment
+142	not_entailment
+143	entailment
+144	entailment
+145	not_entailment
+146	not_entailment
+147	not_entailment
+148	not_entailment
+149	entailment
+150	entailment
+151	entailment
+152	not_entailment
+153	entailment
+154	entailment
+155	not_entailment
+156	not_entailment
+157	entailment
+158	not_entailment
+159	not_entailment
+160	entailment
+161	not_entailment
+162	entailment
+163	not_entailment
+164	not_entailment
+165	entailment
+166	entailment
+167	not_entailment
+168	entailment
+169	not_entailment
+170	entailment
+171	entailment
+172	not_entailment
+173	entailment
+174	entailment
+175	not_entailment
+176	not_entailment
+177	not_entailment
+178	entailment
+179	not_entailment
+180	not_entailment
+181	not_entailment
+182	entailment
+183	not_entailment
+184	entailment
+185	not_entailment
+186	entailment
+187	entailment
+188	not_entailment
+189	not_entailment
+190	entailment
+191	entailment
+192	entailment
+193	not_entailment
+194	entailment
+195	entailment
+196	entailment
+197	not_entailment
+198	not_entailment
+199	not_entailment
+200	entailment
+201	entailment
+202	entailment
+203	not_entailment
+204	not_entailment
+205	entailment
+206	not_entailment
+207	not_entailment
+208	not_entailment
+209	entailment
+210	not_entailment
+211	not_entailment
+212	entailment
+213	entailment
+214	entailment
+215	entailment
+216	not_entailment
+217	not_entailment
+218	entailment
+219	not_entailment
+220	entailment
+221	entailment
+222	entailment
+223	not_entailment
+224	entailment
+225	not_entailment
+226	entailment
+227	entailment
+228	not_entailment
+229	not_entailment
+230	entailment
+231	entailment
+232	not_entailment
+233	not_entailment
+234	entailment
+235	not_entailment
+236	entailment
+237	entailment
+238	not_entailment
+239	entailment
+240	entailment
+241	entailment
+242	entailment
+243	entailment
+244	not_entailment
+245	not_entailment
+246	entailment
+247	not_entailment
+248	entailment
+249	entailment
+250	not_entailment
+251	not_entailment
+252	entailment
+253	not_entailment
+254	entailment
+255	entailment
+256	not_entailment
+257	not_entailment
+258	entailment
+259	entailment
+260	not_entailment
+261	not_entailment
+262	entailment
+263	entailment
+264	entailment
+265	entailment
+266	entailment
+267	not_entailment
+268	entailment
+269	not_entailment
+270	not_entailment
+271	entailment
+272	not_entailment
+273	entailment
+274	entailment
+275	not_entailment
+276	not_entailment
+277	entailment
+278	not_entailment
+279	not_entailment
+280	not_entailment
+281	entailment
+282	entailment
+283	entailment
+284	not_entailment
+285	not_entailment
+286	entailment
+287	not_entailment
+288	not_entailment
+289	entailment
+290	not_entailment
+291	not_entailment
+292	not_entailment
+293	entailment
+294	entailment
+295	not_entailment
+296	not_entailment
+297	not_entailment
+298	entailment
+299	entailment
+300	not_entailment
+301	entailment
+302	entailment
+303	entailment
+304	entailment
+305	entailment
+306	entailment
+307	not_entailment
+308	not_entailment
+309	entailment
+310	not_entailment
+311	entailment
+312	not_entailment
+313	not_entailment
+314	entailment
+315	not_entailment
+316	entailment
+317	not_entailment
+318	entailment
+319	entailment
+320	not_entailment
+321	not_entailment
+322	entailment
+323	entailment
+324	entailment
+325	not_entailment
+326	entailment
+327	entailment
+328	entailment
+329	entailment
+330	not_entailment
+331	not_entailment
+332	not_entailment
+333	not_entailment
+334	not_entailment
+335	entailment
+336	entailment
+337	entailment
+338	entailment
+339	entailment
+340	entailment
+341	entailment
+342	entailment
+343	entailment
+344	not_entailment
+345	entailment
+346	entailment
+347	entailment
+348	not_entailment
+349	not_entailment
+350	entailment
+351	entailment
+352	not_entailment
+353	entailment
+354	entailment
+355	not_entailment
+356	entailment
+357	not_entailment
+358	entailment
+359	entailment
+360	entailment
+361	entailment
+362	entailment
+363	not_entailment
+364	entailment
+365	entailment
+366	not_entailment
+367	entailment
+368	entailment
+369	not_entailment
+370	not_entailment
+371	entailment
+372	not_entailment
+373	entailment
+374	not_entailment
+375	entailment
+376	not_entailment
+377	entailment
+378	not_entailment
+379	not_entailment
+380	not_entailment
+381	not_entailment
+382	entailment
+383	entailment
+384	not_entailment
+385	entailment
+386	not_entailment
+387	entailment
+388	entailment
+389	not_entailment
+390	not_entailment
+391	entailment
+392	entailment
+393	entailment
+394	entailment
+395	entailment
+396	entailment
+397	entailment
+398	entailment
+399	not_entailment
+400	entailment
+401	entailment
+402	entailment
+403	entailment
+404	entailment
+405	entailment
+406	not_entailment
+407	entailment
+408	not_entailment
+409	entailment
+410	not_entailment
+411	entailment
+412	entailment
+413	entailment
+414	not_entailment
+415	not_entailment
+416	not_entailment
+417	not_entailment
+418	not_entailment
+419	not_entailment
+420	entailment
+421	entailment
+422	not_entailment
+423	not_entailment
+424	entailment
+425	entailment
+426	entailment
+427	not_entailment
+428	entailment
+429	entailment
+430	not_entailment
+431	not_entailment
+432	entailment
+433	entailment
+434	not_entailment
+435	not_entailment
+436	not_entailment
+437	entailment
+438	entailment
+439	not_entailment
+440	not_entailment
+441	entailment
+442	entailment
+443	entailment
+444	not_entailment
+445	entailment
+446	entailment
+447	not_entailment
+448	entailment
+449	entailment
+450	entailment
+451	entailment
+452	not_entailment
+453	entailment
+454	not_entailment
+455	not_entailment
+456	entailment
+457	entailment
+458	entailment
+459	entailment
+460	entailment
+461	entailment
+462	entailment
+463	entailment
+464	entailment
+465	entailment
+466	entailment
+467	not_entailment
+468	entailment
+469	entailment
+470	not_entailment
+471	entailment
+472	not_entailment
+473	entailment
+474	entailment
+475	not_entailment
+476	not_entailment
+477	not_entailment
+478	not_entailment
+479	entailment
+480	not_entailment
+481	entailment
+482	entailment
+483	entailment
+484	entailment
+485	entailment
+486	entailment
+487	entailment
+488	not_entailment
+489	entailment
+490	entailment
+491	entailment
+492	not_entailment
+493	entailment
+494	entailment
+495	not_entailment
+496	not_entailment
+497	entailment
+498	entailment
+499	entailment
+500	not_entailment
+501	entailment
+502	entailment
+503	entailment
+504	not_entailment
+505	entailment
+506	entailment
+507	not_entailment
+508	entailment
+509	not_entailment
+510	not_entailment
+511	entailment
+512	not_entailment
+513	entailment
+514	entailment
+515	entailment
+516	not_entailment
+517	entailment
+518	not_entailment
+519	entailment
+520	not_entailment
+521	not_entailment
+522	not_entailment
+523	entailment
+524	entailment
+525	not_entailment
+526	not_entailment
+527	entailment
+528	entailment
+529	not_entailment
+530	entailment
+531	not_entailment
+532	not_entailment
+533	entailment
+534	entailment
+535	entailment
+536	not_entailment
+537	entailment
+538	not_entailment
+539	not_entailment
+540	entailment
+541	not_entailment
+542	not_entailment
+543	entailment
+544	entailment
+545	not_entailment
+546	entailment
+547	entailment
+548	entailment
+549	entailment
+550	not_entailment
+551	entailment
+552	entailment
+553	not_entailment
+554	entailment
+555	not_entailment
+556	not_entailment
+557	entailment
+558	not_entailment
+559	not_entailment
+560	entailment
+561	entailment
+562	not_entailment
+563	entailment
+564	entailment
+565	entailment
+566	entailment
+567	entailment
+568	not_entailment
+569	entailment
+570	not_entailment
+571	entailment
+572	entailment
+573	not_entailment
+574	entailment
+575	entailment
+576	entailment
+577	entailment
+578	not_entailment
+579	entailment
+580	entailment
+581	not_entailment
+582	not_entailment
+583	entailment
+584	entailment
+585	entailment
+586	entailment
+587	not_entailment
+588	entailment
+589	not_entailment
+590	entailment
+591	not_entailment
+592	not_entailment
+593	not_entailment
+594	entailment
+595	entailment
+596	entailment
+597	not_entailment
+598	entailment
+599	not_entailment
+600	entailment
+601	entailment
+602	entailment
+603	not_entailment
+604	not_entailment
+605	entailment
+606	entailment
+607	entailment
+608	not_entailment
+609	entailment
+610	entailment
+611	not_entailment
+612	entailment
+613	entailment
+614	not_entailment
+615	entailment
+616	not_entailment
+617	entailment
+618	not_entailment
+619	entailment
+620	entailment
+621	entailment
+622	entailment
+623	not_entailment
+624	entailment
+625	entailment
+626	entailment
+627	entailment
+628	not_entailment
+629	not_entailment
+630	entailment
+631	entailment
+632	not_entailment
+633	not_entailment
+634	not_entailment
+635	entailment
+636	entailment
+637	entailment
+638	not_entailment
+639	entailment
+640	entailment
+641	not_entailment
+642	entailment
+643	not_entailment
+644	entailment
+645	not_entailment
+646	entailment
+647	entailment
+648	entailment
+649	not_entailment
+650	not_entailment
+651	entailment
+652	entailment
+653	not_entailment
+654	entailment
+655	entailment
+656	entailment
+657	entailment
+658	not_entailment
+659	not_entailment
+660	not_entailment
+661	entailment
+662	entailment
+663	entailment
+664	not_entailment
+665	not_entailment
+666	entailment
+667	entailment
+668	entailment
+669	not_entailment
+670	entailment
+671	entailment
+672	not_entailment
+673	not_entailment
+674	not_entailment
+675	not_entailment
+676	entailment
+677	entailment
+678	entailment
+679	entailment
+680	entailment
+681	entailment
+682	entailment
+683	not_entailment
+684	not_entailment
+685	entailment
+686	entailment
+687	entailment
+688	not_entailment
+689	entailment
+690	entailment
+691	not_entailment
+692	entailment
+693	entailment
+694	entailment
+695	not_entailment
+696	not_entailment
+697	entailment
+698	entailment
+699	entailment
+700	entailment
+701	not_entailment
+702	entailment
+703	not_entailment
+704	not_entailment
+705	not_entailment
+706	not_entailment
+707	entailment
+708	entailment
+709	entailment
+710	not_entailment
+711	entailment
+712	not_entailment
+713	entailment
+714	not_entailment
+715	not_entailment
+716	not_entailment
+717	entailment
+718	not_entailment
+719	not_entailment
+720	entailment
+721	entailment
+722	entailment
+723	not_entailment
+724	entailment
+725	entailment
+726	not_entailment
+727	entailment
+728	entailment
+729	entailment
+730	entailment
+731	entailment
+732	not_entailment
+733	entailment
+734	entailment
+735	entailment
+736	not_entailment
+737	not_entailment
+738	entailment
+739	entailment
+740	not_entailment
+741	entailment
+742	not_entailment
+743	entailment
+744	entailment
+745	entailment
+746	not_entailment
+747	not_entailment
+748	not_entailment
+749	not_entailment
+750	entailment
+751	entailment
+752	not_entailment
+753	not_entailment
+754	entailment
+755	entailment
+756	entailment
+757	entailment
+758	entailment
+759	entailment
+760	entailment
+761	entailment
+762	entailment
+763	entailment
+764	not_entailment
+765	not_entailment
+766	entailment
+767	not_entailment
+768	entailment
+769	not_entailment
+770	entailment
+771	not_entailment
+772	not_entailment
+773	not_entailment
+774	not_entailment
+775	entailment
+776	entailment
+777	entailment
+778	entailment
+779	not_entailment
+780	entailment
+781	entailment
+782	entailment
+783	entailment
+784	entailment
+785	entailment
+786	entailment
+787	entailment
+788	entailment
+789	entailment
+790	not_entailment
+791	entailment
+792	not_entailment
+793	entailment
+794	not_entailment
+795	entailment
+796	not_entailment
+797	entailment
+798	not_entailment
+799	not_entailment
+800	entailment
+801	entailment
+802	not_entailment
+803	entailment
+804	not_entailment
+805	entailment
+806	not_entailment
+807	not_entailment
+808	entailment
+809	not_entailment
+810	not_entailment
+811	entailment
+812	not_entailment
+813	entailment
+814	entailment
+815	entailment
+816	not_entailment
+817	entailment
+818	entailment
+819	not_entailment
+820	entailment
+821	not_entailment
+822	entailment
+823	entailment
+824	entailment
+825	entailment
+826	entailment
+827	entailment
+828	entailment
+829	entailment
+830	entailment
+831	entailment
+832	entailment
+833	not_entailment
+834	entailment
+835	entailment
+836	entailment
+837	not_entailment
+838	not_entailment
+839	entailment
+840	entailment
+841	entailment
+842	entailment
+843	entailment
+844	entailment
+845	entailment
+846	entailment
+847	not_entailment
+848	entailment
+849	not_entailment
+850	not_entailment
+851	entailment
+852	entailment
+853	not_entailment
+854	not_entailment
+855	not_entailment
+856	entailment
+857	not_entailment
+858	entailment
+859	entailment
+860	entailment
+861	not_entailment
+862	entailment
+863	not_entailment
+864	entailment
+865	entailment
+866	entailment
+867	entailment
+868	not_entailment
+869	entailment
+870	entailment
+871	entailment
+872	entailment
+873	entailment
+874	entailment
+875	entailment
+876	not_entailment
+877	entailment
+878	entailment
+879	not_entailment
+880	entailment
+881	entailment
+882	entailment
+883	entailment
+884	entailment
+885	entailment
+886	entailment
+887	entailment
+888	not_entailment
+889	entailment
+890	entailment
+891	entailment
+892	entailment
+893	entailment
+894	entailment
+895	entailment
+896	entailment
+897	not_entailment
+898	not_entailment
+899	not_entailment
+900	not_entailment
+901	not_entailment
+902	entailment
+903	entailment
+904	entailment
+905	entailment
+906	entailment
+907	not_entailment
+908	entailment
+909	entailment
+910	not_entailment
+911	entailment
+912	entailment
+913	entailment
+914	entailment
+915	entailment
+916	entailment
+917	entailment
+918	entailment
+919	not_entailment
+920	entailment
+921	entailment
+922	entailment
+923	not_entailment
+924	not_entailment
+925	entailment
+926	entailment
+927	entailment
+928	entailment
+929	not_entailment
+930	not_entailment
+931	entailment
+932	not_entailment
+933	not_entailment
+934	entailment
+935	entailment
+936	entailment
+937	not_entailment
+938	not_entailment
+939	not_entailment
+940	entailment
+941	entailment
+942	not_entailment
+943	entailment
+944	not_entailment
+945	not_entailment
+946	not_entailment
+947	entailment
+948	not_entailment
+949	not_entailment
+950	entailment
+951	entailment
+952	entailment
+953	not_entailment
+954	entailment
+955	not_entailment
+956	entailment
+957	entailment
+958	not_entailment
+959	entailment
+960	entailment
+961	entailment
+962	entailment
+963	entailment
+964	entailment
+965	entailment
+966	entailment
+967	entailment
+968	entailment
+969	not_entailment
+970	entailment
+971	entailment
+972	not_entailment
+973	entailment
+974	entailment
+975	entailment
+976	entailment
+977	entailment
+978	not_entailment
+979	not_entailment
+980	not_entailment
+981	not_entailment
+982	entailment
+983	entailment
+984	not_entailment
+985	entailment
+986	entailment
+987	entailment
+988	not_entailment
+989	entailment
+990	entailment
+991	not_entailment
+992	not_entailment
+993	not_entailment
+994	entailment
+995	entailment
+996	entailment
+997	not_entailment
+998	entailment
+999	not_entailment
+1000	entailment
+1001	not_entailment
+1002	entailment
+1003	not_entailment
+1004	not_entailment
+1005	entailment
+1006	entailment
+1007	entailment
+1008	entailment
+1009	entailment
+1010	entailment
+1011	not_entailment
+1012	not_entailment
+1013	not_entailment
+1014	not_entailment
+1015	entailment
+1016	entailment
+1017	entailment
+1018	entailment
+1019	entailment
+1020	entailment
+1021	entailment
+1022	not_entailment
+1023	entailment
+1024	entailment
+1025	not_entailment
+1026	not_entailment
+1027	entailment
+1028	entailment
+1029	not_entailment
+1030	not_entailment
+1031	not_entailment
+1032	entailment
+1033	entailment
+1034	entailment
+1035	entailment
+1036	not_entailment
+1037	not_entailment
+1038	entailment
+1039	entailment
+1040	not_entailment
+1041	not_entailment
+1042	entailment
+1043	not_entailment
+1044	not_entailment
+1045	entailment
+1046	not_entailment
+1047	entailment
+1048	entailment
+1049	entailment
+1050	not_entailment
+1051	entailment
+1052	not_entailment
+1053	not_entailment
+1054	entailment
+1055	entailment
+1056	not_entailment
+1057	not_entailment
+1058	not_entailment
+1059	entailment
+1060	entailment
+1061	not_entailment
+1062	not_entailment
+1063	entailment
+1064	not_entailment
+1065	entailment
+1066	entailment
+1067	entailment
+1068	entailment
+1069	not_entailment
+1070	entailment
+1071	entailment
+1072	entailment
+1073	entailment
+1074	not_entailment
+1075	not_entailment
+1076	not_entailment
+1077	entailment
+1078	not_entailment
+1079	not_entailment
+1080	entailment
+1081	not_entailment
+1082	not_entailment
+1083	entailment
+1084	entailment
+1085	entailment
+1086	entailment
+1087	not_entailment
+1088	entailment
+1089	not_entailment
+1090	not_entailment
+1091	entailment
+1092	not_entailment
+1093	not_entailment
+1094	not_entailment
+1095	not_entailment
+1096	entailment
+1097	entailment
+1098	not_entailment
+1099	not_entailment
+1100	not_entailment
+1101	entailment
+1102	not_entailment
+1103	not_entailment
+1104	not_entailment
+1105	not_entailment
+1106	not_entailment
+1107	not_entailment
+1108	not_entailment
+1109	not_entailment
+1110	not_entailment
+1111	not_entailment
+1112	not_entailment
+1113	entailment
+1114	entailment
+1115	not_entailment
+1116	not_entailment
+1117	not_entailment
+1118	not_entailment
+1119	entailment
+1120	not_entailment
+1121	not_entailment
+1122	not_entailment
+1123	entailment
+1124	entailment
+1125	not_entailment
+1126	entailment
+1127	entailment
+1128	not_entailment
+1129	not_entailment
+1130	not_entailment
+1131	not_entailment
+1132	not_entailment
+1133	not_entailment
+1134	not_entailment
+1135	not_entailment
+1136	not_entailment
+1137	not_entailment
+1138	entailment
+1139	entailment
+1140	not_entailment
+1141	entailment
+1142	entailment
+1143	not_entailment
+1144	not_entailment
+1145	entailment
+1146	entailment
+1147	entailment
+1148	not_entailment
+1149	not_entailment
+1150	not_entailment
+1151	not_entailment
+1152	not_entailment
+1153	entailment
+1154	entailment
+1155	entailment
+1156	entailment
+1157	entailment
+1158	not_entailment
+1159	not_entailment
+1160	entailment
+1161	not_entailment
+1162	not_entailment
+1163	not_entailment
+1164	not_entailment
+1165	entailment
+1166	entailment
+1167	entailment
+1168	entailment
+1169	not_entailment
+1170	entailment
+1171	entailment
+1172	entailment
+1173	entailment
+1174	not_entailment
+1175	entailment
+1176	entailment
+1177	entailment
+1178	not_entailment
+1179	not_entailment
+1180	not_entailment
+1181	entailment
+1182	not_entailment
+1183	not_entailment
+1184	not_entailment
+1185	not_entailment
+1186	not_entailment
+1187	not_entailment
+1188	not_entailment
+1189	not_entailment
+1190	not_entailment
+1191	not_entailment
+1192	not_entailment
+1193	not_entailment
+1194	not_entailment
+1195	entailment
+1196	not_entailment
+1197	not_entailment
+1198	not_entailment
+1199	not_entailment
+1200	entailment
+1201	entailment
+1202	not_entailment
+1203	not_entailment
+1204	entailment
+1205	entailment
+1206	entailment
+1207	entailment
+1208	not_entailment
+1209	not_entailment
+1210	entailment
+1211	entailment
+1212	not_entailment
+1213	entailment
+1214	entailment
+1215	not_entailment
+1216	entailment
+1217	not_entailment
+1218	entailment
+1219	entailment
+1220	entailment
+1221	entailment
+1222	entailment
+1223	entailment
+1224	not_entailment
+1225	entailment
+1226	entailment
+1227	not_entailment
+1228	entailment
+1229	entailment
+1230	entailment
+1231	entailment
+1232	entailment
+1233	not_entailment
+1234	entailment
+1235	entailment
+1236	entailment
+1237	entailment
+1238	entailment
+1239	not_entailment
+1240	not_entailment
+1241	entailment
+1242	entailment
+1243	entailment
+1244	entailment
+1245	entailment
+1246	entailment
+1247	not_entailment
+1248	entailment
+1249	entailment
+1250	not_entailment
+1251	entailment
+1252	entailment
+1253	entailment
+1254	not_entailment
+1255	entailment
+1256	not_entailment
+1257	not_entailment
+1258	entailment
+1259	entailment
+1260	not_entailment
+1261	entailment
+1262	not_entailment
+1263	not_entailment
+1264	not_entailment
+1265	entailment
+1266	entailment
+1267	entailment
+1268	not_entailment
+1269	not_entailment
+1270	entailment
+1271	entailment
+1272	entailment
+1273	entailment
+1274	entailment
+1275	entailment
+1276	not_entailment
+1277	entailment
+1278	entailment
+1279	entailment
+1280	entailment
+1281	not_entailment
+1282	entailment
+1283	entailment
+1284	not_entailment
+1285	entailment
+1286	not_entailment
+1287	entailment
+1288	entailment
+1289	not_entailment
+1290	entailment
+1291	entailment
+1292	entailment
+1293	not_entailment
+1294	not_entailment
+1295	not_entailment
+1296	not_entailment
+1297	not_entailment
+1298	entailment
+1299	not_entailment
+1300	entailment
+1301	not_entailment
+1302	entailment
+1303	not_entailment
+1304	entailment
+1305	entailment
+1306	not_entailment
+1307	not_entailment
+1308	entailment
+1309	not_entailment
+1310	not_entailment
+1311	not_entailment
+1312	not_entailment
+1313	entailment
+1314	entailment
+1315	not_entailment
+1316	not_entailment
+1317	entailment
+1318	not_entailment
+1319	not_entailment
+1320	entailment
+1321	entailment
+1322	not_entailment
+1323	not_entailment
+1324	not_entailment
+1325	not_entailment
+1326	entailment
+1327	entailment
+1328	not_entailment
+1329	entailment
+1330	entailment
+1331	not_entailment
+1332	not_entailment
+1333	entailment
+1334	entailment
+1335	entailment
+1336	not_entailment
+1337	entailment
+1338	entailment
+1339	entailment
+1340	entailment
+1341	entailment
+1342	entailment
+1343	entailment
+1344	not_entailment
+1345	not_entailment
+1346	entailment
+1347	entailment
+1348	entailment
+1349	entailment
+1350	not_entailment
+1351	entailment
+1352	entailment
+1353	not_entailment
+1354	entailment
+1355	entailment
+1356	entailment
+1357	entailment
+1358	entailment
+1359	entailment
+1360	not_entailment
+1361	not_entailment
+1362	not_entailment
+1363	not_entailment
+1364	not_entailment
+1365	entailment
+1366	entailment
+1367	not_entailment
+1368	not_entailment
+1369	not_entailment
+1370	not_entailment
+1371	entailment
+1372	not_entailment
+1373	entailment
+1374	entailment
+1375	entailment
+1376	not_entailment
+1377	not_entailment
+1378	not_entailment
+1379	not_entailment
+1380	not_entailment
+1381	entailment
+1382	entailment
+1383	not_entailment
+1384	not_entailment
+1385	not_entailment
+1386	entailment
+1387	entailment
+1388	not_entailment
+1389	not_entailment
+1390	entailment
+1391	not_entailment
+1392	not_entailment
+1393	entailment
+1394	entailment
+1395	not_entailment
+1396	not_entailment
+1397	entailment
+1398	not_entailment
+1399	entailment
+1400	entailment
+1401	not_entailment
+1402	entailment
+1403	entailment
+1404	entailment
+1405	entailment
+1406	entailment
+1407	entailment
+1408	not_entailment
+1409	not_entailment
+1410	entailment
+1411	entailment
+1412	not_entailment
+1413	entailment
+1414	entailment
+1415	entailment
+1416	entailment
+1417	entailment
+1418	entailment
+1419	entailment
+1420	not_entailment
+1421	entailment
+1422	entailment
+1423	entailment
+1424	entailment
+1425	entailment
+1426	entailment
+1427	entailment
+1428	entailment
+1429	not_entailment
+1430	entailment
+1431	entailment
+1432	entailment
+1433	entailment
+1434	entailment
+1435	not_entailment
+1436	entailment
+1437	not_entailment
+1438	entailment
+1439	entailment
+1440	entailment
+1441	entailment
+1442	entailment
+1443	entailment
+1444	entailment
+1445	entailment
+1446	not_entailment
+1447	entailment
+1448	not_entailment
+1449	entailment
+1450	entailment
+1451	not_entailment
+1452	not_entailment
+1453	not_entailment
+1454	not_entailment
+1455	entailment
+1456	entailment
+1457	entailment
+1458	entailment
+1459	entailment
+1460	not_entailment
+1461	entailment
+1462	entailment
+1463	entailment
+1464	not_entailment
+1465	entailment
+1466	entailment
+1467	entailment
+1468	entailment
+1469	not_entailment
+1470	not_entailment
+1471	not_entailment
+1472	not_entailment
+1473	entailment
+1474	entailment
+1475	entailment
+1476	entailment
+1477	entailment
+1478	entailment
+1479	not_entailment
+1480	entailment
+1481	not_entailment
+1482	entailment
+1483	entailment
+1484	entailment
+1485	entailment
+1486	entailment
+1487	entailment
+1488	entailment
+1489	entailment
+1490	entailment
+1491	entailment
+1492	entailment
+1493	entailment
+1494	entailment
+1495	not_entailment
+1496	entailment
+1497	not_entailment
+1498	entailment
+1499	not_entailment
+1500	not_entailment
+1501	entailment
+1502	not_entailment
+1503	not_entailment
+1504	entailment
+1505	entailment
+1506	not_entailment
+1507	entailment
+1508	not_entailment
+1509	not_entailment
+1510	not_entailment
+1511	entailment
+1512	entailment
+1513	not_entailment
+1514	entailment
+1515	entailment
+1516	entailment
+1517	entailment
+1518	entailment
+1519	entailment
+1520	entailment
+1521	entailment
+1522	entailment
+1523	entailment
+1524	entailment
+1525	entailment
+1526	not_entailment
+1527	entailment
+1528	not_entailment
+1529	entailment
+1530	entailment
+1531	entailment
+1532	entailment
+1533	entailment
+1534	entailment
+1535	entailment
+1536	entailment
+1537	not_entailment
+1538	entailment
+1539	not_entailment
+1540	not_entailment
+1541	not_entailment
+1542	not_entailment
+1543	entailment
+1544	entailment
+1545	not_entailment
+1546	not_entailment
+1547	entailment
+1548	entailment
+1549	entailment
+1550	entailment
+1551	entailment
+1552	entailment
+1553	entailment
+1554	not_entailment
+1555	entailment
+1556	entailment
+1557	entailment
+1558	not_entailment
+1559	not_entailment
+1560	entailment
+1561	not_entailment
+1562	entailment
+1563	entailment
+1564	not_entailment
+1565	entailment
+1566	not_entailment
+1567	entailment
+1568	not_entailment
+1569	entailment
+1570	entailment
+1571	not_entailment
+1572	entailment
+1573	entailment
+1574	not_entailment
+1575	not_entailment
+1576	entailment
+1577	entailment
+1578	entailment
+1579	not_entailment
+1580	entailment
+1581	not_entailment
+1582	not_entailment
+1583	not_entailment
+1584	not_entailment
+1585	entailment
+1586	entailment
+1587	entailment
+1588	entailment
+1589	not_entailment
+1590	entailment
+1591	not_entailment
+1592	not_entailment
+1593	not_entailment
+1594	entailment
+1595	entailment
+1596	entailment
+1597	entailment
+1598	entailment
+1599	entailment
+1600	not_entailment
+1601	entailment
+1602	entailment
+1603	not_entailment
+1604	not_entailment
+1605	entailment
+1606	entailment
+1607	not_entailment
+1608	entailment
+1609	entailment
+1610	entailment
+1611	not_entailment
+1612	entailment
+1613	entailment
+1614	not_entailment
+1615	not_entailment
+1616	not_entailment
+1617	entailment
+1618	not_entailment
+1619	not_entailment
+1620	not_entailment
+1621	entailment
+1622	not_entailment
+1623	entailment
+1624	entailment
+1625	not_entailment
+1626	not_entailment
+1627	not_entailment
+1628	not_entailment
+1629	entailment
+1630	not_entailment
+1631	not_entailment
+1632	entailment
+1633	entailment
+1634	entailment
+1635	entailment
+1636	entailment
+1637	not_entailment
+1638	not_entailment
+1639	not_entailment
+1640	entailment
+1641	not_entailment
+1642	not_entailment
+1643	entailment
+1644	not_entailment
+1645	not_entailment
+1646	not_entailment
+1647	not_entailment
+1648	not_entailment
+1649	not_entailment
+1650	entailment
+1651	entailment
+1652	not_entailment
+1653	not_entailment
+1654	entailment
+1655	entailment
+1656	entailment
+1657	entailment
+1658	entailment
+1659	entailment
+1660	not_entailment
+1661	entailment
+1662	entailment
+1663	not_entailment
+1664	entailment
+1665	not_entailment
+1666	entailment
+1667	not_entailment
+1668	entailment
+1669	entailment
+1670	entailment
+1671	entailment
+1672	not_entailment
+1673	not_entailment
+1674	entailment
+1675	not_entailment
+1676	entailment
+1677	entailment
+1678	not_entailment
+1679	entailment
+1680	entailment
+1681	not_entailment
+1682	not_entailment
+1683	entailment
+1684	not_entailment
+1685	not_entailment
+1686	entailment
+1687	entailment
+1688	not_entailment
+1689	not_entailment
+1690	entailment
+1691	entailment
+1692	not_entailment
+1693	not_entailment
+1694	not_entailment
+1695	entailment
+1696	entailment
+1697	not_entailment
+1698	entailment
+1699	not_entailment
+1700	entailment
+1701	entailment
+1702	entailment
+1703	entailment
+1704	entailment
+1705	not_entailment
+1706	not_entailment
+1707	entailment
+1708	not_entailment
+1709	not_entailment
+1710	not_entailment
+1711	not_entailment
+1712	entailment
+1713	entailment
+1714	not_entailment
+1715	entailment
+1716	entailment
+1717	entailment
+1718	not_entailment
+1719	not_entailment
+1720	not_entailment
+1721	entailment
+1722	entailment
+1723	not_entailment
+1724	entailment
+1725	entailment
+1726	entailment
+1727	entailment
+1728	entailment
+1729	entailment
+1730	not_entailment
+1731	entailment
+1732	not_entailment
+1733	entailment
+1734	entailment
+1735	not_entailment
+1736	entailment
+1737	entailment
+1738	not_entailment
+1739	not_entailment
+1740	entailment
+1741	entailment
+1742	not_entailment
+1743	not_entailment
+1744	entailment
+1745	not_entailment
+1746	entailment
+1747	not_entailment
+1748	not_entailment
+1749	not_entailment
+1750	not_entailment
+1751	not_entailment
+1752	entailment
+1753	entailment
+1754	not_entailment
+1755	not_entailment
+1756	not_entailment
+1757	not_entailment
+1758	entailment
+1759	entailment
+1760	entailment
+1761	not_entailment
+1762	entailment
+1763	not_entailment
+1764	entailment
+1765	entailment
+1766	not_entailment
+1767	not_entailment
+1768	entailment
+1769	entailment
+1770	entailment
+1771	not_entailment
+1772	not_entailment
+1773	entailment
+1774	entailment
+1775	entailment
+1776	not_entailment
+1777	entailment
+1778	entailment
+1779	entailment
+1780	entailment
+1781	not_entailment
+1782	not_entailment
+1783	entailment
+1784	not_entailment
+1785	entailment
+1786	entailment
+1787	not_entailment
+1788	entailment
+1789	entailment
+1790	entailment
+1791	entailment
+1792	entailment
+1793	not_entailment
+1794	entailment
+1795	entailment
+1796	not_entailment
+1797	not_entailment
+1798	entailment
+1799	not_entailment
+1800	entailment
+1801	entailment
+1802	entailment
+1803	entailment
+1804	not_entailment
+1805	entailment
+1806	not_entailment
+1807	not_entailment
+1808	not_entailment
+1809	entailment
+1810	entailment
+1811	entailment
+1812	not_entailment
+1813	entailment
+1814	entailment
+1815	entailment
+1816	entailment
+1817	entailment
+1818	entailment
+1819	entailment
+1820	not_entailment
+1821	not_entailment
+1822	entailment
+1823	not_entailment
+1824	entailment
+1825	entailment
+1826	entailment
+1827	not_entailment
+1828	entailment
+1829	entailment
+1830	entailment
+1831	not_entailment
+1832	not_entailment
+1833	not_entailment
+1834	entailment
+1835	entailment
+1836	not_entailment
+1837	entailment
+1838	not_entailment
+1839	not_entailment
+1840	entailment
+1841	entailment
+1842	not_entailment
+1843	entailment
+1844	not_entailment
+1845	not_entailment
+1846	not_entailment
+1847	not_entailment
+1848	not_entailment
+1849	entailment
+1850	not_entailment
+1851	not_entailment
+1852	entailment
+1853	not_entailment
+1854	entailment
+1855	not_entailment
+1856	not_entailment
+1857	not_entailment
+1858	entailment
+1859	not_entailment
+1860	entailment
+1861	entailment
+1862	not_entailment
+1863	entailment
+1864	entailment
+1865	not_entailment
+1866	entailment
+1867	not_entailment
+1868	entailment
+1869	entailment
+1870	entailment
+1871	entailment
+1872	entailment
+1873	not_entailment
+1874	entailment
+1875	entailment
+1876	not_entailment
+1877	not_entailment
+1878	not_entailment
+1879	not_entailment
+1880	not_entailment
+1881	entailment
+1882	not_entailment
+1883	not_entailment
+1884	entailment
+1885	entailment
+1886	entailment
+1887	entailment
+1888	not_entailment
+1889	entailment
+1890	not_entailment
+1891	not_entailment
+1892	entailment
+1893	entailment
+1894	entailment
+1895	not_entailment
+1896	entailment
+1897	entailment
+1898	not_entailment
+1899	entailment
+1900	not_entailment
+1901	entailment
+1902	not_entailment
+1903	not_entailment
+1904	not_entailment
+1905	entailment
+1906	entailment
+1907	not_entailment
+1908	not_entailment
+1909	entailment
+1910	entailment
+1911	entailment
+1912	not_entailment
+1913	not_entailment
+1914	entailment
+1915	not_entailment
+1916	not_entailment
+1917	entailment
+1918	not_entailment
+1919	entailment
+1920	not_entailment
+1921	not_entailment
+1922	entailment
+1923	entailment
+1924	entailment
+1925	entailment
+1926	not_entailment
+1927	not_entailment
+1928	not_entailment
+1929	entailment
+1930	entailment
+1931	entailment
+1932	entailment
+1933	not_entailment
+1934	not_entailment
+1935	entailment
+1936	not_entailment
+1937	not_entailment
+1938	not_entailment
+1939	entailment
+1940	entailment
+1941	not_entailment
+1942	not_entailment
+1943	entailment
+1944	entailment
+1945	not_entailment
+1946	entailment
+1947	entailment
+1948	entailment
+1949	not_entailment
+1950	not_entailment
+1951	not_entailment
+1952	not_entailment
+1953	not_entailment
+1954	not_entailment
+1955	not_entailment
+1956	entailment
+1957	entailment
+1958	entailment
+1959	entailment
+1960	entailment
+1961	not_entailment
+1962	entailment
+1963	not_entailment
+1964	not_entailment
+1965	entailment
+1966	entailment
+1967	entailment
+1968	not_entailment
+1969	not_entailment
+1970	entailment
+1971	entailment
+1972	not_entailment
+1973	entailment
+1974	entailment
+1975	not_entailment
+1976	not_entailment
+1977	not_entailment
+1978	entailment
+1979	entailment
+1980	entailment
+1981	entailment
+1982	not_entailment
+1983	entailment
+1984	entailment
+1985	entailment
+1986	not_entailment
+1987	not_entailment
+1988	not_entailment
+1989	entailment
+1990	entailment
+1991	entailment
+1992	not_entailment
+1993	not_entailment
+1994	not_entailment
+1995	entailment
+1996	entailment
+1997	not_entailment
+1998	entailment
+1999	entailment
+2000	not_entailment
+2001	not_entailment
+2002	entailment
+2003	entailment
+2004	not_entailment
+2005	entailment
+2006	not_entailment
+2007	not_entailment
+2008	not_entailment
+2009	not_entailment
+2010	entailment
+2011	entailment
+2012	entailment
+2013	entailment
+2014	entailment
+2015	entailment
+2016	entailment
+2017	entailment
+2018	not_entailment
+2019	entailment
+2020	not_entailment
+2021	not_entailment
+2022	entailment
+2023	entailment
+2024	entailment
+2025	entailment
+2026	entailment
+2027	not_entailment
+2028	not_entailment
+2029	entailment
+2030	not_entailment
+2031	not_entailment
+2032	entailment
+2033	entailment
+2034	not_entailment
+2035	not_entailment
+2036	entailment
+2037	entailment
+2038	entailment
+2039	not_entailment
+2040	entailment
+2041	entailment
+2042	entailment
+2043	not_entailment
+2044	entailment
+2045	entailment
+2046	entailment
+2047	not_entailment
+2048	not_entailment
+2049	entailment
+2050	entailment
+2051	entailment
+2052	not_entailment
+2053	not_entailment
+2054	not_entailment
+2055	entailment
+2056	not_entailment
+2057	not_entailment
+2058	not_entailment
+2059	not_entailment
+2060	not_entailment
+2061	entailment
+2062	entailment
+2063	entailment
+2064	entailment
+2065	entailment
+2066	not_entailment
+2067	entailment
+2068	entailment
+2069	entailment
+2070	entailment
+2071	not_entailment
+2072	entailment
+2073	entailment
+2074	not_entailment
+2075	entailment
+2076	not_entailment
+2077	entailment
+2078	entailment
+2079	entailment
+2080	entailment
+2081	entailment
+2082	not_entailment
+2083	entailment
+2084	not_entailment
+2085	not_entailment
+2086	not_entailment
+2087	entailment
+2088	entailment
+2089	entailment
+2090	not_entailment
+2091	entailment
+2092	not_entailment
+2093	not_entailment
+2094	entailment
+2095	not_entailment
+2096	not_entailment
+2097	entailment
+2098	not_entailment
+2099	entailment
+2100	not_entailment
+2101	not_entailment
+2102	entailment
+2103	not_entailment
+2104	not_entailment
+2105	not_entailment
+2106	entailment
+2107	entailment
+2108	entailment
+2109	not_entailment
+2110	entailment
+2111	entailment
+2112	entailment
+2113	not_entailment
+2114	not_entailment
+2115	entailment
+2116	not_entailment
+2117	entailment
+2118	not_entailment
+2119	entailment
+2120	entailment
+2121	not_entailment
+2122	entailment
+2123	entailment
+2124	not_entailment
+2125	not_entailment
+2126	entailment
+2127	entailment
+2128	entailment
+2129	not_entailment
+2130	entailment
+2131	not_entailment
+2132	not_entailment
+2133	not_entailment
+2134	entailment
+2135	entailment
+2136	not_entailment
+2137	not_entailment
+2138	not_entailment
+2139	not_entailment
+2140	entailment
+2141	not_entailment
+2142	entailment
+2143	not_entailment
+2144	not_entailment
+2145	entailment
+2146	entailment
+2147	not_entailment
+2148	not_entailment
+2149	entailment
+2150	entailment
+2151	not_entailment
+2152	entailment
+2153	entailment
+2154	entailment
+2155	entailment
+2156	entailment
+2157	entailment
+2158	not_entailment
+2159	entailment
+2160	not_entailment
+2161	entailment
+2162	entailment
+2163	not_entailment
+2164	not_entailment
+2165	not_entailment
+2166	not_entailment
+2167	entailment
+2168	not_entailment
+2169	not_entailment
+2170	not_entailment
+2171	entailment
+2172	entailment
+2173	not_entailment
+2174	entailment
+2175	not_entailment
+2176	not_entailment
+2177	entailment
+2178	entailment
+2179	entailment
+2180	not_entailment
+2181	not_entailment
+2182	not_entailment
+2183	not_entailment
+2184	not_entailment
+2185	not_entailment
+2186	entailment
+2187	entailment
+2188	not_entailment
+2189	not_entailment
+2190	not_entailment
+2191	entailment
+2192	entailment
+2193	not_entailment
+2194	not_entailment
+2195	not_entailment
+2196	not_entailment
+2197	entailment
+2198	not_entailment
+2199	entailment
+2200	entailment
+2201	entailment
+2202	not_entailment
+2203	not_entailment
+2204	not_entailment
+2205	entailment
+2206	not_entailment
+2207	not_entailment
+2208	not_entailment
+2209	not_entailment
+2210	not_entailment
+2211	entailment
+2212	entailment
+2213	entailment
+2214	entailment
+2215	entailment
+2216	not_entailment
+2217	not_entailment
+2218	not_entailment
+2219	not_entailment
+2220	not_entailment
+2221	entailment
+2222	not_entailment
+2223	not_entailment
+2224	not_entailment
+2225	not_entailment
+2226	not_entailment
+2227	not_entailment
+2228	not_entailment
+2229	not_entailment
+2230	not_entailment
+2231	entailment
+2232	not_entailment
+2233	not_entailment
+2234	entailment
+2235	not_entailment
+2236	entailment
+2237	not_entailment
+2238	entailment
+2239	not_entailment
+2240	not_entailment
+2241	not_entailment
+2242	entailment
+2243	not_entailment
+2244	entailment
+2245	entailment
+2246	not_entailment
+2247	not_entailment
+2248	not_entailment
+2249	not_entailment
+2250	entailment
+2251	entailment
+2252	entailment
+2253	entailment
+2254	entailment
+2255	entailment
+2256	entailment
+2257	not_entailment
+2258	not_entailment
+2259	entailment
+2260	entailment
+2261	entailment
+2262	not_entailment
+2263	not_entailment
+2264	not_entailment
+2265	entailment
+2266	not_entailment
+2267	entailment
+2268	not_entailment
+2269	entailment
+2270	not_entailment
+2271	entailment
+2272	entailment
+2273	entailment
+2274	entailment
+2275	entailment
+2276	entailment
+2277	not_entailment
+2278	not_entailment
+2279	entailment
+2280	entailment
+2281	not_entailment
+2282	entailment
+2283	not_entailment
+2284	not_entailment
+2285	entailment
+2286	not_entailment
+2287	not_entailment
+2288	entailment
+2289	entailment
+2290	entailment
+2291	entailment
+2292	entailment
+2293	entailment
+2294	entailment
+2295	entailment
+2296	entailment
+2297	entailment
+2298	entailment
+2299	entailment
+2300	entailment
+2301	entailment
+2302	not_entailment
+2303	entailment
+2304	entailment
+2305	entailment
+2306	entailment
+2307	not_entailment
+2308	not_entailment
+2309	not_entailment
+2310	entailment
+2311	not_entailment
+2312	not_entailment
+2313	entailment
+2314	not_entailment
+2315	entailment
+2316	entailment
+2317	entailment
+2318	entailment
+2319	entailment
+2320	not_entailment
+2321	entailment
+2322	entailment
+2323	not_entailment
+2324	entailment
+2325	entailment
+2326	not_entailment
+2327	not_entailment
+2328	not_entailment
+2329	not_entailment
+2330	entailment
+2331	entailment
+2332	entailment
+2333	entailment
+2334	not_entailment
+2335	not_entailment
+2336	entailment
+2337	entailment
+2338	entailment
+2339	not_entailment
+2340	not_entailment
+2341	entailment
+2342	not_entailment
+2343	entailment
+2344	not_entailment
+2345	entailment
+2346	entailment
+2347	entailment
+2348	not_entailment
+2349	entailment
+2350	entailment
+2351	not_entailment
+2352	entailment
+2353	not_entailment
+2354	entailment
+2355	entailment
+2356	entailment
+2357	entailment
+2358	entailment
+2359	entailment
+2360	entailment
+2361	not_entailment
+2362	not_entailment
+2363	not_entailment
+2364	not_entailment
+2365	not_entailment
+2366	not_entailment
+2367	entailment
+2368	entailment
+2369	entailment
+2370	entailment
+2371	entailment
+2372	entailment
+2373	entailment
+2374	entailment
+2375	entailment
+2376	entailment
+2377	entailment
+2378	not_entailment
+2379	not_entailment
+2380	not_entailment
+2381	not_entailment
+2382	entailment
+2383	entailment
+2384	entailment
+2385	not_entailment
+2386	entailment
+2387	entailment
+2388	not_entailment
+2389	entailment
+2390	entailment
+2391	entailment
+2392	entailment
+2393	entailment
+2394	entailment
+2395	entailment
+2396	not_entailment
+2397	entailment
+2398	not_entailment
+2399	entailment
+2400	not_entailment
+2401	not_entailment
+2402	entailment
+2403	not_entailment
+2404	not_entailment
+2405	entailment
+2406	entailment
+2407	entailment
+2408	entailment
+2409	entailment
+2410	not_entailment
+2411	not_entailment
+2412	not_entailment
+2413	entailment
+2414	entailment
+2415	entailment
+2416	entailment
+2417	not_entailment
+2418	not_entailment
+2419	entailment
+2420	entailment
+2421	entailment
+2422	entailment
+2423	not_entailment
+2424	not_entailment
+2425	entailment
+2426	entailment
+2427	entailment
+2428	not_entailment
+2429	not_entailment
+2430	entailment
+2431	not_entailment
+2432	entailment
+2433	not_entailment
+2434	entailment
+2435	entailment
+2436	entailment
+2437	entailment
+2438	not_entailment
+2439	not_entailment
+2440	not_entailment
+2441	entailment
+2442	not_entailment
+2443	not_entailment
+2444	not_entailment
+2445	entailment
+2446	not_entailment
+2447	entailment
+2448	not_entailment
+2449	entailment
+2450	entailment
+2451	entailment
+2452	entailment
+2453	entailment
+2454	not_entailment
+2455	not_entailment
+2456	entailment
+2457	entailment
+2458	not_entailment
+2459	entailment
+2460	entailment
+2461	not_entailment
+2462	entailment
+2463	not_entailment
+2464	entailment
+2465	entailment
+2466	not_entailment
+2467	entailment
+2468	entailment
+2469	entailment
+2470	entailment
+2471	not_entailment
+2472	not_entailment
+2473	not_entailment
+2474	not_entailment
+2475	not_entailment
+2476	entailment
+2477	entailment
+2478	entailment
+2479	entailment
+2480	entailment
+2481	not_entailment
+2482	entailment
+2483	not_entailment
+2484	not_entailment
+2485	entailment
+2486	not_entailment
+2487	not_entailment
+2488	entailment
+2489	not_entailment
+2490	entailment
+2491	not_entailment
+2492	not_entailment
+2493	entailment
+2494	not_entailment
+2495	entailment
+2496	not_entailment
+2497	entailment
+2498	not_entailment
+2499	not_entailment
+2500	entailment
+2501	not_entailment
+2502	entailment
+2503	entailment
+2504	entailment
+2505	entailment
+2506	entailment
+2507	entailment
+2508	not_entailment
+2509	not_entailment
+2510	not_entailment
+2511	entailment
+2512	entailment
+2513	not_entailment
+2514	not_entailment
+2515	entailment
+2516	not_entailment
+2517	not_entailment
+2518	not_entailment
+2519	entailment
+2520	not_entailment
+2521	not_entailment
+2522	entailment
+2523	entailment
+2524	not_entailment
+2525	entailment
+2526	entailment
+2527	entailment
+2528	not_entailment
+2529	entailment
+2530	not_entailment
+2531	not_entailment
+2532	not_entailment
+2533	entailment
+2534	not_entailment
+2535	entailment
+2536	entailment
+2537	not_entailment
+2538	not_entailment
+2539	entailment
+2540	not_entailment
+2541	entailment
+2542	entailment
+2543	entailment
+2544	entailment
+2545	entailment
+2546	not_entailment
+2547	entailment
+2548	not_entailment
+2549	not_entailment
+2550	entailment
+2551	entailment
+2552	entailment
+2553	entailment
+2554	entailment
+2555	entailment
+2556	entailment
+2557	entailment
+2558	entailment
+2559	entailment
+2560	not_entailment
+2561	not_entailment
+2562	entailment
+2563	entailment
+2564	entailment
+2565	entailment
+2566	not_entailment
+2567	entailment
+2568	entailment
+2569	entailment
+2570	entailment
+2571	not_entailment
+2572	not_entailment
+2573	entailment
+2574	not_entailment
+2575	entailment
+2576	not_entailment
+2577	entailment
+2578	not_entailment
+2579	not_entailment
+2580	entailment
+2581	not_entailment
+2582	not_entailment
+2583	entailment
+2584	entailment
+2585	not_entailment
+2586	entailment
+2587	entailment
+2588	not_entailment
+2589	entailment
+2590	entailment
+2591	not_entailment
+2592	not_entailment
+2593	entailment
+2594	not_entailment
+2595	not_entailment
+2596	not_entailment
+2597	not_entailment
+2598	entailment
+2599	not_entailment
+2600	entailment
+2601	entailment
+2602	not_entailment
+2603	entailment
+2604	entailment
+2605	not_entailment
+2606	not_entailment
+2607	not_entailment
+2608	not_entailment
+2609	not_entailment
+2610	not_entailment
+2611	entailment
+2612	not_entailment
+2613	entailment
+2614	not_entailment
+2615	entailment
+2616	entailment
+2617	entailment
+2618	entailment
+2619	entailment
+2620	not_entailment
+2621	entailment
+2622	entailment
+2623	not_entailment
+2624	entailment
+2625	entailment
+2626	entailment
+2627	entailment
+2628	not_entailment
+2629	not_entailment
+2630	entailment
+2631	entailment
+2632	entailment
+2633	not_entailment
+2634	not_entailment
+2635	entailment
+2636	not_entailment
+2637	not_entailment
+2638	not_entailment
+2639	not_entailment
+2640	not_entailment
+2641	entailment
+2642	not_entailment
+2643	not_entailment
+2644	entailment
+2645	entailment
+2646	entailment
+2647	not_entailment
+2648	entailment
+2649	entailment
+2650	not_entailment
+2651	not_entailment
+2652	not_entailment
+2653	not_entailment
+2654	entailment
+2655	not_entailment
+2656	entailment
+2657	not_entailment
+2658	not_entailment
+2659	entailment
+2660	entailment
+2661	not_entailment
+2662	entailment
+2663	entailment
+2664	not_entailment
+2665	not_entailment
+2666	not_entailment
+2667	not_entailment
+2668	entailment
+2669	entailment
+2670	not_entailment
+2671	entailment
+2672	not_entailment
+2673	not_entailment
+2674	not_entailment
+2675	not_entailment
+2676	not_entailment
+2677	entailment
+2678	not_entailment
+2679	not_entailment
+2680	not_entailment
+2681	not_entailment
+2682	not_entailment
+2683	not_entailment
+2684	not_entailment
+2685	not_entailment
+2686	not_entailment
+2687	entailment
+2688	not_entailment
+2689	not_entailment
+2690	not_entailment
+2691	entailment
+2692	entailment
+2693	not_entailment
+2694	not_entailment
+2695	entailment
+2696	entailment
+2697	entailment
+2698	entailment
+2699	entailment
+2700	entailment
+2701	not_entailment
+2702	entailment
+2703	not_entailment
+2704	entailment
+2705	entailment
+2706	not_entailment
+2707	not_entailment
+2708	entailment
+2709	not_entailment
+2710	not_entailment
+2711	entailment
+2712	entailment
+2713	not_entailment
+2714	entailment
+2715	entailment
+2716	entailment
+2717	entailment
+2718	entailment
+2719	not_entailment
+2720	not_entailment
+2721	entailment
+2722	not_entailment
+2723	not_entailment
+2724	not_entailment
+2725	not_entailment
+2726	not_entailment
+2727	not_entailment
+2728	not_entailment
+2729	entailment
+2730	entailment
+2731	not_entailment
+2732	entailment
+2733	not_entailment
+2734	not_entailment
+2735	not_entailment
+2736	entailment
+2737	entailment
+2738	entailment
+2739	not_entailment
+2740	not_entailment
+2741	not_entailment
+2742	entailment
+2743	not_entailment
+2744	entailment
+2745	entailment
+2746	entailment
+2747	entailment
+2748	entailment
+2749	not_entailment
+2750	entailment
+2751	entailment
+2752	entailment
+2753	entailment
+2754	not_entailment
+2755	not_entailment
+2756	not_entailment
+2757	not_entailment
+2758	not_entailment
+2759	not_entailment
+2760	entailment
+2761	not_entailment
+2762	entailment
+2763	entailment
+2764	not_entailment
+2765	entailment
+2766	not_entailment
+2767	not_entailment
+2768	not_entailment
+2769	not_entailment
+2770	entailment
+2771	not_entailment
+2772	entailment
+2773	entailment
+2774	entailment
+2775	not_entailment
+2776	entailment
+2777	entailment
+2778	not_entailment
+2779	entailment
+2780	entailment
+2781	not_entailment
+2782	entailment
+2783	entailment
+2784	not_entailment
+2785	not_entailment
+2786	entailment
+2787	entailment
+2788	not_entailment
+2789	entailment
+2790	not_entailment
+2791	not_entailment
+2792	not_entailment
+2793	entailment
+2794	entailment
+2795	entailment
+2796	entailment
+2797	entailment
+2798	not_entailment
+2799	not_entailment
+2800	entailment
+2801	not_entailment
+2802	not_entailment
+2803	entailment
+2804	not_entailment
+2805	entailment
+2806	not_entailment
+2807	entailment
+2808	not_entailment
+2809	entailment
+2810	entailment
+2811	entailment
+2812	entailment
+2813	entailment
+2814	entailment
+2815	entailment
+2816	not_entailment
+2817	not_entailment
+2818	entailment
+2819	not_entailment
+2820	not_entailment
+2821	not_entailment
+2822	not_entailment
+2823	entailment
+2824	entailment
+2825	entailment
+2826	entailment
+2827	entailment
+2828	entailment
+2829	not_entailment
+2830	not_entailment
+2831	not_entailment
+2832	not_entailment
+2833	not_entailment
+2834	not_entailment
+2835	entailment
+2836	not_entailment
+2837	not_entailment
+2838	not_entailment
+2839	not_entailment
+2840	not_entailment
+2841	entailment
+2842	not_entailment
+2843	entailment
+2844	not_entailment
+2845	entailment
+2846	entailment
+2847	not_entailment
+2848	not_entailment
+2849	not_entailment
+2850	entailment
+2851	not_entailment
+2852	entailment
+2853	entailment
+2854	not_entailment
+2855	entailment
+2856	not_entailment
+2857	entailment
+2858	entailment
+2859	entailment
+2860	entailment
+2861	not_entailment
+2862	entailment
+2863	not_entailment
+2864	not_entailment
+2865	entailment
+2866	entailment
+2867	not_entailment
+2868	entailment
+2869	entailment
+2870	not_entailment
+2871	not_entailment
+2872	entailment
+2873	entailment
+2874	entailment
+2875	not_entailment
+2876	entailment
+2877	entailment
+2878	not_entailment
+2879	entailment
+2880	entailment
+2881	not_entailment
+2882	not_entailment
+2883	not_entailment
+2884	entailment
+2885	entailment
+2886	not_entailment
+2887	entailment
+2888	not_entailment
+2889	entailment
+2890	entailment
+2891	entailment
+2892	not_entailment
+2893	not_entailment
+2894	not_entailment
+2895	entailment
+2896	entailment
+2897	not_entailment
+2898	not_entailment
+2899	entailment
+2900	not_entailment
+2901	entailment
+2902	entailment
+2903	entailment
+2904	not_entailment
+2905	not_entailment
+2906	not_entailment
+2907	not_entailment
+2908	entailment
+2909	entailment
+2910	entailment
+2911	entailment
+2912	not_entailment
+2913	not_entailment
+2914	entailment
+2915	not_entailment
+2916	not_entailment
+2917	not_entailment
+2918	entailment
+2919	entailment
+2920	not_entailment
+2921	not_entailment
+2922	entailment
+2923	entailment
+2924	not_entailment
+2925	not_entailment
+2926	not_entailment
+2927	not_entailment
+2928	not_entailment
+2929	not_entailment
+2930	not_entailment
+2931	entailment
+2932	not_entailment
+2933	entailment
+2934	entailment
+2935	entailment
+2936	entailment
+2937	not_entailment
+2938	not_entailment
+2939	entailment
+2940	not_entailment
+2941	not_entailment
+2942	not_entailment
+2943	entailment
+2944	entailment
+2945	entailment
+2946	not_entailment
+2947	entailment
+2948	entailment
+2949	entailment
+2950	not_entailment
+2951	entailment
+2952	not_entailment
+2953	not_entailment
+2954	not_entailment
+2955	entailment
+2956	not_entailment
+2957	not_entailment
+2958	not_entailment
+2959	not_entailment
+2960	entailment
+2961	not_entailment
+2962	entailment
+2963	not_entailment
+2964	not_entailment
+2965	not_entailment
+2966	not_entailment
+2967	entailment
+2968	not_entailment
+2969	entailment
+2970	entailment
+2971	entailment
+2972	not_entailment
+2973	entailment
+2974	entailment
+2975	not_entailment
+2976	entailment
+2977	entailment
+2978	entailment
+2979	not_entailment
+2980	entailment
+2981	entailment
+2982	not_entailment
+2983	entailment
+2984	entailment
+2985	entailment
+2986	not_entailment
+2987	not_entailment
+2988	entailment
+2989	entailment
+2990	not_entailment
+2991	entailment
+2992	entailment
+2993	not_entailment
+2994	entailment
+2995	entailment
+2996	not_entailment
+2997	entailment
+2998	not_entailment
+2999	not_entailment

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/all_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 30.0,
+    "eval_accuracy": 0.8916967509025271,
+    "eval_loss": 0.9903465509414673,
+    "eval_runtime": 0.4597,
+    "eval_samples": 277,
+    "eval_samples_per_second": 602.522,
+    "eval_steps_per_second": 2.175
+}

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/eval_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "epoch": 30.0,
+    "eval_accuracy": 0.8916967509025271,
+    "eval_loss": 0.9903465509414673,
+    "eval_runtime": 0.4597,
+    "eval_samples": 277,
+    "eval_samples_per_second": 602.522,
+    "eval_steps_per_second": 2.175
+}

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/added_tokens.json ADDED Viewed

	@@ -0,0 +1,3 @@

+{
+  "[MASK]": 128000
+}

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,15 @@

+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/spm.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
+size 2464616

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/ft/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,60 @@

+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128000": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "padding_side": "right",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

nlu/glue_exp/rte/2dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/trainer_state.json ADDED Viewed

	@@ -0,0 +1,411 @@

+{
+  "best_global_step": 1600,
+  "best_metric": 0.8916967509025271,
+  "best_model_checkpoint": "./glue_exp/rte/dr0.15,mlr5e-04,clr1e-03,ep=30.0t=18d21h08m36/checkpoint-1600",
+  "epoch": 30.0,
+  "eval_steps": 100,
+  "global_step": 2340,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.282051282051282,
+      "grad_norm": 3.0298995971679688,
+      "learning_rate": 0.00099,
+      "loss": 0.6535,
+      "step": 100
+    },
+    {
+      "epoch": 1.282051282051282,
+      "eval_accuracy": 0.7978339350180506,
+      "eval_loss": 0.4904983341693878,
+      "eval_runtime": 0.5544,
+      "eval_samples_per_second": 499.638,
+      "eval_steps_per_second": 1.804,
+      "step": 100
+    },
+    {
+      "epoch": 2.564102564102564,
+      "grad_norm": 5.644094944000244,
+      "learning_rate": 0.0009950567941864378,
+      "loss": 0.3826,
+      "step": 200
+    },
+    {
+      "epoch": 2.564102564102564,
+      "eval_accuracy": 0.8303249097472925,
+      "eval_loss": 0.40063101053237915,
+      "eval_runtime": 0.4602,
+      "eval_samples_per_second": 601.893,
+      "eval_steps_per_second": 2.173,
+      "step": 200
+    },
+    {
+      "epoch": 3.8461538461538463,
+      "grad_norm": 4.687230587005615,
+      "learning_rate": 0.000980127008204213,
+      "loss": 0.2649,
+      "step": 300
+    },
+    {
+      "epoch": 3.8461538461538463,
+      "eval_accuracy": 0.8664259927797834,
+      "eval_loss": 0.3754575550556183,
+      "eval_runtime": 0.4604,
+      "eval_samples_per_second": 601.657,
+      "eval_steps_per_second": 2.172,
+      "step": 300
+    },
+    {
+      "epoch": 5.128205128205128,
+      "grad_norm": 2.7542014122009277,
+      "learning_rate": 0.0009555113246230442,
+      "loss": 0.1596,
+      "step": 400
+    },
+    {
+      "epoch": 5.128205128205128,
+      "eval_accuracy": 0.8700361010830325,
+      "eval_loss": 0.4731525778770447,
+      "eval_runtime": 0.4614,
+      "eval_samples_per_second": 600.315,
+      "eval_steps_per_second": 2.167,
+      "step": 400
+    },
+    {
+      "epoch": 6.410256410256411,
+      "grad_norm": 1.4972096681594849,
+      "learning_rate": 0.0009217063307759098,
+      "loss": 0.1288,
+      "step": 500
+    },
+    {
+      "epoch": 6.410256410256411,
+      "eval_accuracy": 0.851985559566787,
+      "eval_loss": 0.6349920034408569,
+      "eval_runtime": 0.4651,
+      "eval_samples_per_second": 595.544,
+      "eval_steps_per_second": 2.15,
+      "step": 500
+    },
+    {
+      "epoch": 7.6923076923076925,
+      "grad_norm": 4.662834644317627,
+      "learning_rate": 0.0008793939956067378,
+      "loss": 0.0925,
+      "step": 600
+    },
+    {
+      "epoch": 7.6923076923076925,
+      "eval_accuracy": 0.8700361010830325,
+      "eval_loss": 0.5567091703414917,
+      "eval_runtime": 0.4608,
+      "eval_samples_per_second": 601.113,
+      "eval_steps_per_second": 2.17,
+      "step": 600
+    },
+    {
+      "epoch": 8.974358974358974,
+      "grad_norm": 7.242234230041504,
+      "learning_rate": 0.0008294279118912267,
+      "loss": 0.0912,
+      "step": 700
+    },
+    {
+      "epoch": 8.974358974358974,
+      "eval_accuracy": 0.8628158844765343,
+      "eval_loss": 0.779809832572937,
+      "eval_runtime": 0.4593,
+      "eval_samples_per_second": 603.125,
+      "eval_steps_per_second": 2.177,
+      "step": 700
+    },
+    {
+      "epoch": 10.256410256410255,
+      "grad_norm": 0.10321938246488571,
+      "learning_rate": 0.0007728160761844938,
+      "loss": 0.0628,
+      "step": 800
+    },
+    {
+      "epoch": 10.256410256410255,
+      "eval_accuracy": 0.8808664259927798,
+      "eval_loss": 0.6088728904724121,
+      "eval_runtime": 0.4514,
+      "eval_samples_per_second": 613.593,
+      "eval_steps_per_second": 2.215,
+      "step": 800
+    },
+    {
+      "epoch": 11.538461538461538,
+      "grad_norm": 0.0666593536734581,
+      "learning_rate": 0.0007107005538862646,
+      "loss": 0.0465,
+      "step": 900
+    },
+    {
+      "epoch": 11.538461538461538,
+      "eval_accuracy": 0.855595667870036,
+      "eval_loss": 0.9525722861289978,
+      "eval_runtime": 0.46,
+      "eval_samples_per_second": 602.236,
+      "eval_steps_per_second": 2.174,
+      "step": 900
+    },
+    {
+      "epoch": 12.820512820512821,
+      "grad_norm": 0.07844485342502594,
+      "learning_rate": 0.0006443344396527548,
+      "loss": 0.0375,
+      "step": 1000
+    },
+    {
+      "epoch": 12.820512820512821,
+      "eval_accuracy": 0.8808664259927798,
+      "eval_loss": 0.7848650217056274,
+      "eval_runtime": 0.4581,
+      "eval_samples_per_second": 604.703,
+      "eval_steps_per_second": 2.183,
+      "step": 1000
+    },
+    {
+      "epoch": 14.102564102564102,
+      "grad_norm": 1.5535038709640503,
+      "learning_rate": 0.0005750565779470368,
+      "loss": 0.0307,
+      "step": 1100
+    },
+    {
+      "epoch": 14.102564102564102,
+      "eval_accuracy": 0.8736462093862816,
+      "eval_loss": 0.8866338133811951,
+      "eval_runtime": 0.456,
+      "eval_samples_per_second": 607.481,
+      "eval_steps_per_second": 2.193,
+      "step": 1100
+    },
+    {
+      "epoch": 15.384615384615385,
+      "grad_norm": 0.0072989086620509624,
+      "learning_rate": 0.0005042645537057819,
+      "loss": 0.0208,
+      "step": 1200
+    },
+    {
+      "epoch": 15.384615384615385,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 0.8754919171333313,
+      "eval_runtime": 0.4117,
+      "eval_samples_per_second": 672.814,
+      "eval_steps_per_second": 2.429,
+      "step": 1200
+    },
+    {
+      "epoch": 16.666666666666668,
+      "grad_norm": 4.213427543640137,
+      "learning_rate": 0.00043338649799828244,
+      "loss": 0.022,
+      "step": 1300
+    },
+    {
+      "epoch": 16.666666666666668,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 0.8318637609481812,
+      "eval_runtime": 0.4614,
+      "eval_samples_per_second": 600.39,
+      "eval_steps_per_second": 2.167,
+      "step": 1300
+    },
+    {
+      "epoch": 17.94871794871795,
+      "grad_norm": 0.0056141032837331295,
+      "learning_rate": 0.00036385227745954163,
+      "loss": 0.0111,
+      "step": 1400
+    },
+    {
+      "epoch": 17.94871794871795,
+      "eval_accuracy": 0.8736462093862816,
+      "eval_loss": 0.9078261256217957,
+      "eval_runtime": 0.459,
+      "eval_samples_per_second": 603.53,
+      "eval_steps_per_second": 2.179,
+      "step": 1400
+    },
+    {
+      "epoch": 19.23076923076923,
+      "grad_norm": 0.0017077454831451178,
+      "learning_rate": 0.0002970646487107289,
+      "loss": 0.012,
+      "step": 1500
+    },
+    {
+      "epoch": 19.23076923076923,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 0.9111207723617554,
+      "eval_runtime": 0.4541,
+      "eval_samples_per_second": 609.988,
+      "eval_steps_per_second": 2.202,
+      "step": 1500
+    },
+    {
+      "epoch": 20.51282051282051,
+      "grad_norm": 0.0007726816111244261,
+      "learning_rate": 0.00023437095968662668,
+      "loss": 0.0053,
+      "step": 1600
+    },
+    {
+      "epoch": 20.51282051282051,
+      "eval_accuracy": 0.8916967509025271,
+      "eval_loss": 0.9903465509414673,
+      "eval_runtime": 0.4604,
+      "eval_samples_per_second": 601.631,
+      "eval_steps_per_second": 2.172,
+      "step": 1600
+    },
+    {
+      "epoch": 21.794871794871796,
+      "grad_norm": 0.0005964247975498438,
+      "learning_rate": 0.00017703596875660643,
+      "loss": 0.006,
+      "step": 1700
+    },
+    {
+      "epoch": 21.794871794871796,
+      "eval_accuracy": 0.8808664259927798,
+      "eval_loss": 1.0135316848754883,
+      "eval_runtime": 0.4563,
+      "eval_samples_per_second": 607.055,
+      "eval_steps_per_second": 2.192,
+      "step": 1700
+    },
+    {
+      "epoch": 23.076923076923077,
+      "grad_norm": 0.00156286614947021,
+      "learning_rate": 0.00012621632997573078,
+      "loss": 0.0045,
+      "step": 1800
+    },
+    {
+      "epoch": 23.076923076923077,
+      "eval_accuracy": 0.8844765342960289,
+      "eval_loss": 1.0323450565338135,
+      "eval_runtime": 0.4591,
+      "eval_samples_per_second": 603.386,
+      "eval_steps_per_second": 2.178,
+      "step": 1800
+    },
+    {
+      "epoch": 24.358974358974358,
+      "grad_norm": 0.06641647219657898,
+      "learning_rate": 8.293725919071515e-05,
+      "loss": 0.0019,
+      "step": 1900
+    },
+    {
+      "epoch": 24.358974358974358,
+      "eval_accuracy": 0.8808664259927798,
+      "eval_loss": 1.0457179546356201,
+      "eval_runtime": 0.4578,
+      "eval_samples_per_second": 605.093,
+      "eval_steps_per_second": 2.184,
+      "step": 1900
+    },
+    {
+      "epoch": 25.641025641025642,
+      "grad_norm": 0.0004662807914428413,
+      "learning_rate": 4.807185172974976e-05,
+      "loss": 0.0048,
+      "step": 2000
+    },
+    {
+      "epoch": 25.641025641025642,
+      "eval_accuracy": 0.8916967509025271,
+      "eval_loss": 1.0062867403030396,
+      "eval_runtime": 0.4592,
+      "eval_samples_per_second": 603.257,
+      "eval_steps_per_second": 2.178,
+      "step": 2000
+    },
+    {
+      "epoch": 26.923076923076923,
+      "grad_norm": 0.0021972227841615677,
+      "learning_rate": 2.2323468913155842e-05,
+      "loss": 0.0012,
+      "step": 2100
+    },
+    {
+      "epoch": 26.923076923076923,
+      "eval_accuracy": 0.8880866425992779,
+      "eval_loss": 1.0196377038955688,
+      "eval_runtime": 0.4445,
+      "eval_samples_per_second": 623.159,
+      "eval_steps_per_second": 2.25,
+      "step": 2100
+    },
+    {
+      "epoch": 28.205128205128204,
+      "grad_norm": 0.0024973733816295862,
+      "learning_rate": 6.211548712633297e-06,
+      "loss": 0.0004,
+      "step": 2200
+    },
+    {
+      "epoch": 28.205128205128204,
+      "eval_accuracy": 0.8844765342960289,
+      "eval_loss": 1.0207929611206055,
+      "eval_runtime": 0.4673,
+      "eval_samples_per_second": 592.762,
+      "eval_steps_per_second": 2.14,
+      "step": 2200
+    },
+    {
+      "epoch": 29.487179487179485,
+      "grad_norm": 0.03873719647526741,
+      "learning_rate": 6.112680940389969e-08,
+      "loss": 0.0007,
+      "step": 2300
+    },
+    {
+      "epoch": 29.487179487179485,
+      "eval_accuracy": 0.8844765342960289,
+      "eval_loss": 1.0203951597213745,
+      "eval_runtime": 0.4603,
+      "eval_samples_per_second": 601.834,
+      "eval_steps_per_second": 2.173,
+      "step": 2300
+    },
+    {
+      "epoch": 30.0,
+      "step": 2340,
+      "total_flos": 1.2464312631552e+16,
+      "train_loss": 0.08723565932934403,
+      "train_runtime": 570.4535,
+      "train_samples_per_second": 130.948,
+      "train_steps_per_second": 4.102
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 2340,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 30,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.2464312631552e+16,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}

nlu/test.sh ADDED Viewed

	@@ -0,0 +1,81 @@

+#!/bin/bash
+export OMINI_CONFIG=./config/glue.yaml
+export TOKENIZERS_PARALLELISM=true
+export CPATH=$CPATH:$CUDA_INCLUDE_PATH
+export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:$CUDA_INCLUDE_PATH
+# echo "CPATH is set to: $CPATH"
+# echo "CPLUS_INCLUDE_PATH is set to: $CPLUS_INCLUDE_PATH"
+# export PYTHONPATH=$PYTHONPATH:$(pwd)/../nl_tasks # for rpeft
+export WANDB_PROJECT="DeBERTaV3-GLUE-Test"
+export OMP_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export NUMEXPR_NUM_THREADS=1
+date +"%F %T"
+MODEL_LRS=("1e-4")
+CLS_LRS=("2e-3")
+DROPOUT_RATES=("0.1")
+TEXT=("oft" "boft" "hra" "loco")
+STEPS=2000
+EPOCHS=11
+for m_lr in "${MODEL_LRS[@]}"; do
+  for c_lr in "${CLS_LRS[@]}"; do
+    for drop_out in "${DROPOUT_RATES[@]}"; do
+        echo ">>> Params: model_lr=$m_lr, cls_lr=$c_lr, dropout=$drop_out, step=$STEPS, epoch=$EPOCHS"
+        python -m src.test \
+            --config_path $OMINI_CONFIG --trainer_args.output_dir "./glue_testYY" --run_text 'oft' \
+            --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 6 \
+            --trainer_args.gradient_accumulation_steps 1 \
+            --glue.is_debug False --rotation_adapter_config.drop_out "$drop_out" \
+            --glue.task_name qnli --trainer_args.metric_for_best_model accuracy \
+            --trainer_args.num_train_epochs $EPOCHS --trainer_args.max_steps=405 --trainer_args.warmup_steps 200 \
+            --glue.model_lr "$m_lr" --glue.cls_lr "$c_lr" \
+            --trainer_args.logging_step $STEPS --trainer_args.eval_step $STEPS --trainer_args.save_steps $STEPS \
+            --trainer_args.report_to none \
+            --glue.max_seq_length 512 \
+            --trainer_args.per_device_train_batch_size 64 --trainer_args.per_device_eval_batch_size 128 \
+            --trainer_args.eval_strategy '"no"' \
+            --trainer_args.load_best_model_at_end False \
+            --trainer_args.save_strategy '"no"'
+    done
+  done
+done
+date +"%F %T"
+# for m_lr in "${MODEL_LRS[@]}"; do
+#   for c_lr in "${CLS_LRS[@]}"; do
+#     for drop_out in "${DROPOUT_RATES[@]}"; do
+#       for text in "${TEXT[@]}"; do
+#         echo ">>> Params: model_lr=$m_lr, cls_lr=$c_lr, dropout=$drop_out, step=$STEPS, epoch=$EPOCHS"
+#         python -m src.test \
+#             --config_path $OMINI_CONFIG --trainer_args.output_dir "./glue_testYY" --run_text "$text" \
+#             --rotation_adapter_config.num_rotations 1 --rotation_adapter_config.r 16 \
+#             --trainer_args.gradient_accumulation_steps 1 \
+#             --glue.is_debug False --rotation_adapter_config.drop_out "$drop_out" \
+#             --glue.task_name qnli --trainer_args.metric_for_best_model accuracy \
+#             --trainer_args.num_train_epochs $EPOCHS --trainer_args.max_steps=405 --trainer_args.warmup_steps 200 \
+#             --glue.model_lr "$m_lr" --glue.cls_lr "$c_lr" \
+#             --trainer_args.logging_step $STEPS --trainer_args.eval_step $STEPS --trainer_args.save_steps $STEPS \
+#             --trainer_args.report_to none \
+#             --glue.max_seq_length 512 \
+#             --trainer_args.per_device_train_batch_size 64 --trainer_args.per_device_eval_batch_size 128 \
+#             --trainer_args.eval_strategy '"no"' \
+#             --trainer_args.load_best_model_at_end False \
+#             --trainer_args.save_strategy '"no"'
+#       done
+#     done
+#   done
+# done

nlu/training_metrics_bs8.json ADDED Viewed

	@@ -0,0 +1,26 @@

+{
+    "metadata": {
+        "run_name": "Experiment_BatchSize_8",
+        "timestamp": "2025-12-25 17:31:34",
+        "python_version": "3.11.3",
+        "pytorch_version": "2.9.0+cu128",
+        "gpu_info": {
+            "name": "NVIDIA H200",
+            "count": 1,
+            "capability": [
+                9,
+                0
+            ]
+        },
+        "configuration": {
+            "batch_size_per_device": 64,
+            "learning_rate": 0.0002,
+            "max_steps": 405,
+            "num_train_epochs": 11.0,
+            "fp16": false,
+            "bf16": false,
+            "optim": "adamw_torch"
+        }
+    },
+    "metrics": []
+}

omini/train_flux/train_spatial_alignment.py ADDED Viewed

	@@ -0,0 +1,211 @@

+import torch
+from torch.utils.data import Dataset
+import torchvision.transforms as T
+import os
+import random
+import numpy as np
+from PIL import Image, ImageDraw
+from datasets import load_dataset
+from .trainer import OminiModel, get_config, train
+from ..pipeline.flux_omini import Condition, convert_to_condition, generate
+class ImageConditionDataset(Dataset):
+    def __init__(
+        self,
+        base_dataset,
+        condition_size=(512, 512),
+        target_size=(512, 512),
+        condition_type: str = "canny",
+        drop_text_prob: float = 0.1,
+        drop_image_prob: float = 0.1,
+        return_pil_image: bool = False,
+        position_scale=1.0,
+    ):
+        self.base_dataset = base_dataset
+        self.condition_size = condition_size
+        self.target_size = target_size
+        self.condition_type = condition_type
+        self.drop_text_prob = drop_text_prob
+        self.drop_image_prob = drop_image_prob
+        self.return_pil_image = return_pil_image
+        self.position_scale = position_scale
+        self.to_tensor = T.ToTensor()
+    def __len__(self):
+        return len(self.base_dataset)
+    def __get_condition__(self, image, condition_type):
+        condition_size = self.condition_size
+        position_delta = np.array([0, 0])
+        if condition_type in ["canny", "coloring", "deblurring", "depth"]:
+            image, kwargs = image.resize(condition_size), {}
+            if condition_type == "deblurring":
+                blur_radius = random.randint(1, 10)
+                kwargs["blur_radius"] = blur_radius
+            condition_img = convert_to_condition(condition_type, image, **kwargs)
+        elif condition_type == "depth_pred":
+            depth_img = convert_to_condition("depth", image)
+            condition_img = image.resize(condition_size)
+            image = depth_img.resize(condition_size)
+        elif condition_type == "fill":
+            condition_img = image.resize(condition_size).convert("RGB")
+            w, h = image.size
+            x1, x2 = sorted([random.randint(0, w), random.randint(0, w)])
+            y1, y2 = sorted([random.randint(0, h), random.randint(0, h)])
+            mask = Image.new("L", image.size, 0)
+            draw = ImageDraw.Draw(mask)
+            draw.rectangle([x1, y1, x2, y2], fill=255)
+            if random.random() > 0.5:
+                mask = Image.eval(mask, lambda a: 255 - a)
+            condition_img = Image.composite(
+                image, Image.new("RGB", image.size, (0, 0, 0)), mask
+            )
+        elif condition_type == "sr":
+            condition_img = image.resize(condition_size)
+            position_delta = np.array([0, -condition_size[0] // 16])
+        else:
+            raise ValueError(f"Condition type {condition_type} is not  implemented.")
+        return condition_img, position_delta
+    def __getitem__(self, idx):
+        image = self.base_dataset[idx]["jpg"]
+        image = image.resize(self.target_size).convert("RGB")
+        description = self.base_dataset[idx]["json"]["prompt"]
+        condition_size = self.condition_size
+        position_scale = self.position_scale
+        condition_img, position_delta = self.__get_condition__(
+            image, self.condition_type
+        )
+        # Randomly drop text or image (for training)
+        drop_text = random.random() < self.drop_text_prob
+        drop_image = random.random() < self.drop_image_prob
+        if drop_text:
+            description = ""
+        if drop_image:
+            condition_img = Image.new("RGB", condition_size, (0, 0, 0))
+        return {
+            "image": self.to_tensor(image),
+            "condition_0": self.to_tensor(condition_img),
+            "condition_type_0": self.condition_type,
+            "position_delta_0": position_delta,
+            "description": description,
+            **({"pil_image": [image, condition_img]} if self.return_pil_image else {}),
+            **({"position_scale_0": position_scale} if position_scale != 1.0 else {}),
+        }
+@torch.no_grad()
+def test_function(model, save_path, file_name):
+    condition_size = model.training_config["dataset"]["condition_size"]
+    target_size = model.training_config["dataset"]["target_size"]
+    position_delta = model.training_config["dataset"].get("position_delta", [0, 0])
+    position_scale = model.training_config["dataset"].get("position_scale", 1.0)
+    adapter = model.adapter_names[2]
+    condition_type = model.training_config["condition_type"]
+    test_list = []
+    if condition_type in ["canny", "coloring", "deblurring", "depth"]:
+        image = Image.open("assets/vase_hq.jpg")
+        image = image.resize(condition_size)
+        condition_img = convert_to_condition(condition_type, image, 5)
+        condition = Condition(condition_img, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    elif condition_type == "depth_pred":
+        image = Image.open("assets/vase_hq.jpg")
+        image = image.resize(condition_size)
+        condition = Condition(image, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    elif condition_type == "fill":
+        condition_img = (
+            Image.open("./assets/vase_hq.jpg").resize(condition_size).convert("RGB")
+        )
+        mask = Image.new("L", condition_img.size, 0)
+        draw = ImageDraw.Draw(mask)
+        a = condition_img.size[0] // 4
+        b = a * 3
+        draw.rectangle([a, a, b, b], fill=255)
+        condition_img = Image.composite(
+            condition_img, Image.new("RGB", condition_img.size, (0, 0, 0)), mask
+        )
+        condition = Condition(condition, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    elif condition_type == "super_resolution":
+        image = Image.open("assets/vase_hq.jpg")
+        image = image.resize(condition_size)
+        condition = Condition(image, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    else:
+        raise NotImplementedError
+    os.makedirs(save_path, exist_ok=True)
+    for i, (condition, prompt) in enumerate(test_list):
+        generator = torch.Generator(device=model.device)
+        generator.manual_seed(42)
+        res = generate(
+            model.flux_pipe,
+            prompt=prompt,
+            conditions=[condition],
+            height=target_size[1],
+            width=target_size[0],
+            generator=generator,
+            model_config=model.model_config,
+            kv_cache=model.model_config.get("independent_condition", False),
+        )
+        file_path = os.path.join(save_path, f"{file_name}_{condition_type}_{i}.jpg")
+        res.images[0].save(file_path)
+def main():
+    # Initialize
+    config = get_config()
+    training_config = config["train"]
+    torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", 0)))
+    # Load dataset text-to-image-2M
+    dataset = load_dataset(
+        "webdataset",
+        data_files={"train": training_config["dataset"]["urls"]},
+        split="train",
+        cache_dir="cache/t2i2m",
+        num_proc=32,
+    )
+    # Initialize custom dataset
+    dataset = ImageConditionDataset(
+        dataset,
+        condition_size=training_config["dataset"]["condition_size"],
+        target_size=training_config["dataset"]["target_size"],
+        condition_type=training_config["condition_type"],
+        drop_text_prob=training_config["dataset"]["drop_text_prob"],
+        drop_image_prob=training_config["dataset"]["drop_image_prob"],
+        position_scale=training_config["dataset"].get("position_scale", 1.0),
+    )
+    # Initialize model
+    trainable_model = OminiModel(
+        flux_pipe_id=config["flux_path"],
+        lora_config=training_config["lora_config"],
+        device=f"cuda",
+        dtype=getattr(torch, config["dtype"]),
+        optimizer_config=training_config["optimizer"],
+        model_config=config.get("model", {}),
+        gradient_checkpointing=training_config.get("gradient_checkpointing", False),
+    )
+    train(dataset, trainable_model, config, test_function)
+if __name__ == "__main__":
+    main()

omini/train_flux/train_spatial_alignment_rotation.py ADDED Viewed

	@@ -0,0 +1,211 @@

+import torch
+from torch.utils.data import Dataset
+import torchvision.transforms as T
+import os
+import random
+import numpy as np
+from PIL import Image, ImageDraw
+from datasets import load_dataset
+from .trainer_rotation import OminiModelRotation, get_config, train
+from ..pipeline.flux_omini import Condition, convert_to_condition, generate
+class ImageConditionDataset(Dataset):
+    def __init__(
+        self,
+        base_dataset,
+        condition_size=(512, 512),
+        target_size=(512, 512),
+        condition_type: str = "canny",
+        drop_text_prob: float = 0.1,
+        drop_image_prob: float = 0.1,
+        return_pil_image: bool = False,
+        position_scale=1.0,
+    ):
+        self.base_dataset = base_dataset
+        self.condition_size = condition_size
+        self.target_size = target_size
+        self.condition_type = condition_type
+        self.drop_text_prob = drop_text_prob
+        self.drop_image_prob = drop_image_prob
+        self.return_pil_image = return_pil_image
+        self.position_scale = position_scale
+        self.to_tensor = T.ToTensor()
+    def __len__(self):
+        return len(self.base_dataset)
+    def __get_condition__(self, image, condition_type):
+        condition_size = self.condition_size
+        position_delta = np.array([0, 0])
+        if condition_type in ["canny", "coloring", "deblurring", "depth"]:
+            image, kwargs = image.resize(condition_size), {}
+            if condition_type == "deblurring":
+                blur_radius = random.randint(1, 10)
+                kwargs["blur_radius"] = blur_radius
+            condition_img = convert_to_condition(condition_type, image, **kwargs)
+        elif condition_type == "depth_pred":
+            depth_img = convert_to_condition("depth", image)
+            condition_img = image.resize(condition_size)
+            image = depth_img.resize(condition_size)
+        elif condition_type == "fill":
+            condition_img = image.resize(condition_size).convert("RGB")
+            w, h = image.size
+            x1, x2 = sorted([random.randint(0, w), random.randint(0, w)])
+            y1, y2 = sorted([random.randint(0, h), random.randint(0, h)])
+            mask = Image.new("L", image.size, 0)
+            draw = ImageDraw.Draw(mask)
+            draw.rectangle([x1, y1, x2, y2], fill=255)
+            if random.random() > 0.5:
+                mask = Image.eval(mask, lambda a: 255 - a)
+            condition_img = Image.composite(
+                image, Image.new("RGB", image.size, (0, 0, 0)), mask
+            )
+        elif condition_type == "sr":
+            condition_img = image.resize(condition_size)
+            position_delta = np.array([0, -condition_size[0] // 16])
+        else:
+            raise ValueError(f"Condition type {condition_type} is not  implemented.")
+        return condition_img, position_delta
+    def __getitem__(self, idx):
+        image = self.base_dataset[idx]["jpg"]
+        image = image.resize(self.target_size).convert("RGB")
+        description = self.base_dataset[idx]["json"]["prompt"]
+        condition_size = self.condition_size
+        position_scale = self.position_scale
+        condition_img, position_delta = self.__get_condition__(
+            image, self.condition_type
+        )
+        # Randomly drop text or image (for training)
+        drop_text = random.random() < self.drop_text_prob
+        drop_image = random.random() < self.drop_image_prob
+        if drop_text:
+            description = ""
+        if drop_image:
+            condition_img = Image.new("RGB", condition_size, (0, 0, 0))
+        return {
+            "image": self.to_tensor(image),
+            "condition_0": self.to_tensor(condition_img),
+            "condition_type_0": self.condition_type,
+            "position_delta_0": position_delta,
+            "description": description,
+            **({"pil_image": [image, condition_img]} if self.return_pil_image else {}),
+            **({"position_scale_0": position_scale} if position_scale != 1.0 else {}),
+        }
+@torch.no_grad()
+def test_function(model, save_path, file_name):
+    condition_size = model.training_config["dataset"]["condition_size"]
+    target_size = model.training_config["dataset"]["target_size"]
+    position_delta = model.training_config["dataset"].get("position_delta", [0, 0])
+    position_scale = model.training_config["dataset"].get("position_scale", 1.0)
+    adapter = model.adapter_names[2]
+    condition_type = model.training_config["condition_type"]
+    test_list = []
+    if condition_type in ["canny", "coloring", "deblurring", "depth"]:
+        image = Image.open("assets/vase_hq.jpg")
+        image = image.resize(condition_size)
+        condition_img = convert_to_condition(condition_type, image, 5)
+        condition = Condition(condition_img, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    elif condition_type == "depth_pred":
+        image = Image.open("assets/vase_hq.jpg")
+        image = image.resize(condition_size)
+        condition = Condition(image, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    elif condition_type == "fill":
+        condition_img = (
+            Image.open("./assets/vase_hq.jpg").resize(condition_size).convert("RGB")
+        )
+        mask = Image.new("L", condition_img.size, 0)
+        draw = ImageDraw.Draw(mask)
+        a = condition_img.size[0] // 4
+        b = a * 3
+        draw.rectangle([a, a, b, b], fill=255)
+        condition_img = Image.composite(
+            condition_img, Image.new("RGB", condition_img.size, (0, 0, 0)), mask
+        )
+        condition = Condition(condition, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    elif condition_type == "super_resolution":
+        image = Image.open("assets/vase_hq.jpg")
+        image = image.resize(condition_size)
+        condition = Condition(image, adapter, position_delta, position_scale)
+        test_list.append((condition, "A beautiful vase on a table."))
+    else:
+        raise NotImplementedError
+    os.makedirs(save_path, exist_ok=True)
+    for i, (condition, prompt) in enumerate(test_list):
+        generator = torch.Generator(device=model.device)
+        generator.manual_seed(42)
+        res = generate(
+            model.flux_pipe,
+            prompt=prompt,
+            conditions=[condition],
+            height=target_size[1],
+            width=target_size[0],
+            generator=generator,
+            model_config=model.model_config,
+            kv_cache=model.model_config.get("independent_condition", False),
+        )
+        file_path = os.path.join(save_path, f"{file_name}_{condition_type}_{i}.jpg")
+        res.images[0].save(file_path)
+def main():
+    # Initialize
+    config = get_config()
+    training_config = config["train"]
+    torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", 0)))
+    # Load dataset text-to-image-2M
+    dataset = load_dataset(
+        "webdataset",
+        data_files={"train": training_config["dataset"]["urls"]},
+        split="train",
+        cache_dir="cache/t2i2m",
+        num_proc=32,
+    )
+    # Initialize custom dataset
+    dataset = ImageConditionDataset(
+        dataset,
+        condition_size=training_config["dataset"]["condition_size"],
+        target_size=training_config["dataset"]["target_size"],
+        condition_type=training_config["condition_type"],
+        drop_text_prob=training_config["dataset"]["drop_text_prob"],
+        drop_image_prob=training_config["dataset"]["drop_image_prob"],
+        position_scale=training_config["dataset"].get("position_scale", 1.0),
+    )
+    # Initialize model
+    trainable_model = OminiModelRotation(
+        flux_pipe_id=config["flux_path"],
+        rotation_adapter_config=training_config["rotation_adapter_config"],
+        device=f"cuda",
+        dtype=getattr(torch, config["dtype"]),
+        optimizer_config=training_config["optimizer"],
+        model_config=config.get("model", {}),
+        gradient_checkpointing=training_config.get("gradient_checkpointing", False),
+    )
+    train(dataset, trainable_model, config, test_function)
+if __name__ == "__main__":
+    main()

omini/train_flux/train_subject.py ADDED Viewed

	@@ -0,0 +1,205 @@

+import torch
+from torch.utils.data import Dataset
+import torchvision.transforms as T
+import os
+import random
+import numpy as np
+from PIL import Image
+from datasets import load_dataset
+from .trainer import OminiModel, get_config, train
+from ..pipeline.flux_omini import Condition, generate
+class Subject200KDataset(Dataset):
+    def __init__(
+        self,
+        base_dataset,
+        condition_size=(512, 512),
+        target_size=(512, 512),
+        image_size: int = 512,
+        padding: int = 0,
+        condition_type: str = "subject",
+        drop_text_prob: float = 0.1,
+        drop_image_prob: float = 0.1,
+        return_pil_image: bool = False,
+    ):
+        self.base_dataset = base_dataset
+        self.condition_size = condition_size
+        self.target_size = target_size
+        self.image_size = image_size
+        self.padding = padding
+        self.condition_type = condition_type
+        self.drop_text_prob = drop_text_prob
+        self.drop_image_prob = drop_image_prob
+        self.return_pil_image = return_pil_image
+        self.to_tensor = T.ToTensor()
+    def __len__(self):
+        return len(self.base_dataset) * 2
+    def __getitem__(self, idx):
+        # If target is 0, left image is target, right image is condition
+        target = idx % 2
+        item = self.base_dataset[idx // 2]
+        # Crop the image to target and condition
+        image = item["image"]
+        left_img = image.crop(
+            (
+                self.padding,
+                self.padding,
+                self.image_size + self.padding,
+                self.image_size + self.padding,
+            )
+        )
+        right_img = image.crop(
+            (
+                self.image_size + self.padding * 2,
+                self.padding,
+                self.image_size * 2 + self.padding * 2,
+                self.image_size + self.padding,
+            )
+        )
+        # Get the target and condition image
+        target_image, condition_img = (
+            (left_img, right_img) if target == 0 else (right_img, left_img)
+        )
+        # Resize the image
+        condition_img = condition_img.resize(self.condition_size).convert("RGB")
+        target_image = target_image.resize(self.target_size).convert("RGB")
+        # Get the description
+        description = item["description"][
+            "description_0" if target == 0 else "description_1"
+        ]
+        # Randomly drop text or image
+        drop_text = random.random() < self.drop_text_prob
+        drop_image = random.random() < self.drop_image_prob
+        if drop_text:
+            description = ""
+        if drop_image:
+            condition_img = Image.new("RGB", self.condition_size, (0, 0, 0))
+        # 16 is the downscale factor of the image.
+        # More details about position delta can be found in the documentation.
+        position_delta = np.array([0, -self.condition_size[0] // 16])
+        return {
+            "image": self.to_tensor(target_image),
+            "condition_0": self.to_tensor(condition_img),
+            "condition_type_0": self.condition_type,
+            "position_delta_0": position_delta,
+            "description": description,
+            **({"pil_image": image} if self.return_pil_image else {}),
+        }
+@torch.no_grad()
+def test_function(model, save_path, file_name):
+    condition_size = model.training_config["dataset"]["condition_size"]
+    target_size = model.training_config["dataset"]["target_size"]
+    # More details about position delta can be found in the documentation.
+    position_delta = [0, -condition_size[0] // 16]
+    # Set adapters
+    adapter = model.adapter_names[2]
+    condition_type = model.training_config["condition_type"]
+    test_list = []
+    # Test case1 (in-distribution test case)
+    image = Image.open("assets/test_in.jpg")
+    image = image.resize(condition_size)
+    prompt = "Resting on the picnic table at a lakeside campsite, it's caught in the golden glow of early morning, with mist rising from the water and tall pines casting long shadows behind the scene."
+    condition = Condition(image, adapter, position_delta)
+    test_list.append((condition, prompt))
+    # Test case2 (out-of-distribution test case)
+    image = Image.open("assets/test_out.jpg")
+    image = image.resize(condition_size)
+    prompt = "In a bright room. It is placed on a table."
+    condition = Condition(image, adapter, position_delta)
+    test_list.append((condition, prompt))
+    # Generate images
+    os.makedirs(save_path, exist_ok=True)
+    for i, (condition, prompt) in enumerate(test_list):
+        generator = torch.Generator(device=model.device)
+        generator.manual_seed(42)
+        res = generate(
+            model.flux_pipe,
+            prompt=prompt,
+            conditions=[condition],
+            height=target_size[1],
+            width=target_size[0],
+            generator=generator,
+            model_config=model.model_config,
+            kv_cache=model.model_config.get("independent_condition", False),
+        )
+        file_path = os.path.join(save_path, f"{file_name}_{condition_type}_{i}.jpg")
+        res.images[0].save(file_path)
+def main():
+    # Initialize
+    config = get_config()
+    training_config = config["train"]
+    torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", 0)))
+    # Initialize raw dataset
+    raw_dataset = load_dataset("Yuanshi/Subjects200K")
+    # Define filter function to filter out low-quality images from Subjects200K
+    def filter_func(item):
+        if not item.get("quality_assessment"):
+            return False
+        return all(
+            item["quality_assessment"].get(key, 0) >= 5
+            for key in ["compositeStructure", "objectConsistency", "imageQuality"]
+        )
+    # Filter dataset
+    if not os.path.exists("./cache/dataset"):
+        os.makedirs("./cache/dataset")
+    data_valid = raw_dataset["train"].filter(
+        filter_func,
+        num_proc=16,
+        cache_file_name="./cache/dataset/data_valid.arrow",
+    )
+    # Initialize the dataset
+    dataset = Subject200KDataset(
+        data_valid,
+        condition_size=training_config["dataset"]["condition_size"],
+        target_size=training_config["dataset"]["target_size"],
+        image_size=training_config["dataset"]["image_size"],
+        padding=training_config["dataset"]["padding"],
+        condition_type=training_config["condition_type"],
+        drop_text_prob=training_config["dataset"]["drop_text_prob"],
+        drop_image_prob=training_config["dataset"]["drop_image_prob"],
+    )
+    # Initialize model
+    trainable_model = OminiModel(
+        flux_pipe_id=config["flux_path"],
+        lora_config=training_config["lora_config"],
+        device=f"cuda",
+        dtype=getattr(torch, config["dtype"]),
+        optimizer_config=training_config["optimizer"],
+        model_config=config.get("model", {}),
+        gradient_checkpointing=training_config.get("gradient_checkpointing", False),
+    )
+    train(dataset, trainable_model, config, test_function)
+if __name__ == "__main__":
+    main()

omini/train_flux/train_subject_rotation.py ADDED Viewed

	@@ -0,0 +1,205 @@

+import torch
+from torch.utils.data import Dataset
+import torchvision.transforms as T
+import os
+import random
+import numpy as np
+from PIL import Image
+from datasets import load_dataset
+from .trainer_rotation import OminiModelRotation, get_config, train
+from ..pipeline.flux_omini import Condition, generate
+class Subject200KDataset(Dataset):
+    def __init__(
+        self,
+        base_dataset,
+        condition_size=(512, 512),
+        target_size=(512, 512),
+        image_size: int = 512,
+        padding: int = 0,
+        condition_type: str = "subject",
+        drop_text_prob: float = 0.1,
+        drop_image_prob: float = 0.1,
+        return_pil_image: bool = False,
+    ):
+        self.base_dataset = base_dataset
+        self.condition_size = condition_size
+        self.target_size = target_size
+        self.image_size = image_size
+        self.padding = padding
+        self.condition_type = condition_type
+        self.drop_text_prob = drop_text_prob
+        self.drop_image_prob = drop_image_prob
+        self.return_pil_image = return_pil_image
+        self.to_tensor = T.ToTensor()
+    def __len__(self):
+        return len(self.base_dataset) * 2
+    def __getitem__(self, idx):
+        # If target is 0, left image is target, right image is condition
+        target = idx % 2
+        item = self.base_dataset[idx // 2]
+        # Crop the image to target and condition
+        image = item["image"]
+        left_img = image.crop(
+            (
+                self.padding,
+                self.padding,
+                self.image_size + self.padding,
+                self.image_size + self.padding,
+            )
+        )
+        right_img = image.crop(
+            (
+                self.image_size + self.padding * 2,
+                self.padding,
+                self.image_size * 2 + self.padding * 2,
+                self.image_size + self.padding,
+            )
+        )
+        # Get the target and condition image
+        target_image, condition_img = (
+            (left_img, right_img) if target == 0 else (right_img, left_img)
+        )
+        # Resize the image
+        condition_img = condition_img.resize(self.condition_size).convert("RGB")
+        target_image = target_image.resize(self.target_size).convert("RGB")
+        # Get the description
+        description = item["description"][
+            "description_0" if target == 0 else "description_1"
+        ]
+        # Randomly drop text or image
+        drop_text = random.random() < self.drop_text_prob
+        drop_image = random.random() < self.drop_image_prob
+        if drop_text:
+            description = ""
+        if drop_image:
+            condition_img = Image.new("RGB", self.condition_size, (0, 0, 0))
+        # 16 is the downscale factor of the image.
+        # More details about position delta can be found in the documentation.
+        position_delta = np.array([0, -self.condition_size[0] // 16])
+        return {
+            "image": self.to_tensor(target_image),
+            "condition_0": self.to_tensor(condition_img),
+            "condition_type_0": self.condition_type,
+            "position_delta_0": position_delta,
+            "description": description,
+            **({"pil_image": image} if self.return_pil_image else {}),
+        }
+@torch.no_grad()
+def test_function(model, save_path, file_name):
+    condition_size = model.training_config["dataset"]["condition_size"]
+    target_size = model.training_config["dataset"]["target_size"]
+    # More details about position delta can be found in the documentation.
+    position_delta = [0, -condition_size[0] // 16]
+    # Set adapters
+    adapter = model.adapter_names[2]
+    condition_type = model.training_config["condition_type"]
+    test_list = []
+    # Test case1 (in-distribution test case)
+    image = Image.open("assets/test_in.jpg")
+    image = image.resize(condition_size)
+    prompt = "Resting on the picnic table at a lakeside campsite, it's caught in the golden glow of early morning, with mist rising from the water and tall pines casting long shadows behind the scene."
+    condition = Condition(image, adapter, position_delta)
+    test_list.append((condition, prompt))
+    # Test case2 (out-of-distribution test case)
+    image = Image.open("assets/test_out.jpg")
+    image = image.resize(condition_size)
+    prompt = "In a bright room. It is placed on a table."
+    condition = Condition(image, adapter, position_delta)
+    test_list.append((condition, prompt))
+    # Generate images
+    os.makedirs(save_path, exist_ok=True)
+    for i, (condition, prompt) in enumerate(test_list):
+        generator = torch.Generator(device=model.device)
+        generator.manual_seed(42)
+        res = generate(
+            model.flux_pipe,
+            prompt=prompt,
+            conditions=[condition],
+            height=target_size[1],
+            width=target_size[0],
+            generator=generator,
+            model_config=model.model_config,
+            kv_cache=model.model_config.get("independent_condition", False),
+        )
+        file_path = os.path.join(save_path, f"{file_name}_{condition_type}_{i}.jpg")
+        res.images[0].save(file_path)
+def main():
+    # Initialize
+    config = get_config()
+    training_config = config["train"]
+    torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", 0)))
+    # Initialize raw dataset
+    raw_dataset = load_dataset("Yuanshi/Subjects200K")
+    # Define filter function to filter out low-quality images from Subjects200K
+    def filter_func(item):
+        if not item.get("quality_assessment"):
+            return False
+        return all(
+            item["quality_assessment"].get(key, 0) >= 5
+            for key in ["compositeStructure", "objectConsistency", "imageQuality"]
+        )
+    # Filter dataset
+    if not os.path.exists("./cache/dataset"):
+        os.makedirs("./cache/dataset")
+    data_valid = raw_dataset["train"].filter(
+        filter_func,
+        num_proc=16,
+        cache_file_name="./cache/dataset/data_valid.arrow",
+    )
+    # Initialize the dataset
+    dataset = Subject200KDataset(
+        data_valid,
+        condition_size=training_config["dataset"]["condition_size"],
+        target_size=training_config["dataset"]["target_size"],
+        image_size=training_config["dataset"]["image_size"],
+        padding=training_config["dataset"]["padding"],
+        condition_type=training_config["condition_type"],
+        drop_text_prob=training_config["dataset"]["drop_text_prob"],
+        drop_image_prob=training_config["dataset"]["drop_image_prob"],
+    )
+    # Initialize model
+    trainable_model = OminiModelRotation(
+        flux_pipe_id=config["flux_path"],
+        rotation_adapter_config=training_config["rotation_adapter_config"],
+        device=f"cuda",
+        dtype=getattr(torch, config["dtype"]),
+        optimizer_config=training_config["optimizer"],
+        model_config=config.get("model", {}),
+        gradient_checkpointing=training_config.get("gradient_checkpointing", False),
+    )
+    train(dataset, trainable_model, config, test_function)
+if __name__ == "__main__":
+    main()

omini/train_flux/train_token_integration.py ADDED Viewed

	@@ -0,0 +1,136 @@

+import torch
+import os
+import random
+from PIL import Image, ImageDraw
+from datasets import load_dataset
+from .trainer import OminiModel, get_config, train
+from ..pipeline.flux_omini import Condition, generate
+from .train_spatial_alignment import ImageConditionDataset
+class TokenIntergrationDataset(ImageConditionDataset):
+    def __getitem__(self, idx):
+        image = self.base_dataset[idx]["jpg"]
+        image = image.resize(self.target_size).convert("RGB")
+        description = self.base_dataset[idx]["json"]["prompt"]
+        assert self.condition_type == "token_intergration"
+        assert (
+            image.size[0] % 16 == 0 and image.size[1] % 16 == 0
+        ), "Condition size must be divisible by 16"
+        # Randomly drop text or image (for training)
+        description = "" if random.random() < self.drop_text_prob else description
+        # Generate a latent mask
+        w, h = image.size[0] // 16, image.size[1] // 16
+        while True:
+            x1, x2 = sorted([random.randint(0, w), random.randint(0, w)])
+            y1, y2 = sorted([random.randint(0, h), random.randint(0, h)])
+            is_zero = x1 == x2 or y1 == y2
+            is_full = x1 == 0 and y1 == 0 and x2 == w and y2 == h
+            if not (is_zero or is_full):
+                break
+        mask = Image.new("L", (w, h), 0)
+        draw = ImageDraw.Draw(mask)
+        draw.rectangle([x1, y1, x2, y2], fill=255)
+        if random.random() > 0.5:
+            mask = Image.eval(mask, lambda a: 255 - a)
+        mask = self.to_tensor(mask).to(bool).reshape(-1)
+        return {
+            "image": self.to_tensor(image),
+            "image_latent_mask": torch.logical_not(mask),
+            "condition_0": self.to_tensor(image),
+            "condition_type_0": self.condition_type,
+            "condition_latent_mask_0": mask,
+            "description": description,
+        }
+@torch.no_grad()
+def test_function(model, save_path, file_name):
+    target_size = model.training_config["dataset"]["target_size"]
+    condition_type = model.training_config["condition_type"]
+    test_list = []
+    # Generate two masks to test inpainting and outpainting.
+    mask1 = torch.ones((32, 32), dtype=bool)
+    mask1[8:24, 8:24] = False
+    mask2 = torch.logical_not(mask1)
+    image = Image.open("assets/vase_hq.jpg").resize(target_size)
+    condition1 = Condition(
+        image, model.adapter_names[2], latent_mask=mask1, is_complement=True
+    )
+    condition2 = Condition(
+        image, model.adapter_names[2], latent_mask=mask2, is_complement=True
+    )
+    test_list.append((condition1, "A beautiful vase on a table.", mask2))
+    test_list.append((condition2, "A beautiful vase on a table.", mask1))
+    os.makedirs(save_path, exist_ok=True)
+    for i, (condition, prompt, latent_mask) in enumerate(test_list):
+        generator = torch.Generator(device=model.device)
+        generator.manual_seed(42)
+        res = generate(
+            model.flux_pipe,
+            prompt=prompt,
+            conditions=[condition],
+            height=target_size[0],
+            width=target_size[1],
+            generator=generator,
+            model_config=model.model_config,
+            kv_cache=model.model_config.get("independent_condition", False),
+            latent_mask=latent_mask,
+        )
+        file_path = os.path.join(save_path, f"{file_name}_{condition_type}_{i}.jpg")
+        res.images[0].save(file_path)
+def main():
+    # Initialize
+    config = get_config()
+    training_config = config["train"]
+    torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", 0)))
+    # Load dataset text-to-image-2M
+    dataset = load_dataset(
+        "webdataset",
+        data_files={"train": training_config["dataset"]["urls"]},
+        split="train",
+        cache_dir="cache/t2i2m",
+        num_proc=32,
+    )
+    dataset = TokenIntergrationDataset(
+        dataset,
+        condition_size=training_config["dataset"]["condition_size"],
+        target_size=training_config["dataset"]["target_size"],
+        condition_type=training_config["condition_type"],
+        drop_text_prob=training_config["dataset"]["drop_text_prob"],
+        drop_image_prob=training_config["dataset"]["drop_image_prob"],
+        position_scale=training_config["dataset"].get("position_scale", 1.0),
+    )
+    # Initialize model
+    trainable_model = OminiModel(
+        flux_pipe_id=config["flux_path"],
+        lora_config=training_config["lora_config"],
+        device=f"cuda",
+        dtype=getattr(torch, config["dtype"]),
+        optimizer_config=training_config["optimizer"],
+        model_config=config.get("model", {}),
+        gradient_checkpointing=training_config.get("gradient_checkpointing", False),
+        adapter_names=[None, None, "default"],
+    )
+    train(dataset, trainable_model, config, test_function)
+if __name__ == "__main__":
+    main()

omini/train_flux/trainer.py ADDED Viewed

	@@ -0,0 +1,384 @@

+import lightning as L
+from diffusers.pipelines import FluxPipeline
+import torch
+import wandb
+import os
+import yaml
+from peft import LoraConfig, get_peft_model_state_dict
+from torch.utils.data import DataLoader
+import time
+from typing import List
+import prodigyopt
+from ..pipeline.flux_omini import transformer_forward, encode_images
+def get_rank():
+    try:
+        rank = int(os.environ.get("LOCAL_RANK"))
+    except:
+        rank = 0
+    return rank
+def get_config():
+    config_path = os.environ.get("OMINI_CONFIG")
+    assert config_path is not None, "Please set the OMINI_CONFIG environment variable"
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+    return config
+def init_wandb(wandb_config, run_name):
+    import wandb
+    try:
+        assert os.environ.get("WANDB_API_KEY") is not None
+        wandb.init(
+            project=wandb_config["project"],
+            name=run_name,
+            config={},
+        )
+    except Exception as e:
+        print("Failed to initialize WanDB:", e)
+class OminiModel(L.LightningModule):
+    def __init__(
+        self,
+        flux_pipe_id: str,
+        lora_path: str = None,
+        lora_config: dict = None,
+        device: str = "cuda",
+        dtype: torch.dtype = torch.bfloat16,
+        model_config: dict = {},
+        adapter_names: List[str] = [None, None, "default"],
+        optimizer_config: dict = None,
+        gradient_checkpointing: bool = False,
+    ):
+        # Initialize the LightningModule
+        super().__init__()
+        self.model_config = model_config
+        self.optimizer_config = optimizer_config
+        # Load the Flux pipeline
+        self.flux_pipe: FluxPipeline = FluxPipeline.from_pretrained(
+            flux_pipe_id, torch_dtype=dtype
+        ).to(device)
+        self.transformer = self.flux_pipe.transformer
+        self.transformer.gradient_checkpointing = gradient_checkpointing
+        self.transformer.train()
+        # Freeze the Flux pipeline
+        self.flux_pipe.text_encoder.requires_grad_(False).eval()
+        self.flux_pipe.text_encoder_2.requires_grad_(False).eval()
+        self.flux_pipe.vae.requires_grad_(False).eval()
+        self.adapter_names = adapter_names
+        self.adapter_set = set([each for each in adapter_names if each is not None])
+        # Initialize LoRA layers
+        self.lora_layers = self.init_lora(lora_path, lora_config)
+        self.to(device).to(dtype)
+    def init_lora(self, lora_path: str, lora_config: dict):
+        assert lora_path or lora_config
+        if lora_path:
+            # TODO: Implement this
+            raise NotImplementedError
+        else:
+            for adapter_name in self.adapter_set:
+                self.transformer.add_adapter(
+                    LoraConfig(**lora_config), adapter_name=adapter_name
+                )
+            # TODO: Check if this is correct (p.requires_grad)
+            lora_layers = filter(
+                lambda p: p.requires_grad, self.transformer.parameters()
+            )
+        return list(lora_layers)
+    def save_lora(self, path: str):
+        for adapter_name in self.adapter_set:
+            FluxPipeline.save_lora_weights(
+                save_directory=path,
+                weight_name=f"{adapter_name}.safetensors",
+                transformer_lora_layers=get_peft_model_state_dict(
+                    self.transformer, adapter_name=adapter_name
+                ),
+                safe_serialization=True,
+            )
+    def configure_optimizers(self):
+        # Freeze the transformer
+        self.transformer.requires_grad_(False)
+        opt_config = self.optimizer_config
+        # Set the trainable parameters
+        self.trainable_params = self.lora_layers
+        # Unfreeze trainable parameters
+        for p in self.trainable_params:
+            p.requires_grad_(True)
+        # Initialize the optimizer
+        if opt_config["type"] == "AdamW":
+            optimizer = torch.optim.AdamW(self.trainable_params, **opt_config["params"])
+        elif opt_config["type"] == "Prodigy":
+            optimizer = prodigyopt.Prodigy(
+                self.trainable_params,
+                **opt_config["params"],
+            )
+        elif opt_config["type"] == "SGD":
+            optimizer = torch.optim.SGD(self.trainable_params, **opt_config["params"])
+        else:
+            raise NotImplementedError("Optimizer not implemented.")
+        return optimizer
+    def training_step(self, batch, batch_idx):
+        imgs, prompts = batch["image"], batch["description"]
+        image_latent_mask = batch.get("image_latent_mask", None)
+        # Get the conditions and position deltas from the batch
+        conditions, position_deltas, position_scales, latent_masks = [], [], [], []
+        for i in range(1000):
+            if f"condition_{i}" not in batch:
+                break
+            conditions.append(batch[f"condition_{i}"])
+            position_deltas.append(batch.get(f"position_delta_{i}", [[0, 0]]))
+            position_scales.append(batch.get(f"position_scale_{i}", [1.0])[0])
+            latent_masks.append(batch.get(f"condition_latent_mask_{i}", None))
+        # Prepare inputs
+        with torch.no_grad():
+            # Prepare image input
+            x_0, img_ids = encode_images(self.flux_pipe, imgs)
+            # Prepare text input
+            (
+                prompt_embeds,
+                pooled_prompt_embeds,
+                text_ids,
+            ) = self.flux_pipe.encode_prompt(
+                prompt=prompts,
+                prompt_2=None,
+                prompt_embeds=None,
+                pooled_prompt_embeds=None,
+                device=self.flux_pipe.device,
+                num_images_per_prompt=1,
+                max_sequence_length=self.model_config.get("max_sequence_length", 512),
+                lora_scale=None,
+            )
+            # Prepare t and x_t
+            t = torch.sigmoid(torch.randn((imgs.shape[0],), device=self.device))
+            x_1 = torch.randn_like(x_0).to(self.device)
+            t_ = t.unsqueeze(1).unsqueeze(1)
+            x_t = ((1 - t_) * x_0 + t_ * x_1).to(self.dtype)
+            if image_latent_mask is not None:
+                x_0 = x_0[:, image_latent_mask[0]]
+                x_1 = x_1[:, image_latent_mask[0]]
+                x_t = x_t[:, image_latent_mask[0]]
+                img_ids = img_ids[image_latent_mask[0]]
+            # Prepare conditions
+            condition_latents, condition_ids = [], []
+            for cond, p_delta, p_scale, latent_mask in zip(
+                conditions, position_deltas, position_scales, latent_masks
+            ):
+                # Prepare conditions
+                c_latents, c_ids = encode_images(self.flux_pipe, cond)
+                # Scale the position (see OminiConrtol2)
+                if p_scale != 1.0:
+                    scale_bias = (p_scale - 1.0) / 2
+                    c_ids[:, 1:] *= p_scale
+                    c_ids[:, 1:] += scale_bias
+                # Add position delta (see OminiControl)
+                c_ids[:, 1] += p_delta[0][0]
+                c_ids[:, 2] += p_delta[0][1]
+                if len(p_delta) > 1:
+                    print("Warning: only the first position delta is used.")
+                # Append to the list
+                if latent_mask is not None:
+                    c_latents, c_ids = c_latents[latent_mask], c_ids[latent_mask[0]]
+                condition_latents.append(c_latents)
+                condition_ids.append(c_ids)
+            # Prepare guidance
+            guidance = (
+                torch.ones_like(t).to(self.device)
+                if self.transformer.config.guidance_embeds
+                else None
+            )
+        branch_n = 2 + len(conditions)
+        group_mask = torch.ones([branch_n, branch_n], dtype=torch.bool).to(self.device)
+        # Disable the attention cross different condition branches
+        group_mask[2:, 2:] = torch.diag(torch.tensor([1] * len(conditions)))
+        # Disable the attention from condition branches to image branch and text branch
+        if self.model_config.get("independent_condition", False):
+            group_mask[2:, :2] = False
+        # Forward pass
+        transformer_out = transformer_forward(
+            self.transformer,
+            image_features=[x_t, *(condition_latents)],
+            text_features=[prompt_embeds],
+            img_ids=[img_ids, *(condition_ids)],
+            txt_ids=[text_ids],
+            # There are three timesteps for the three branches
+            # (text, image, and the condition)
+            timesteps=[t, t] + [torch.zeros_like(t)] * len(conditions),
+            # Same as above
+            pooled_projections=[pooled_prompt_embeds] * branch_n,
+            guidances=[guidance] * branch_n,
+            # The LoRA adapter names of each branch
+            adapters=self.adapter_names,
+            return_dict=False,
+            group_mask=group_mask,
+        )
+        pred = transformer_out[0]
+        # Compute loss
+        step_loss = torch.nn.functional.mse_loss(pred, (x_1 - x_0), reduction="mean")
+        self.last_t = t.mean().item()
+        self.log_loss = (
+            step_loss.item()
+            if not hasattr(self, "log_loss")
+            else self.log_loss * 0.95 + step_loss.item() * 0.05
+        )
+        return step_loss
+    def generate_a_sample(self):
+        raise NotImplementedError("Generate a sample not implemented.")
+class TrainingCallback(L.Callback):
+    def __init__(self, run_name, training_config: dict = {}, test_function=None):
+        self.run_name, self.training_config = run_name, training_config
+        self.print_every_n_steps = training_config.get("print_every_n_steps", 10)
+        self.save_interval = training_config.get("save_interval", 1000)
+        self.sample_interval = training_config.get("sample_interval", 1000)
+        self.save_path = training_config.get("save_path", "./output")
+        self.wandb_config = training_config.get("wandb", None)
+        self.use_wandb = (
+            wandb is not None and os.environ.get("WANDB_API_KEY") is not None
+        )
+        self.total_steps = 0
+        self.test_function = test_function
+    def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+        gradient_size = 0
+        max_gradient_size = 0
+        count = 0
+        for _, param in pl_module.named_parameters():
+            if param.grad is not None:
+                gradient_size += param.grad.norm(2).item()
+                max_gradient_size = max(max_gradient_size, param.grad.norm(2).item())
+                count += 1
+        if count > 0:
+            gradient_size /= count
+        self.total_steps += 1
+        # Print training progress every n steps
+        if self.use_wandb:
+            report_dict = {
+                "steps": batch_idx,
+                "steps": self.total_steps,
+                "epoch": trainer.current_epoch,
+                "gradient_size": gradient_size,
+            }
+            loss_value = outputs["loss"].item() * trainer.accumulate_grad_batches
+            report_dict["loss"] = loss_value
+            report_dict["t"] = pl_module.last_t
+            wandb.log(report_dict)
+        if self.total_steps % self.print_every_n_steps == 0:
+            print(
+                f"Epoch: {trainer.current_epoch}, Steps: {self.total_steps}, Batch: {batch_idx}, Loss: {pl_module.log_loss:.4f}, Gradient size: {gradient_size:.4f}, Max gradient size: {max_gradient_size:.4f}"
+            )
+        # Save LoRA weights at specified intervals
+        if self.total_steps % self.save_interval == 0:
+            print(
+                f"Epoch: {trainer.current_epoch}, Steps: {self.total_steps} - Saving LoRA weights"
+            )
+            pl_module.save_lora(
+                f"{self.save_path}/{self.run_name}/ckpt/{self.total_steps}"
+            )
+        # Generate and save a sample image at specified intervals
+        if self.total_steps % self.sample_interval == 0 and self.test_function:
+            print(
+                f"Epoch: {trainer.current_epoch}, Steps: {self.total_steps} - Generating a sample"
+            )
+            pl_module.eval()
+            self.test_function(
+                pl_module,
+                f"{self.save_path}/{self.run_name}/output",
+                f"lora_{self.total_steps}",
+            )
+            pl_module.train()
+def train(dataset, trainable_model, config, test_function):
+    # Initialize
+    is_main_process, rank = get_rank() == 0, get_rank()
+    torch.cuda.set_device(rank)
+    # config = get_config()
+    training_config = config["train"]
+    run_name = time.strftime("%Y%m%d-%H%M%S")
+    # Initialize WanDB
+    wandb_config = training_config.get("wandb", None)
+    if wandb_config is not None and is_main_process:
+        init_wandb(wandb_config, run_name)
+    print("Rank:", rank)
+    if is_main_process:
+        print("Config:", config)
+    # Initialize dataloader
+    print("Dataset length:", len(dataset))
+    train_loader = DataLoader(
+        dataset,
+        batch_size=training_config.get("batch_size", 1),
+        shuffle=True,
+        num_workers=training_config["dataloader_workers"],
+    )
+    # Callbacks for testing and saving checkpoints
+    if is_main_process:
+        callbacks = [TrainingCallback(run_name, training_config, test_function)]
+    # Initialize trainer
+    trainer = L.Trainer(
+        accumulate_grad_batches=training_config["accumulate_grad_batches"],
+        callbacks=callbacks if is_main_process else [],
+        enable_checkpointing=False,
+        enable_progress_bar=False,
+        logger=False,
+        max_steps=training_config.get("max_steps", -1),
+        max_epochs=training_config.get("max_epochs", -1),
+        gradient_clip_val=training_config.get("gradient_clip_val", 0.5),
+    )
+    setattr(trainer, "training_config", training_config)
+    setattr(trainable_model, "training_config", training_config)
+    # Save the training config
+    save_path = training_config.get("save_path", "./output")
+    if is_main_process:
+        os.makedirs(f"{save_path}/{run_name}")
+        with open(f"{save_path}/{run_name}/config.yaml", "w") as f:
+            yaml.dump(config, f)
+    # Start training
+    trainer.fit(trainable_model, train_loader)

omini/train_flux/trainer_rotation.py ADDED Viewed

	@@ -0,0 +1,449 @@

+import lightning as L
+from diffusers.pipelines import FluxPipeline
+import torch
+import wandb
+import os
+import yaml
+from peft import LoraConfig, get_peft_model_state_dict
+from torch.utils.data import DataLoader
+import time
+from typing import List
+import prodigyopt
+from ..pipeline.flux_omini import transformer_forward, encode_images
+from ..rotation import RotationTuner, RotationConfig
+def get_rank():
+    try:
+        rank = int(os.environ.get("LOCAL_RANK"))
+    except:
+        rank = 0
+    return rank
+def get_config():
+    config_path = os.environ.get("OMINI_CONFIG")
+    assert config_path is not None, "Please set the OMINI_CONFIG environment variable"
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+    return config
+def init_wandb(wandb_config, run_name):
+    import wandb
+    try:
+        assert os.environ.get("WANDB_API_KEY") is not None
+        wandb.init(
+            project=wandb_config["project"],
+            name=run_name,
+            config={},
+        )
+    except Exception as e:
+        print("Failed to initialize WanDB:", e)
+class OminiModelRotation(L.LightningModule):
+    def __init__(
+        self,
+        flux_pipe_id: str,
+        rotation_adapter_path: str = None,
+        rotation_adapter_config: dict = None,
+        device: str = "cuda",
+        dtype: torch.dtype = torch.bfloat16,
+        model_config: dict = {},
+        adapter_names: List[str] = [None, None, "default"],
+        optimizer_config: dict = None,
+        gradient_checkpointing: bool = False,
+    ):
+        # Initialize the LightningModule
+        super().__init__()
+        self.model_config = model_config
+        self.optimizer_config = optimizer_config
+        # Load the Flux pipeline
+        self.flux_pipe: FluxPipeline = FluxPipeline.from_pretrained(
+            flux_pipe_id, torch_dtype=dtype
+        ).to(device)
+        self.transformer = self.flux_pipe.transformer
+        self.transformer.gradient_checkpointing = gradient_checkpointing
+        self.transformer.train()
+        # Freeze the Flux pipeline
+        self.flux_pipe.text_encoder.requires_grad_(False).eval()
+        self.flux_pipe.text_encoder_2.requires_grad_(False).eval()
+        self.flux_pipe.vae.requires_grad_(False).eval()
+        self.adapter_names = adapter_names
+        self.adapter_set = set([each for each in adapter_names if each is not None])
+        self.rotation_layers = self.init_rotation(rotation_adapter_path, rotation_adapter_config)
+        print(f"Total trainable parameters: {sum(p.numel() for p in self.rotation_layers)}")
+        self.to(device).to(dtype)
+    def init_rotation(self, rotation_adapter_path: str, rotation_adapter_config: dict):
+        assert rotation_adapter_path or rotation_adapter_config
+        if rotation_adapter_path:
+            # TODO: Implement this
+            raise NotImplementedError
+        else:
+            for adapter_name in self.adapter_set:
+                print(f"Initializing rotation adapter: {adapter_name}")
+                if not self.transformer._hf_peft_config_loaded:
+                    self.transformer._hf_peft_config_loaded = True
+                elif adapter_name in self.transformer.peft_config:
+                    raise ValueError(f"Adapter with name {adapter_name} already exists. Please use a different name.")
+                config = RotationConfig(**rotation_adapter_config)
+                rotation_tuner = RotationTuner(
+                    self.transformer,
+                    config,
+                    adapter_name=adapter_name,
+                )
+                # self.transformer = rotation_tuner.model
+                self.transformer.set_adapter(adapter_name)
+            rotation_layers = filter(
+                lambda p: p.requires_grad, self.transformer.parameters()
+            )
+            return list(rotation_layers)
+    # def save_lora(self, path: str):
+    #     for adapter_name in self.adapter_set:
+    #         FluxPipeline.save_lora_weights(
+    #             save_directory=path,
+    #             weight_name=f"{adapter_name}.safetensors",
+    #             transformer_lora_layers=get_peft_model_state_dict(
+    #                 self.transformer, adapter_name=adapter_name
+    #             ),
+    #             safe_serialization=True,
+    #         )
+    def save_rotation(self, path: str):
+        import os
+        from safetensors.torch import save_file
+        os.makedirs(path, exist_ok=True)
+        # Get the full model state dict (handles DDP, FSDP, etc.)
+        state_dict = self.transformer.state_dict()
+        for adapter_name in self.adapter_set:
+            to_return = {}
+            for k, v in state_dict.items():
+                if f".rotation.{adapter_name}." in k:
+                    # Remove DDP/FSDP prefixes if present
+                    clean_key = k.replace("module.", "").replace("_fsdp_wrapped_module.", "")
+                    to_return[clean_key] = v
+            if len(to_return) == 0:
+                print(f"Warning: No rotation parameters found for adapter {adapter_name}")
+                print(f"Available keys sample: {list(state_dict.keys())[:5]}")
+                continue
+            # Remove adapter name from keys (following PEFT convention)
+            # This makes loading easier and follows the pattern used by LoRA
+            to_return = {k.replace(f".{adapter_name}", ""): v for k, v in to_return.items()}
+            save_path = os.path.join(path, f"{adapter_name}.safetensors")
+            # Convert to CPU and detach before saving
+            to_return_cpu = {k: v.cpu().detach() for k, v in to_return.items()}
+            save_file(to_return_cpu, save_path)
+            total_params = sum(p.numel() for p in to_return.values())
+            num_U_params = sum(p.numel() for k, p in to_return.items() if '.U' in k)
+            num_V_params = sum(p.numel() for k, p in to_return.items() if '.V' in k)
+            print(f"Saved adapter '{adapter_name}':")
+            print(f"  - {len(to_return)} tensors ({total_params:,} total parameters)")
+            print(f"  - U parameters: {num_U_params:,}, V parameters: {num_V_params:,}")
+            print(f"  - Path: {save_path}")
+    def configure_optimizers(self):
+        # Freeze the transformer
+        self.transformer.requires_grad_(False)
+        opt_config = self.optimizer_config
+        # Set the trainable parameters
+        self.trainable_params = self.rotation_layers
+        print(f"Number of trainable parameters: {sum(p.numel() for p in self.trainable_params)}")
+        # Unfreeze trainable parameters
+        for p in self.trainable_params:
+            p.requires_grad_(True)
+        # Initialize the optimizer
+        if opt_config["type"] == "AdamW":
+            optimizer = torch.optim.AdamW(self.trainable_params, **opt_config["params"])
+        elif opt_config["type"] == "Prodigy":
+            optimizer = prodigyopt.Prodigy(
+                self.trainable_params,
+                **opt_config["params"],
+            )
+        elif opt_config["type"] == "SGD":
+            optimizer = torch.optim.SGD(self.trainable_params, **opt_config["params"])
+        else:
+            raise NotImplementedError("Optimizer not implemented.")
+        return optimizer
+    def training_step(self, batch, batch_idx):
+        imgs, prompts = batch["image"], batch["description"]
+        image_latent_mask = batch.get("image_latent_mask", None)
+        # Get the conditions and position deltas from the batch
+        conditions, position_deltas, position_scales, latent_masks = [], [], [], []
+        for i in range(1000):
+            if f"condition_{i}" not in batch:
+                break
+            conditions.append(batch[f"condition_{i}"])
+            position_deltas.append(batch.get(f"position_delta_{i}", [[0, 0]]))
+            position_scales.append(batch.get(f"position_scale_{i}", [1.0])[0])
+            latent_masks.append(batch.get(f"condition_latent_mask_{i}", None))
+        # Prepare inputs
+        with torch.no_grad():
+            # Prepare image input
+            x_0, img_ids = encode_images(self.flux_pipe, imgs)
+            # Prepare text input
+            (
+                prompt_embeds,
+                pooled_prompt_embeds,
+                text_ids,
+            ) = self.flux_pipe.encode_prompt(
+                prompt=prompts,
+                prompt_2=None,
+                prompt_embeds=None,
+                pooled_prompt_embeds=None,
+                device=self.flux_pipe.device,
+                num_images_per_prompt=1,
+                max_sequence_length=self.model_config.get("max_sequence_length", 512),
+                lora_scale=None,
+            )
+            # Prepare t and x_t
+            t = torch.sigmoid(torch.randn((imgs.shape[0],), device=self.device))
+            x_1 = torch.randn_like(x_0).to(self.device)
+            t_ = t.unsqueeze(1).unsqueeze(1)
+            x_t = ((1 - t_) * x_0 + t_ * x_1).to(self.dtype)
+            if image_latent_mask is not None:
+                x_0 = x_0[:, image_latent_mask[0]]
+                x_1 = x_1[:, image_latent_mask[0]]
+                x_t = x_t[:, image_latent_mask[0]]
+                img_ids = img_ids[image_latent_mask[0]]
+            # Prepare conditions
+            condition_latents, condition_ids = [], []
+            for cond, p_delta, p_scale, latent_mask in zip(
+                conditions, position_deltas, position_scales, latent_masks
+            ):
+                # Prepare conditions
+                c_latents, c_ids = encode_images(self.flux_pipe, cond)
+                # Scale the position (see OminiConrtol2)
+                if p_scale != 1.0:
+                    scale_bias = (p_scale - 1.0) / 2
+                    c_ids[:, 1:] *= p_scale
+                    c_ids[:, 1:] += scale_bias
+                # Add position delta (see OminiControl)
+                c_ids[:, 1] += p_delta[0][0]
+                c_ids[:, 2] += p_delta[0][1]
+                if len(p_delta) > 1:
+                    print("Warning: only the first position delta is used.")
+                # Append to the list
+                if latent_mask is not None:
+                    c_latents, c_ids = c_latents[latent_mask], c_ids[latent_mask[0]]
+                condition_latents.append(c_latents)
+                condition_ids.append(c_ids)
+            # Prepare guidance
+            guidance = (
+                torch.ones_like(t).to(self.device)
+                if self.transformer.config.guidance_embeds
+                else None
+            )
+        branch_n = 2 + len(conditions)
+        group_mask = torch.ones([branch_n, branch_n], dtype=torch.bool).to(self.device)
+        # Disable the attention cross different condition branches
+        group_mask[2:, 2:] = torch.diag(torch.tensor([1] * len(conditions)))
+        # Disable the attention from condition branches to image branch and text branch
+        if self.model_config.get("independent_condition", False):
+            group_mask[2:, :2] = False
+        # Forward pass
+        transformer_out = transformer_forward(
+            self.transformer,
+            image_features=[x_t, *(condition_latents)],
+            text_features=[prompt_embeds],
+            img_ids=[img_ids, *(condition_ids)],
+            txt_ids=[text_ids],
+            # There are three timesteps for the three branches
+            # (text, image, and the condition)
+            timesteps=[t, t] + [torch.zeros_like(t)] * len(conditions),
+            # Same as above
+            pooled_projections=[pooled_prompt_embeds] * branch_n,
+            guidances=[guidance] * branch_n,
+            # The LoRA adapter names of each branch
+            adapters=self.adapter_names,
+            return_dict=False,
+            group_mask=group_mask,
+        )
+        pred = transformer_out[0]
+        # Compute loss
+        step_loss = torch.nn.functional.mse_loss(pred, (x_1 - x_0), reduction="mean")
+        self.last_t = t.mean().item()
+        self.log_loss = (
+            step_loss.item()
+            if not hasattr(self, "log_loss")
+            else self.log_loss * 0.95 + step_loss.item() * 0.05
+        )
+        return step_loss
+    def generate_a_sample(self):
+        raise NotImplementedError("Generate a sample not implemented.")
+class TrainingCallback(L.Callback):
+    def __init__(self, run_name, training_config: dict = {}, test_function=None):
+        self.run_name, self.training_config = run_name, training_config
+        self.print_every_n_steps = training_config.get("print_every_n_steps", 10)
+        self.save_interval = training_config.get("save_interval", 1000)
+        self.sample_interval = training_config.get("sample_interval", 1000)
+        self.save_path = training_config.get("save_path", "./output")
+        self.wandb_config = training_config.get("wandb", None)
+        self.use_wandb = (
+            wandb is not None and os.environ.get("WANDB_API_KEY") is not None
+        )
+        self.total_steps = 0
+        self.test_function = test_function
+    def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
+        gradient_size = 0
+        max_gradient_size = 0
+        count = 0
+        for _, param in pl_module.named_parameters():
+            if param.grad is not None:
+                gradient_size += param.grad.norm(2).item()
+                max_gradient_size = max(max_gradient_size, param.grad.norm(2).item())
+                count += 1
+        if count > 0:
+            gradient_size /= count
+        self.total_steps += 1
+        # Print training progress every n steps
+        if self.use_wandb:
+            report_dict = {
+                "steps": batch_idx,
+                "steps": self.total_steps,
+                "epoch": trainer.current_epoch,
+                "gradient_size": gradient_size,
+            }
+            loss_value = outputs["loss"].item() * trainer.accumulate_grad_batches
+            report_dict["loss"] = loss_value
+            report_dict["t"] = pl_module.last_t
+            wandb.log(report_dict)
+        if self.total_steps % self.print_every_n_steps == 0:
+            print(
+                f"Epoch: {trainer.current_epoch}, Steps: {self.total_steps}, Batch: {batch_idx}, Loss: {pl_module.log_loss:.4f}, Gradient size: {gradient_size:.4f}, Max gradient size: {max_gradient_size:.4f}"
+            )
+        # Save LoRA weights at specified intervals
+        if self.total_steps % self.save_interval == 0:
+            print(
+                f"Epoch: {trainer.current_epoch}, Steps: {self.total_steps} - Saving LoRA weights"
+            )
+            pl_module.save_rotation(
+                f"{self.save_path}/{self.run_name}/ckpt/{self.total_steps}"
+            )
+            # pl_module.save_lora(
+            #     f"{self.save_path}/{self.run_name}/ckpt/{self.total_steps}"
+            # )
+        # Generate and save a sample image at specified intervals
+        if self.total_steps % self.sample_interval == 0 and self.test_function:
+            print(
+                f"Epoch: {trainer.current_epoch}, Steps: {self.total_steps} - Generating a sample"
+            )
+            pl_module.eval()
+            self.test_function(
+                pl_module,
+                f"{self.save_path}/{self.run_name}/output",
+                f"lora_{self.total_steps}",
+            )
+            pl_module.train()
+def train(dataset, trainable_model, config, test_function):
+    # Initialize
+    is_main_process, rank = get_rank() == 0, get_rank()
+    torch.cuda.set_device(rank)
+    # config = get_config()
+    training_config = config["train"]
+    run_name = time.strftime("%Y%m%d-%H%M%S")
+    # Initialize WanDB
+    wandb_config = training_config.get("wandb", None)
+    if wandb_config is not None and is_main_process:
+        init_wandb(wandb_config, run_name)
+    print("Rank:", rank)
+    if is_main_process:
+        print("Config:", config)
+    # Initialize dataloader
+    print("Dataset length:", len(dataset))
+    train_loader = DataLoader(
+        dataset,
+        batch_size=training_config.get("batch_size", 1),
+        shuffle=True,
+        num_workers=training_config["dataloader_workers"],
+    )
+    # Callbacks for testing and saving checkpoints
+    if is_main_process:
+        callbacks = [TrainingCallback(run_name, training_config, test_function)]
+    # Initialize trainer
+    trainer = L.Trainer(
+        accumulate_grad_batches=training_config["accumulate_grad_batches"],
+        callbacks=callbacks if is_main_process else [],
+        enable_checkpointing=False,
+        enable_progress_bar=False,
+        logger=False,
+        max_steps=training_config.get("max_steps", -1),
+        max_epochs=training_config.get("max_epochs", -1),
+        gradient_clip_val=training_config.get("gradient_clip_val", 0.5),
+    )
+    setattr(trainer, "training_config", training_config)
+    setattr(trainable_model, "training_config", training_config)
+    # Save the training config
+    save_path = training_config.get("save_path", "./output")
+    if is_main_process:
+        os.makedirs(f"{save_path}/{run_name}")
+        with open(f"{save_path}/{run_name}/config.yaml", "w") as f:
+            yaml.dump(config, f)
+    # Start training
+    trainer.fit(trainable_model, train_loader)

train/README.md ADDED Viewed

	@@ -0,0 +1,253 @@

+# Training for FLUX
+## Table of Contents
+- [Training for FLUX](#training-for-flux)
+  - [Table of Contents](#table-of-contents)
+  - [Environment Setup](#environment-setup)
+  - [Dataset Preparation](#dataset-preparation)
+  - [Quick Start](#quick-start)
+  - [Basic Training](#basic-training)
+    - [Tasks from OminiControl](#tasks-from-ominicontrol)
+    - [Creating Your Own Task](#creating-your-own-task)
+    - [Training Configuration](#training-configuration)
+      - [Batch Size](#batch-size)
+      - [Optimizer](#optimizer)
+      - [LoRA Configuration](#lora-configuration)
+      - [Trainable Modules](#trainable-modules)
+  - [Advanced Training](#advanced-training)
+    - [Multi-condition](#multi-condition)
+    - [Efficient Generation (OminiControl2)](#efficient-generation-ominicontrol2)
+      - [Feature Reuse (KV-Cache)](#feature-reuse-kv-cache)
+      - [Compact Encoding Representation](#compact-encoding-representation)
+      - [Token Integration (for Fill task)](#token-integration-for-fill-task)
+  - [Citation](#citation)
+## Environment Setup
+1. Create and activate a new conda environment:
+   ```bash
+   conda create -n omini python=3.10
+   conda activate omini
+   ```
+2. Install required packages:
+   ```bash
+   pip install -r requirements.txt
+   ```
+## Dataset Preparation
+1. Download [Subject200K](https://huggingface.co/datasets/Yuanshi/Subjects200K) dataset for subject-driven generation:
+   ```bash
+   bash train/script/data_download/data_download1.sh
+   ```
+2. Download [text-to-image-2M](https://huggingface.co/datasets/jackyhate/text-to-image-2M) dataset for spatial alignment control tasks:
+   ```bash
+   bash train/script/data_download/data_download2.sh
+   ```
+   **Note:** By default, only a few files will be downloaded. You can edit `data_download2.sh` to download more data, and update the config file accordingly.
+## Quick Start
+Use these scripts to start training immediately:
+1. **Subject-driven generation**:
+   ```bash
+   bash train/script/train_subject.sh
+   ```
+2. **Spatial control tasks** (Canny-to-image, colorization, depth map, etc.):
+   ```bash
+   bash train/script/train_spatial_alignment.sh
+   ```
+3. **Multi-condition training**:
+   ```bash
+   bash train/script/train_multi_condition.sh
+   ```
+4. **Feature reuse** (OminiControl2):
+   ```bash
+   bash train/script/train_feature_reuse.sh
+   ```
+5. **Compact token representation** (OminiControl2):
+   ```bash
+   bash train/script/train_compact_token_representation.sh
+   ```
+6. **Token integration** (OminiControl2):
+   ```bash
+   bash train/script/train_token_intergration.sh
+   ```
+## Basic Training
+### Tasks from OminiControl
+<a href="https://arxiv.org/abs/2411.15098"><img src="https://img.shields.io/badge/ariXv-2411.15098-A42C25.svg" alt="arXiv"></a>
+1. Subject-driven generation:
+   ```bash
+   bash train/script/train_subject.sh
+   ```
+2. Spatial control tasks (using canny-to-image as example):
+   ```bash
+   bash train/script/train_spatial_alignment.sh
+   ```
+   <details>
+   <summary>Supported tasks</summary>
+   * Canny edge to image (`canny`)
+   * Image colorization (`coloring`)
+   * Image deblurring (`deblurring`)
+   * Depth map to image (`depth`)
+   * Image to depth map (`depth_pred`)
+   * Image inpainting (`fill`)
+   * Super resolution (`sr`)
+   🌟 Change the `condition_type` parameter in the config file to switch between tasks.
+   </details>
+**Note**: Check the **script files** (`train/script/`) and **config files** (`train/configs/`) for WanDB and GPU settings.
+### Creating Your Own Task
+You can create a custom task by building a new dataset and modifying the test code:
+1. **Create a custom dataset:**
+   Your custom dataset should follow the format of `Subject200KDataset` in `omini/train_flux/train_subject.py`. Each sample should contain:
+   - Image: the target image (`image`)
+   - Text: description of the image (`description`)
+   - Conditions: image conditions for generation
+   - Position delta:
+     - Use `position_delta = (0, 0)` to align the condition with the generated image
+     - Use `position_delta = (0, -a)` to separate them (a = condition width / 16)
+   > **Explanation:**
+   > The model places both the condition and generated image in a shared coordinate system. `position_delta` shifts the condition image in this space.
+   >
+   > Each unit equals one patch (16 pixels). For a 512px-wide condition image (32 patches), `position_delta = (0, -32)` moves it fully to the left.
+   >
+   > This controls whether conditions and generated images share space or appear side-by-side.
+2. **Modify the test code:**
+   Define `test_function()` in `train_custom.py`. Refer to the function in `train_subject.py` for examples. Make sure to keep the `position_delta` parameter consistent with your dataset.
+### Training Configuration
+#### Batch Size
+We recommend a batch size of 1 for stable training. And you can set `accumulate_grad_batches` to n to simulate a batch size of n.
+#### Optimizer
+The default optimizer is `Prodigy`. To use `AdamW` instead, modify the config file:
+```yaml
+optimizer:
+  type: AdamW
+  lr: 1e-4
+  weight_decay: 0.001
+```
+#### LoRA Configuration
+Default LoRA rank is 4. Increase it for complex tasks (keep `r` and `lora_alpha` parameters the same):
+```yaml
+lora_config:
+  r: 128
+  lora_alpha: 128
+```
+#### Trainable Modules
+The `target_modules` parameter uses regex patterns to specify which modules to train. See [PEFT Documentation](https://huggingface.co/docs/peft/package_reference/lora) for details.
+Default configuration trains all modules affecting image tokens:
+```yaml
+target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+```
+To train only attention components (`to_q`, `to_k`, `to_v`), use:
+```yaml
+target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v)"
+```
+## Advanced Training
+### Multi-condition
+A basic multi-condition implementation is available in `train_multi_condition.py`:
+```bash
+bash train/script/train_multi_condition.sh
+```
+### Efficient Generation (OminiControl2)
+<a href="https://arxiv.org/abs/2503.08280"><img src="https://img.shields.io/badge/ariXv-2503.08280-A42C25.svg" alt="arXiv"></a>
+[OminiControl2](https://arxiv.org/abs/2503.08280) introduces techniques to improve generation efficiency:
+#### Feature Reuse (KV-Cache)
+1. Enable `independent_condition` in the config file during training:
+   ```yaml
+   model:
+     independent_condition: true
+   ```
+2. During inference, set `kv_cache = True` in the `generate` function to speed up generation.
+*Example:*
+```bash
+bash train/script/train_feature_reuse.sh
+```
+**Note:** Feature reuse speeds up generation but may slightly reduce performance and increase training time.
+#### Compact Encoding Representation
+Reduce the condition image resolution and use `position_scale` to align it with the output image:
+```diff
+train:
+  dataset:
+    condition_size:
+-     - 512
+-     - 512
++     - 256
++     - 256
++   position_scale: 2
+    target_size:
+      - 512
+      - 512
+```
+*Example:*
+```bash
+bash train/script/train_compact_token_representation.sh
+```
+#### Token Integration (for Fill task)
+Further reduce tokens by merging condition and generation tokens into a unified sequence. (Refer to [the paper](https://arxiv.org/abs/2503.08280) for details.)
+*Example:*
+```bash
+bash train/script/train_token_intergration.sh
+```
+## Citation
+If you find this code useful, please cite our papers:
+```
+@article{tan2024ominicontrol,
+  title={OminiControl: Minimal and Universal Control for Diffusion Transformer},
+  author={Tan, Zhenxiong and Liu, Songhua and Yang, Xingyi and Xue, Qiaochu and Wang, Xinchao},
+  journal={arXiv preprint arXiv:2411.15098},
+  year={2024}
+}
+@article{tan2025ominicontrol2,
+  title={OminiControl2: Efficient Conditioning for Diffusion Transformers},
+  author={Tan, Zhenxiong and Xue, Qiaochu and Yang, Xingyi and Liu, Songhua and Wang, Xinchao},
+  journal={arXiv preprint arXiv:2503.08280},
+  year={2025}
+}
+```

train/config/compact_token_representation.yaml ADDED Viewed

	@@ -0,0 +1,65 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: false
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: true # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  # Options: ["canny", "coloring", "deblurring", "depth", "depth_pred", "fill"]
+  condition_type: "canny"
+  dataset:
+    type: "img"
+    urls:
+      # (Uncomment the following lines to use more data)
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000040.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000041.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000042.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000043.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000044.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000045.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000046.tar"
+    cache_name: "data_512_2M"
+    condition_size:
+      - 256
+      - 256
+    position_scale: 2.0
+    target_size:
+      - 512
+      - 512
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControl"
+  lora_config:
+    r: 4
+    lora_alpha: 4
+    init_lora_weights: "gaussian"
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/config/feature_reuse.yaml ADDED Viewed

	@@ -0,0 +1,64 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: true
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: true # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  # Options: ["canny", "coloring", "deblurring", "depth", "depth_pred", "fill"]
+  condition_type: "canny"
+  dataset:
+    type: "img"
+    urls:
+      # (Uncomment the following lines to use more data)
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000040.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000041.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000042.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000043.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000044.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000045.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000046.tar"
+    cache_name: "data_512_2M"
+    condition_size:
+      - 512
+      - 512
+    target_size:
+      - 512
+      - 512
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControl"
+  lora_config:
+    r: 4
+    lora_alpha: 4
+    init_lora_weights: "gaussian"
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/config/multi_condition.yaml ADDED Viewed

	@@ -0,0 +1,68 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: false
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: true # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  # Options: ["canny", "coloring", "deblurring", "depth", "depth_pred", "fill"]
+  condition_type:
+    - "canny"
+    - "deblurring"
+    - "depth"
+    - "fill"
+  dataset:
+    type: "img"
+    urls:
+      # (Uncomment the following lines to use more data)
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000040.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000041.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000042.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000043.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000044.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000045.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000046.tar"
+    cache_name: "data_512_2M"
+    condition_size:
+      - 512
+      - 512
+    target_size:
+      - 512
+      - 512
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControl"
+  lora_config:
+    r: 4
+    lora_alpha: 4
+    init_lora_weights: "gaussian"
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/config/spatial_alignment.yaml ADDED Viewed

	@@ -0,0 +1,64 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: false
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: true # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  # Options: ["canny", "coloring", "deblurring", "depth", "depth_pred", "fill"]
+  condition_type: "canny"
+  dataset:
+    type: "img"
+    urls:
+      # (Uncomment the following lines to use more data)
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000040.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000041.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000042.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000043.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000044.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000045.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000046.tar"
+    cache_name: "data_512_2M"
+    condition_size:
+      - 512
+      - 512
+    target_size:
+      - 512
+      - 512
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControl"
+  lora_config:
+    r: 4
+    lora_alpha: 4
+    init_lora_weights: "gaussian"
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/config/spatial_alignment_rotation.yaml ADDED Viewed

	@@ -0,0 +1,63 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: false
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: false # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  # Options: ["canny", "coloring", "deblurring", "depth", "depth_pred", "fill"]
+  condition_type: "canny"
+  dataset:
+    type: "img"
+    urls:
+      # (Uncomment the following lines to use more data)
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000040.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000041.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000042.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000043.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000044.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000045.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000046.tar"
+    cache_name: "data_512_2M"
+    condition_size:
+      - 512
+      - 512
+    target_size:
+      - 512
+      - 512
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControlRotation"
+  rotation_adapter_config:
+    r: 4
+    num_rotations: 4
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/config/subject.yaml ADDED Viewed

	@@ -0,0 +1,54 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: false
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: true # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  condition_type: "subject"
+  dataset:
+    type: "subject"
+    condition_size:
+      - 512
+      - 512
+    target_size:
+      - 512
+      - 512
+    image_size: 512
+    padding: 8
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControl"
+  lora_config:
+    r: 16
+    lora_alpha: 16
+    init_lora_weights: "gaussian"
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/config/subject_rotation.yaml ADDED Viewed

	@@ -0,0 +1,53 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: false
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: false # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  condition_type: "subject"
+  dataset:
+    type: "subject"
+    condition_size:
+      - 512
+      - 512
+    target_size:
+      - 512
+      - 512
+    image_size: 512
+    padding: 8
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControRotation"
+  rotation_adapter_config:
+    r: 4
+    num_rotations: 4
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/config/token_integration.yaml ADDED Viewed

	@@ -0,0 +1,63 @@

+flux_path: "black-forest-labs/FLUX.1-dev"
+dtype: "bfloat16"
+model:
+  independent_condition: false
+train:
+  accumulate_grad_batches: 1
+  dataloader_workers: 5
+  save_interval: 1000
+  sample_interval: 100
+  max_steps: -1
+  gradient_checkpointing: true # (Turn off for faster training)
+  save_path: "runs"
+  # Specify the type of condition to use.
+  condition_type: "token_intergration"
+  dataset:
+    type: "img"
+    urls:
+      # (Uncomment the following lines to use more data)
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000040.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000041.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000042.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000043.tar"
+      # - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000044.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000045.tar"
+      - "https://huggingface.co/datasets/jackyhate/text-to-image-2M/resolve/main/data_512_2M/data_000046.tar"
+    cache_name: "data_512_2M"
+    condition_size:
+      - 512
+      - 512
+    target_size:
+      - 512
+      - 512
+    drop_text_prob: 0.1
+    drop_image_prob: 0.1
+  wandb:
+    project: "OminiControl"
+  lora_config:
+    r: 4
+    lora_alpha: 4
+    init_lora_weights: "gaussian"
+    target_modules: "(.*x_embedder|.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_v|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_out\\.0|.*(?<!single_)transformer_blocks\\.[0-9]+\\.ff\\.net\\.2|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.proj_mlp|.*single_transformer_blocks\\.[0-9]+\\.proj_out|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q|.*single_transformer_blocks\\.[0-9]+\\.attn.to_v|.*single_transformer_blocks\\.[0-9]+\\.attn.to_out)"
+    # (Uncomment the following lines to train less parameters while keeping the similar performance)
+    # target_modules: "(.*(?<!single_)transformer_blocks\\.[0-9]+\\.norm1\\.linear|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_k|.*(?<!single_)transformer_blocks\\.[0-9]+\\.attn\\.to_q|.*single_transformer_blocks\\.[0-9]+\\.norm\\.linear|.*single_transformer_blocks\\.[0-9]+\\.attn.to_k|.*single_transformer_blocks\\.[0-9]+\\.attn.to_q)"
+  optimizer:
+    type: "Prodigy"
+    params:
+      lr: 1
+      use_bias_correction: true
+      safeguard_warmup: true
+      weight_decay: 0.01
+  # (To use AdamW Optimizer, uncomment the following lines)
+  # optimizer:
+  #   type: AdamW
+  #   lr: 1e-4
+  #   weight_decay: 0.001

train/requirements.txt ADDED Viewed

	@@ -0,0 +1,15 @@

+diffusers==0.31.0
+transformers
+peft
+opencv-python
+protobuf
+sentencepiece
+gradio
+jupyter
+torchao
+lightning
+datasets
+torchvision
+prodigyopt
+wandb

train/script/data_download/data_download1.sh ADDED Viewed

	@@ -0,0 +1 @@


1	+ huggingface-cli download --repo-type dataset Yuanshi/Subjects200K

train/script/data_download/data_download2.sh ADDED Viewed

	@@ -0,0 +1,3 @@

+huggingface-cli download --repo-type dataset jackyhate/text-to-image-2M data_512_2M/data_000045.tar
+huggingface-cli download --repo-type dataset jackyhate/text-to-image-2M data_512_2M/data_000046.tar
+huggingface-cli download --repo-type dataset jackyhate/text-to-image-2M data_1024_10K/data_000000.tar

train/script/train_compact_token_representation.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/compact_token_representation.yaml
+# *[Specify the WANDB API key]
+# export WANDB_API_KEY='YOUR_WANDB_API_KEY'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41353 -m omini.train_flux.train_spatial_alignment

train/script/train_feature_reuse.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/feature_reuse.yaml
+# *[Specify the WANDB API key]
+# export WANDB_API_KEY='YOUR_WANDB_API_KEY'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41353 -m omini.train_flux.train_spatial_alignment

train/script/train_multi_condition.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/multi_condition.yaml
+# *[Specify the WANDB API key]
+# export WANDB_API_KEY='YOUR_WANDB_API_KEY'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41353 -m omini.train_flux.train_multi_condition

train/script/train_spatial_alignment.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/spatial_alignment.yaml
+# *[Specify the WANDB API key]
+# export WANDB_API_KEY='YOUR_WANDB_API_KEY'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41353 -m omini.train_flux.train_spatial_alignment

train/script/train_spatial_alignment_rotation.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/spatial_alignment_rotation.yaml
+# *[Specify the WANDB API key]
+export WANDB_API_KEY='675d7815d8ad1cf89954fd5415071f18007386fd'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41353 -m omini.train_flux.train_spatial_alignment_rotation

train/script/train_subject.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/subject.yaml
+# *[Specify the WANDB API key]
+# export WANDB_API_KEY='YOUR_WANDB_API_KEY'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41353 -m omini.train_flux.train_subject

train/script/train_subject_rotation.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/subject_rotation.yaml
+# *[Specify the WANDB API key]
+export WANDB_API_KEY='675d7815d8ad1cf89954fd5415071f18007386fd'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41354 -m omini.train_flux.train_subject_rotation

train/script/train_token_intergration.sh ADDED Viewed

	@@ -0,0 +1,13 @@

+# *[Specify the config file path and the GPU devices to use]
+# export CUDA_VISIBLE_DEVICES=0,1
+# *[Specify the config file path]
+export OMINI_CONFIG=./train/config/token_integration.yaml
+# *[Specify the WANDB API key]
+# export WANDB_API_KEY='YOUR_WANDB_API_KEY'
+echo $OMINI_CONFIG
+export TOKENIZERS_PARALLELISM=true
+accelerate launch --main_process_port 41353 -m omini.train_flux.train_token_integration