Further trained model

README.md

@@ -11,4 +11,48 @@ license: cc-by-nc-4.0
 
 This model was fine-tuned on light and web novel for Japanese to English translation.
 
-Can translate sentences and paragraphs not exceeding 128 tokens.
+It can translate sentences and paragraphs up to 512 tokens.
+
+
+## Usage
+```python
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
+tokenizer = AutoTokenizer.from_pretrained("thefrigidliquidation/nllb-jaen-1.3B-lightnovels")
+model = AutoModelForSeq2SeqLM.from_pretrained("thefrigidliquidation/nllb-jaen-1.3B-lightnovels")
+
+generated_tokens = model.generate(
+    **inputs,
+    forced_bos_token_id=tokenizer.lang_code_to_id[tokenizer.tgt_lang],
+    max_new_tokens=1024,
+    no_repeat_ngram_size=6,
+).cpu()
+
+translated_text = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0]
+```
+
+Generating with diverse beam search seems to work best. Add the following to `model.generate`:
+```python
+num_beams=8,
+num_beam_groups=4,
+do_sample=False,
+```
+
+
+## Glossary
+You can provide up to 10 custom translations for nouns and character names at runtime. To do so, surround the Japanese term with term tokens. Prefix the word with one of <t0>, <t1>, ..., <t9> and suffix the word with </t>. The term will be translated as the prefix term token which can then be string replaced.
+
+For example, in "マイン、ルッツが迎えに来たよ" if you wish to have "マイン" translated as "Myne" you would replace "マイン" with "<t0>マイン</t>". The model will translate "<t0>マイン</t>、ルッツが迎えに来たよ" as "<t0>, Lutz is here to pick you up." Then simply do a string replacement on the output, replacing "<t0>" with "Myne".
+
+
+## Honorifics
+You can force the model to generate or ignore honorifics.
+
+```python
+# default, the model decides whether to use honorifics
+tokenizer.tgt_lang = "jpn_Jpan"
+# no honorifics, the model is discouraged from using honorifics
+tokenizer.tgt_lang = "zsm_Latn"
+# honorifics, the model is encouraged to use honorifics
+tokenizer.tgt_lang = "zul_Latn"
+```

added_tokens.json

@@ -0,0 +1,1039 @@
+{
+  "</fs>": 257229,
+  "</t>": 257230,
+  "<fs>": 257228,
+  "<t0>": 257231,
+  "<t1>": 257232,
+  "<t2>": 257233,
+  "<t3>": 257234,
+  "<t4>": 257235,
+  "<t5>": 257236,
+  "<t6>": 257237,
+  "<t7>": 257238,
+  "<t8>": 257239,
+  "<t9>": 257240,
+  "⦅": 257198,
+  "〆": 256692,
+  "〒": 256881,
+  "〘": 257139,
+  "そ嚼": 257013,
+  "ひ翠": 257194,
+  "ゎ": 257125,
+  "゠": 257211,
+  "ヲ": 256383,
+  "丙": 257136,
+  "丞": 256747,
+  "丼": 256438,
+  "乖": 256767,
+  "仄": 256379,
+  "伎": 256723,
+  "伽": 256564,
+  "伽噺": 256717,
+  "佃": 257227,
+  "佇": 256265,
+  "侘": 256894,
+  "侭": 256715,
+  "俎": 257142,
+  "俟": 257014,
+  "俵": 256648,
+  "俸": 256990,
+  "倅": 257017,
+  "倖": 256693,
+  "倶": 256398,
+  "倹": 256980,
+  "偃": 257212,
+  "偲": 256955,
+  "傀": 256739,
+  "傀儡": 256489,
+  "傅": 256891,
+  "傻": 257204,
+  "僥": 256700,
+  "僥倖": 256760,
+  "僭": 256628,
+  "僻": 256709,
+  "儂": 256215,
+  "儚": 256309,
+  "儡": 256745,
+  "兎": 256236,
+  "兎鎧": 257202,
+  "兜": 256248,
+  "兜蟹": 257115,
+  "冑": 256333,
+  "冨": 256565,
+  "冴": 256267,
+  "冶": 256231,
+  "冽": 256927,
+  "凛": 256227,
+  "凜": 256338,
+  "凧": 257034,
+  "凪": 256335,
+  "凹": 256369,
+  "刎": 256512,
+  "刮": 257032,
+  "刹": 256272,
+  "剋": 256802,
+  "剌": 256913,
+  "剝": 256233,
+  "劈": 256979,
+  "劾": 256844,
+  "勅": 256598,
+  "匡": 256241,
+  "卍": 257149,
+  "卯": 256678,
+  "厠": 257065,
+  "厩": 256538,
+  "厰": 256938,
+  "叢": 256813,
+  "叭": 257173,
+  "吶": 256817,
+  "吼": 256368,
+  "吽": 256964,
+  "呃": 257177,
+  "呉": 256790,
+  "呐": 257216,
+  "呗": 256873,
+  "呟": 256205,
+  "呵": 256513,
+  "呷": 256469,
+  "呻": 256230,
+  "咀": 256511,
+  "咀嚼": 256544,
+  "咄": 256282,
+  "咄嗟": 256264,
+  "咆": 256337,
+  "咆吼": 257049,
+  "咆哮": 256294,
+  "咢": 257188,
+  "咤": 256495,
+  "咥": 256366,
+  "咦": 257106,
+  "咫": 256968,
+  "咱": 256641,
+  "咽": 256410,
+  "哂": 257051,
+  "哄": 256503,
+  "哮": 256348,
+  "唖": 256298,
+  "啄": 257150,
+  "啖": 256721,
+  "啖呵": 257036,
+  "啜": 256389,
+  "啞": 256382,
+  "啼": 256869,
+  "喔": 256849,
+  "嗄": 256792,
+  "嗇": 257196,
+  "嗚": 256399,
+  "嗚咽": 256560,
+  "嗟": 256257,
+  "嗤": 256320,
+  "嗯": 256753,
+  "嗾": 257214,
+  "嘯": 256652,
+  "嘶": 256707,
+  "噌": 256319,
+  "噎": 256899,
+  "噓": 256212,
+  "噤": 256363,
+  "噺": 256764,
+  "嚙": 256219,
+  "嚢": 256708,
+  "嚥": 256801,
+  "嚼": 256579,
+  "囀": 256827,
+  "囂": 256996,
+  "囃": 256694,
+  "囓": 256662,
+  "囮": 256313,
+  "坤": 257080,
+  "坩堝": 256886,
+  "埒": 256372,
+  "埜": 256246,
+  "埠": 256908,
+  "埴": 256521,
+  "埼": 256575,
+  "堀": 256214,
+  "堰": 256591,
+  "堺": 256779,
+  "塀": 256345,
+  "塁": 256719,
+  "塒": 256972,
+  "塡": 256549,
+  "塹": 257071,
+  "塹壕": 256758,
+  "墾": 256729,
+  "壕": 256763,
+  "壜": 256966,
+  "壬": 256228,
+  "壱": 257183,
+  "壷": 256548,
+  "壺": 256270,
+  "妳": 256623,
+  "妾": 256240,
+  "姑": 256547,
+  "姜": 256635,
+  "婉": 256897,
+  "婿": 256311,
+  "媚": 256436,
+  "媛": 256889,
+  "嫋": 257178,
+  "嫡": 256529,
+  "嫣": 256936,
+  "嬌": 256468,
+  "嬪": 257090,
+  "嬲": 256571,
+  "孵": 256620,
+  "宋": 256868,
+  "宦": 256371,
+  "寇": 256783,
+  "寞": 257021,
+  "寥": 256784,
+  "屁": 256629,
+  "屑": 256365,
+  "屓": 256780,
+  "屯": 256501,
+  "屹": 256663,
+  "岬": 256443,
+  "峠": 256822,
+  "峨": 257137,
+  "峯": 257068,
+  "崑": 257164,
+  "崑崙": 257043,
+  "崗": 257157,
+  "崙": 257165,
+  "嵩": 256589,
+  "嶋": 256329,
+  "嶮": 257206,
+  "嶺": 256447,
+  "嶺醍": 256740,
+  "嶽": 256999,
+  "巌": 256733,
+  "巳": 256614,
+  "巷": 256646,
+  "帛": 256706,
+  "帥": 256435,
+  "帷": 256583,
+  "幇": 257084,
+  "幌": 256496,
+  "幡": 256315,
+  "庵": 256425,
+  "庸": 256552,
+  "弐": 257050,
+  "弑": 257119,
+  "弔": 256603,
+  "弘": 256224,
+  "弛": 256430,
+  "弩": 256612,
+  "彌": 257054,
+  "彿": 256478,
+  "徊": 256670,
+  "徘": 256697,
+  "徘徊": 256567,
+  "徨": 256316,
+  "徽": 256607,
+  "忖": 257063,
+  "忸": 256847,
+  "忸怩": 256901,
+  "怩": 256848,
+  "恃": 257070,
+  "恍": 256572,
+  "恙": 257042,
+  "悄": 256775,
+  "悍": 256466,
+  "悧": 256762,
+  "悴": 256752,
+  "悸": 256650,
+  "惣": 256845,
+  "愴": 256785,
+  "愾": 256698,
+  "慄": 256286,
+  "慇": 256793,
+  "慇懃": 256816,
+  "慟": 256605,
+  "憊": 256659,
+  "憔": 256768,
+  "憔悴": 256788,
+  "憚": 256446,
+  "憬": 256505,
+  "憮": 256431,
+  "憺": 256864,
+  "懃": 256799,
+  "懊": 256695,
+  "懣": 256916,
+  "懦": 256993,
+  "懺": 256573,
+  "戟": 256479,
+  "扁": 257099,
+  "扈": 256741,
+  "抉": 256285,
+  "拌": 256965,
+  "拮": 256482,
+  "拱": 257193,
+  "拵": 256604,
+  "捌": 256297,
+  "捩": 256535,
+  "捲": 256414,
+  "捺": 256924,
+  "捻": 256225,
+  "掏": 257123,
+  "掣": 257140,
+  "掬": 256499,
+  "揄": 256627,
+  "揉": 256234,
+  "揶": 256622,
+  "揶揄": 256600,
+  "搅拌": 256984,
+  "搔": 256232,
+  "搦": 256527,
+  "摑": 256211,
+  "摺": 256394,
+  "撑": 257217,
+  "撓": 256917,
+  "撥": 256377,
+  "撹": 256734,
+  "撻": 257220,
+  "擢": 256726,
+  "擱": 256923,
+  "擲": 256310,
+  "擽": 256910,
+  "攀": 256836,
+  "攣": 256306,
+  "攪": 256465,
+  "攪拌": 256770,
+  "攫": 256269,
+  "斂": 257052,
+  "斃": 256453,
+  "斟": 256971,
+  "斡": 256656,
+  "昌": 256455,
+  "昴": 256714,
+  "晃": 256773,
+  "晦": 256661,
+  "暈": 257215,
+  "曙": 257108,
+  "曳": 256703,
+  "曽": 256451,
+  "朕": 256976,
+  "朦": 256595,
+  "朦朧": 256609,
+  "朧": 256408,
+  "朶": 256581,
+  "杓": 256728,
+  "杞": 256412,
+  "杳": 257117,
+  "杵": 257131,
+  "枡": 257126,
+  "枷": 256351,
+  "柑": 256806,
+  "柑橘": 256956,
+  "柘": 256934,
+  "柚": 256798,
+  "柩": 256866,
+  "柵": 256289,
+  "柿": 256887,
+  "栂": 256912,
+  "栅": 257081,
+  "栖": 256879,
+  "栞": 256669,
+  "桂": 256312,
+  "桐": 256213,
+  "桓": 256884,
+  "桔": 256675,
+  "桝": 256911,
+  "桟": 256428,
+  "桧": 257027,
+  "桿": 256687,
+  "梃": 257007,
+  "梓": 256396,
+  "梟": 256789,
+  "梢": 256658,
+  "梳": 256508,
+  "梶": 256619,
+  "棹": 257011,
+  "椀": 256542,
+  "椋": 257059,
+  "椒": 256439,
+  "椰": 257219,
+  "椿": 256419,
+  "楊": 256743,
+  "楓": 256247,
+  "楔": 256592,
+  "楕": 256673,
+  "楠": 256599,
+  "楯": 256626,
+  "榊": 257097,
+  "榎": 256634,
+  "槃": 257130,
+  "槌": 256251,
+  "樋": 256526,
+  "樫": 256287,
+  "樵": 256888,
+  "橇": 257046,
+  "橘": 256442,
+  "橙": 256361,
+  "檀": 256742,
+  "檄": 256854,
+  "檎": 256331,
+  "檜": 256373,
+  "櫂": 256210,
+  "櫓": 256689,
+  "櫛": 256235,
+  "欅": 257201,
+  "欒": 256925,
+  "殲": 256276,
+  "毅": 256357,
+  "毘": 257035,
+  "毟": 256494,
+  "毬": 257005,
+  "氈": 257195,
+  "氾": 256639,
+  "汀": 256445,
+  "汐": 257012,
+  "沁": 256649,
+  "沐": 257057,
+  "沫": 256336,
+  "沱": 256960,
+  "沽": 256876,
+  "洒": 256303,
+  "洟": 256862,
+  "浚": 257009,
+  "浩": 256288,
+  "浩浩": 256969,
+  "涎": 256473,
+  "涛": 256637,
+  "涨": 257205,
+  "涸": 256685,
+  "淀": 256346,
+  "淆": 257222,
+  "淑": 256359,
+  "淳": 256613,
+  "渠": 257072,
+  "渾": 256300,
+  "湘": 257110,
+  "湛": 256358,
+  "溌": 257022,
+  "滂": 256988,
+  "滓": 256416,
+  "滔": 256696,
+  "滲": 256221,
+  "滸": 257067,
+  "漉": 256810,
+  "漕": 256391,
+  "漣": 257143,
+  "漿": 256833,
+  "潟": 257184,
+  "澪": 256307,
+  "澱": 256539,
+  "澹": 256737,
+  "濁": 256242,
+  "濘": 256861,
+  "濛": 257096,
+  "濠": 256953,
+  "濤": 256545,
+  "瀉": 256820,
+  "瀑": 256781,
+  "瀟": 256896,
+  "瀟洒": 257015,
+  "瀧": 256949,
+  "瀾": 256987,
+  "灸": 256954,
+  "炒": 256339,
+  "炙": 256429,
+  "炬": 256610,
+  "炬燵": 256720,
+  "炯": 257045,
+  "烙": 256718,
+  "烹": 256998,
+  "焔": 256393,
+  "焙": 257056,
+  "煌": 256220,
+  "煎": 256520,
+  "熔": 256800,
+  "熾": 256418,
+  "燎": 256546,
+  "燐": 256388,
+  "燕": 256536,
+  "燦": 256611,
+  "燭": 256340,
+  "燵": 256690,
+  "燻": 256360,
+  "燼": 256853,
+  "牒": 256858,
+  "牝": 256839,
+  "牡": 256777,
+  "牡蠣": 257133,
+  "狛": 257092,
+  "狡": 256516,
+  "狡猾": 256722,
+  "狢": 257160,
+  "狸": 256491,
+  "狽": 256308,
+  "猊": 256618,
+  "猥": 256557,
+  "猴": 256883,
+  "猾": 256667,
+  "獪": 256914,
+  "獰": 256362,
+  "玲": 256518,
+  "玲瓏": 257225,
+  "珈": 257181,
+  "珈琲": 256602,
+  "珪": 257048,
+  "琉": 256846,
+  "琢": 256349,
+  "琥": 256299,
+  "琲": 257158,
+  "瑕": 256850,
+  "瑕疵": 257132,
+  "瑙": 256830,
+  "瑛": 256818,
+  "瑠": 256341,
+  "瑣": 256950,
+  "瑾": 256554,
+  "璽": 257135,
+  "瓢": 257134,
+  "瓢箪": 257019,
+  "甕": 256991,
+  "畔": 256550,
+  "畝": 257168,
+  "疇": 256370,
+  "疚": 257001,
+  "疵": 257002,
+  "痍": 256488,
+  "痒": 256356,
+  "痔": 257109,
+  "痙": 256475,
+  "痙攣": 256449,
+  "痣": 256562,
+  "瘡": 257111,
+  "瘴": 256347,
+  "瘴焔": 256933,
+  "癇": 256593,
+  "癇癪": 256814,
+  "癪": 256411,
+  "皐": 256249,
+  "皙": 256874,
+  "皺": 256275,
+  "盃": 256638,
+  "盥": 256851,
+  "盪": 256890,
+  "眇": 256506,
+  "眈": 256838,
+  "眦": 256577,
+  "眩": 256222,
+  "眩暈": 256563,
+  "眸": 256258,
+  "睥": 256590,
+  "睥睨": 256754,
+  "睨": 256206,
+  "睫": 256461,
+  "睾": 257073,
+  "瞠": 256328,
+  "瞥": 256263,
+  "瞰": 256502,
+  "瞼": 256271,
+  "矜": 256385,
+  "砥": 256859,
+  "硝": 256325,
+  "碁": 256671,
+  "碧": 256334,
+  "磊": 257076,
+  "磋": 257053,
+  "磋琢": 257163,
+  "磔": 256702,
+  "磺": 257028,
+  "礫": 256274,
+  "祇": 256647,
+  "祓": 256531,
+  "祟": 256759,
+  "祠": 256498,
+  "祢": 256559,
+  "禄": 256578,
+  "禊": 256937,
+  "禎": 256952,
+  "禿": 256483,
+  "秣": 257127,
+  "秦": 256665,
+  "稜": 256757,
+  "稟": 257186,
+  "穣": 256454,
+  "穹": 256644,
+  "穽": 257047,
+  "窩": 256594,
+  "窪": 256386,
+  "窯": 256616,
+  "窶": 257091,
+  "竄": 256630,
+  "竈": 256553,
+  "竦": 256229,
+  "竪": 256929,
+  "竿": 256586,
+  "笏": 256975,
+  "笠": 256417,
+  "笥": 256989,
+  "笨": 256892,
+  "笹": 256480,
+  "笹藪": 256712,
+  "筍": 257191,
+  "筏": 256940,
+  "筐": 256749,
+  "箍": 256997,
+  "箒": 256421,
+  "箔": 256645,
+  "箕": 256824,
+  "箝": 257016,
+  "箪": 257008,
+  "箪笥": 256857,
+  "箸": 256322,
+  "篝": 256654,
+  "篠": 256420,
+  "篠窪": 257094,
+  "篩": 257207,
+  "篭": 256375,
+  "簀": 256653,
+  "簒": 256657,
+  "簞": 257221,
+  "簞笥": 257192,
+  "簪": 256467,
+  "簾": 256782,
+  "籐": 256821,
+  "籾": 257151,
+  "粟": 256533,
+  "粟楠": 257000,
+  "粥": 256470,
+  "糠": 257024,
+  "糺": 257102,
+  "紆": 256746,
+  "紡": 256261,
+  "紺": 256262,
+  "紺碧": 256878,
+  "絢": 256631,
+  "絨": 256321,
+  "絹": 256330,
+  "綜": 256732,
+  "綯": 257146,
+  "綸": 256556,
+  "綽": 256921,
+  "綾": 256208,
+  "緋": 256422,
+  "緋鯉": 256982,
+  "緘": 257182,
+  "緞": 257156,
+  "緻": 256404,
+  "縊": 256904,
+  "縋": 256281,
+  "縒": 257088,
+  "縞": 256666,
+  "縷": 256686,
+  "縺": 256961,
+  "繍": 256327,
+  "繕": 256253,
+  "繡": 256683,
+  "繭": 256664,
+  "纂": 257004,
+  "缝": 257189,
+  "缪": 256676,
+  "罅": 256472,
+  "羹": 257077,
+  "翠": 256279,
+  "翠苓": 256566,
+  "翡": 256524,
+  "翡翠": 256463,
+  "翳": 256457,
+  "耄": 257060,
+  "聡": 256342,
+  "聳": 256522,
+  "聾": 256985,
+  "肛": 257147,
+  "肴": 256655,
+  "胤": 256476,
+  "胱": 257058,
+  "胴": 256237,
+  "脛": 256580,
+  "腋": 257018,
+  "腑": 256326,
+  "腥": 257083,
+  "腱": 256691,
+  "膂": 256440,
+  "膳": 256423,
+  "膵": 256973,
+  "膿": 256796,
+  "臀": 256918,
+  "臍": 256815,
+  "臙": 256935,
+  "臼": 256625,
+  "舐": 256223,
+  "舫": 257100,
+  "舳": 256907,
+  "舵": 256433,
+  "舷": 256608,
+  "艶": 256239,
+  "芋": 256314,
+  "芒": 256574,
+  "芙": 256977,
+  "芙蓉": 256407,
+  "芥": 256724,
+  "芦": 256292,
+  "芹": 257129,
+  "芻": 256500,
+  "苅": 257176,
+  "苑": 256794,
+  "苓": 257086,
+  "苔": 256374,
+  "苫": 256713,
+  "苺": 256395,
+  "茅": 256588,
+  "茜": 256284,
+  "茫": 256462,
+  "茸": 256493,
+  "茹": 256448,
+  "荊": 256946,
+  "荻": 257098,
+  "荼": 257010,
+  "莢": 256585,
+  "莪": 256660,
+  "莵": 256731,
+  "菅": 256803,
+  "菊": 256273,
+  "菖蒲": 257200,
+  "菫": 256291,
+  "菱": 256842,
+  "萌": 256280,
+  "萎": 256353,
+  "萝": 256317,
+  "葦": 256994,
+  "葱": 256826,
+  "葵": 256252,
+  "葺": 257159,
+  "蒐": 256558,
+  "蒜": 257037,
+  "蒲": 257141,
+  "蓉": 256736,
+  "蓑": 256776,
+  "蓬": 256943,
+  "蔦": 256456,
+  "蔭": 256584,
+  "蕎": 256786,
+  "蕗": 256397,
+  "薔": 256278,
+  "薙": 256245,
+  "薫": 256381,
+  "藁": 256364,
+  "藪": 256528,
+  "藹": 256840,
+  "虱": 256974,
+  "蚕": 256332,
+  "蚤": 257112,
+  "蚯蚓": 256867,
+  "蛆": 256582,
+  "蛍": 256485,
+  "蛙": 256392,
+  "蛞蝓": 257162,
+  "蛟": 257154,
+  "蛭": 256727,
+  "蛸": 256882,
+  "蛹": 257155,
+  "蛾": 256677,
+  "蜃": 256679,
+  "蜥蜴": 256244,
+  "蜻蛉": 256981,
+  "蝉": 256930,
+  "蝋": 256424,
+  "蝋燭": 256534,
+  "蝎": 257085,
+  "蝗": 256632,
+  "蝸": 256597,
+  "蝿": 256902,
+  "蟄": 257128,
+  "蟠": 256797,
+  "蟬": 256725,
+  "蟷螂": 256860,
+  "蟹": 256384,
+  "蟻": 256444,
+  "蠅": 256537,
+  "蠍": 256716,
+  "蠕": 257107,
+  "蠟": 256490,
+  "蠟燭": 256738,
+  "蠱": 256680,
+  "衒": 256769,
+  "衾": 256809,
+  "袁": 257175,
+  "袂": 256863,
+  "袈": 256477,
+  "袈裟": 256367,
+  "袱": 256871,
+  "袴": 256774,
+  "裟": 256507,
+  "裾": 256243,
+  "褄": 256856,
+  "褐": 256290,
+  "褥": 257166,
+  "褪": 256390,
+  "褸": 257062,
+  "襖": 256606,
+  "襞": 257179,
+  "襤": 257061,
+  "襤褸": 256905,
+  "覯": 257171,
+  "覿": 256807,
+  "訃": 257180,
+  "訛": 256804,
+  "訥": 256837,
+  "訶": 256808,
+  "詈": 256711,
+  "詠": 256218,
+  "詣": 256509,
+  "誂": 256805,
+  "誅": 256831,
+  "誑": 256621,
+  "誼": 256674,
+  "諌": 256684,
+  "諍": 256541,
+  "諒": 257026,
+  "諤": 257190,
+  "諫": 256523,
+  "諳": 257039,
+  "諺": 256909,
+  "謁": 256293,
+  "謐": 256525,
+  "謡": 256302,
+  "譚": 256352,
+  "譫": 257041,
+  "豹": 256387,
+  "貂": 256633,
+  "賎": 256995,
+  "贋": 256452,
+  "贔": 256772,
+  "贔屓": 256596,
+  "贤": 257044,
+  "赌": 256958,
+  "赚": 257038,
+  "趙": 257020,
+  "趙迂": 256561,
+  "跋": 256699,
+  "跋扈": 256841,
+  "踝": 257197,
+  "踵": 256255,
+  "蹂": 256415,
+  "蹂躙": 256344,
+  "蹄": 256406,
+  "蹲": 256409,
+  "躓": 256555,
+  "躙": 256354,
+  "躯": 256350,
+  "躰": 257209,
+  "躱": 256283,
+  "躾": 256405,
+  "軀": 256318,
+  "軋": 256260,
+  "軋轢": 256855,
+  "軛": 257226,
+  "輌": 256825,
+  "輓": 257172,
+  "輜": 256919,
+  "輻": 257218,
+  "輿": 256540,
+  "轍": 256484,
+  "轡": 256761,
+  "轢": 256426,
+  "辣": 256343,
+  "辰": 256301,
+  "��巳": 257079,
+  "辻": 256532,
+  "辻褄": 256835,
+  "迂": 256268,
+  "迂闊": 256376,
+  "迭": 257148,
+  "迸": 256304,
+  "逅": 256682,
+  "逞": 256486,
+  "逡": 256323,
+  "逬": 257213,
+  "遁": 256617,
+  "遽": 256510,
+  "邂": 256681,
+  "邂逅": 256672,
+  "邏": 256688,
+  "鄙": 256640,
+  "鄭": 256515,
+  "酊": 257030,
+  "酋": 257185,
+  "酎": 256865,
+  "酢": 256519,
+  "酩": 257029,
+  "酩酊": 256778,
+  "醍": 256259,
+  "醍醐": 256872,
+  "醐": 256906,
+  "醤": 256434,
+  "醬": 256750,
+  "釜": 256497,
+  "鈎": 257066,
+  "鉈": 256487,
+  "鉤": 256458,
+  "鉾": 257078,
+  "銚": 256756,
+  "銛": 257040,
+  "銜": 257167,
+  "鋏": 256636,
+  "鋒": 256569,
+  "鋤": 256944,
+  "鋲": 256834,
+  "鋸": 256710,
+  "錆": 256305,
+  "錐": 256492,
+  "錐揉": 257033,
+  "錘": 256875,
+  "錦": 256877,
+  "錦鯉": 257075,
+  "錨": 256880,
+  "鍔": 256471,
+  "鍬": 256829,
+  "鍮": 256843,
+  "鍼": 256978,
+  "鎌": 256254,
+  "鎌鼬": 257113,
+  "鎚": 256400,
+  "鎧": 256207,
+  "鎧兜": 256748,
+  "鎬": 256942,
+  "鏃": 256543,
+  "鏖": 256744,
+  "鏤": 257116,
+  "鐙": 256967,
+  "鑢": 257161,
+  "鑼": 256939,
+  "鑽": 256517,
+  "閂": 256832,
+  "閏": 256464,
+  "間冴": 257174,
+  "閨": 256895,
+  "閾": 257187,
+  "闊": 256324,
+  "闖": 256791,
+  "闢": 256926,
+  "闹": 257138,
+  "阜": 256986,
+  "隈": 256481,
+  "隘": 257003,
+  "隧": 257023,
+  "隼": 256459,
+  "雁": 256403,
+  "雉": 257145,
+  "雫": 256216,
+  "雹": 256928,
+  "霆": 256701,
+  "霜": 256530,
+  "霞": 256277,
+  "霰": 257006,
+  "霹": 256970,
+  "靂": 256962,
+  "靄": 256355,
+  "靡": 256504,
+  "靭": 256402,
+  "靱": 256704,
+  "鞄": 256226,
+  "鞍": 256460,
+  "鞘": 256250,
+  "鞴": 257101,
+  "韋": 256601,
+  "韜晦": 257105,
+  "韻": 256437,
+  "頚": 256931,
+  "頰": 256209,
+  "頴": 257152,
+  "頷": 256204,
+  "頽": 256705,
+  "顎": 256217,
+  "顎髭": 257089,
+  "顎鬚": 256920,
+  "顚": 256624,
+  "顛": 256576,
+  "顰": 256266,
+  "颇": 257203,
+  "颯": 256450,
+  "颶": 257118,
+  "飄": 256474,
+  "飆": 257223,
+  "飴": 256378,
+  "餃": 256812,
+  "餉": 256651,
+  "餞": 256771,
+  "餡": 256947,
+  "饅": 256568,
+  "饐": 257210,
+  "饗": 256870,
+  "駁": 256828,
+  "駈": 257114,
+  "駿": 256401,
+  "騾": 257069,
+  "驀": 256885,
+  "驟": 257087,
+  "骰": 256957,
+  "髏": 257121,
+  "髑": 257169,
+  "髑髏": 256735,
+  "髭": 256256,
+  "髯": 257144,
+  "鬚": 256941,
+  "鬣": 256945,
+  "鬨": 256642,
+  "鬩": 257170,
+  "魁": 256755,
+  "魃": 257124,
+  "魄": 256668,
+  "魍": 257122,
+  "魍魎": 257199,
+  "魎": 256932,
+  "魏": 256893,
+  "魏嶽": 256751,
+  "魑": 257025,
+  "魘": 257031,
+  "鮫": 256570,
+  "鮭": 256959,
+  "鯉": 256765,
+  "鯖": 257064,
+  "鯛": 257074,
+  "鰊": 256983,
+  "鰐": 256903,
+  "鰓": 257082,
+  "鰭": 256963,
+  "鰹": 257153,
+  "鰻": 256730,
+  "鱈": 256615,
+  "鱗": 256238,
+  "鲱": 256819,
+  "鳳": 256787,
+  "鳳凰": 256441,
+  "鳶": 256551,
+  "鴨": 256587,
+  "鵜": 256514,
+  "鵞絨": 257095,
+  "鵠": 256823,
+  "鵺": 256951,
+  "鶴": 256295,
+  "鷲": 256296,
+  "鷲摑": 256992,
+  "鷺": 257208,
+  "鸚鵡": 257103,
+  "鹵": 256811,
+  "鹸": 256380,
+  "鹼": 256898,
+  "麒麟": 257055,
+  "麓": 256427,
+  "麹": 257104,
+  "麺": 256432,
+  "麺麭": 257093,
+  "麾": 256900,
+  "黛": 256922,
+  "黴": 256915,
+  "鼬": 257224,
+  "鼾": 257120,
+  "齎": 256643,
+  "齟": 256766,
+  "齟齬": 256852,
+  "齧": 256413,
+  "齬": 256795,
+  "𠮟": 256948
+}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "llm-ft-lightnovels/nllb-1b3-lnft2-final",
+  "_name_or_path": "nllb-1b3-lnft2.4/final",
   "activation_dropout": 0.0,
   "activation_function": "relu",
   "architectures": [
@@ -28,7 +28,7 @@
   "pad_token_id": 1,
   "scale_embedding": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.22.0.dev0",
+  "transformers_version": "4.25.1",
   "use_cache": false,
-  "vocab_size": 256206
+  "vocab_size": 257241
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85ed1617966b9c9ea0ce5d260e5a172164f72f0880c9c6ac12220bf689901456
-size 5482902982
+oid sha256:10ab3397ca2e368ed8060607113e62a05b4c7140caf2c4abd4c94a6bae94ad41
+size 5495548056

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a0054af1fd37599939637d51108be192eef687e6e370ccaeaa92c773f394fb0
-size 17331294
+oid sha256:73176d471f0de64785a4c076e56b7b5ca3b4bdaa1bd8a252f8ccad6aa0b112e2
+size 17518141

tokenizer_config.json

@@ -12,13 +12,13 @@
     "single_word": false
   },
   "model_max_length": 1024,
-  "name_or_path": "llm-ft-lightnovels/nllb-1b3-lnft2-final",
+  "name_or_path": "facebook/nllb-200-distilled-600M",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "sp_model_kwargs": {},
   "special_tokens_map_file": null,
   "src_lang": "jpn_Jpan",
   "tgt_lang": "eng_Latn",
-  "tokenizer_class": "NllbTokenizer",
+  "tokenizer_class": "TrainingNllbTokenizer",
   "unk_token": "<unk>"
 }