Update files/functions.py

files/functions.py

@@ -52,15 +52,21 @@ import pytesseract
 
 ## model / feature extractor / tokenizer
 
-from transformers import LayoutLMv2ForTokenClassification # LayoutXLMTokenizerFast, 
-
 import torch
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-# model
-# tokenizer = LayoutXLMTokenizerFast.from_pretrained(model_id)
-model = LayoutLMv2ForTokenClassification.from_pretrained(model_id);
-model.to(device);
+# model 1
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+model_id = "pierreguillou/lilt-xlm-roberta-base-finetuned-with-DocLayNet-base-at-linelevel-ml384"
+tokenizer1 = AutoTokenizer.from_pretrained(model_id)
+model1 = AutoModelForTokenClassification.from_pretrained(model_id);
+model1.to(device);
+
+from transformers import LayoutLMv2ForTokenClassification 
+# model 2
+model_id = "pierreguillou/layout-xlm-base-finetuned-with-DocLayNet-base-at-linelevel-ml384"
+model2 = LayoutLMv2ForTokenClassification.from_pretrained(model_id);
+model2.to(device);
 
 # feature extractor
 from transformers import LayoutLMv2FeatureExtractor
@@ -68,7 +74,8 @@ feature_extractor = LayoutLMv2FeatureExtractor(apply_ocr=False)
 
 # tokenizer
 from transformers import AutoTokenizer
-tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)
+tokenizer_id = "xlm-roberta-base"
+tokenizer2 = AutoTokenizer.from_pretrained(tokenizer_id)
 
 ## Key parameters