fixed bug for Mistral

llm_toolkit/llm_utils.py

@@ -90,6 +90,8 @@ def load_model(
         tokenizer.add_special_tokens({"pad_token": "<pad>"})
         model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=32)
 
+    model.generation_config.pad_token_id = tokenizer.pad_token_id
+
     return model, tokenizer
 
 

llm_toolkit/translation_utils.py

@@ -138,8 +138,8 @@ def load_translation_dataset(data_path, tokenizer=None):
 
             model_name = os.getenv("MODEL_NAME")
 
-            if "mistral" in model_name.lower():
-                messages = messages[1:]
+            # if "mistral" in model_name.lower():
+            # messages = messages[1:]
 
             texts = []
             prompts = []

notebooks/00a_Data Analysis_greedy_decoding.ipynb

@@ -0,0 +1 @@
+{"cells":[{"cell_type":"code","execution_count":10,"metadata":{"executionInfo":{"elapsed":476,"status":"ok","timestamp":1720679526275,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"uWKRSV6eZsCn"},"outputs":[{"name":"stdout","output_type":"stream","text":["The autoreload extension is already loaded. To reload it, use:\n","  %reload_ext autoreload\n"]}],"source":["%load_ext autoreload\n","%autoreload 2"]},{"cell_type":"code","execution_count":11,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"6d394937-6c99-4a7c-9d32-7600a280032f","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"G5pNu3zgZBrL","outputId":"160a554f-fb08-4aa0-bc00-0422fb7c1fac"},"outputs":[{"name":"stdout","output_type":"stream","text":["workding dir: /Users/inflaton/code/engd/papers/rapget-translation\n"]}],"source":["import os\n","import sys\n","from pathlib import Path\n","\n","# check if workding_dir is in local variables\n","if \"workding_dir\" not in locals():\n","    workding_dir = str(Path.cwd().parent)\n","\n","os.chdir(workding_dir)\n","sys.path.append(workding_dir)\n","print(\"workding dir:\", workding_dir)"]},{"cell_type":"code","execution_count":12,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"9f67ec60-2f24-411c-84eb-0dd664b44775","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"hPCC-6m7ZBrM","outputId":"c7aa2c96-5e99-440a-c148-201d79465ff9"},"outputs":[{"name":"stdout","output_type":"stream","text":["loading env vars from: /Users/inflaton/code/engd/papers/rapget-translation/.env\n"]},{"data":{"text/plain":["True"]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["from dotenv import find_dotenv, load_dotenv\n","\n","found_dotenv = find_dotenv(\".env\")\n","\n","if len(found_dotenv) == 0:\n","    found_dotenv = find_dotenv(\".env.example\")\n","print(f\"loading env vars from: {found_dotenv}\")\n","load_dotenv(found_dotenv, override=True)"]},{"cell_type":"code","execution_count":13,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"f1597656-8042-4878-9d3b-9ebfb8dd86dc","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1720679529345,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"1M3IraVtZBrM","outputId":"29ab35f6-2970-4ade-d85d-3174acf8cda0"},"outputs":[{"name":"stdout","output_type":"stream","text":["Qwen/Qwen2-7B-Instruct None False datasets/mac/mac.tsv results/mac-results_greedy_decoding.csv\n"]}],"source":["import os\n","\n","model_name = os.getenv(\"MODEL_NAME\")\n","adapter_name_or_path = os.getenv(\"ADAPTER_NAME_OR_PATH\")\n","load_in_4bit = os.getenv(\"LOAD_IN_4BIT\") == \"true\"\n","data_path = os.getenv(\"DATA_PATH\")\n","results_path = \"results/mac-results_greedy_decoding.csv\"  # os.getenv(\"RESULTS_PATH\")\n","use_english_datasets = os.getenv(\"USE_ENGLISH_DATASETS\") == \"true\"\n","\n","print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)"]},{"cell_type":"code","execution_count":14,"metadata":{"application/vnd.databricks.v1+cell":{"cellMetadata":{"byteLimit":2048000,"rowLimit":10000},"inputWidgets":{},"nuid":"b2a43943-9324-4839-9a47-cfa72de2244b","showTitle":false,"title":""},"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":564,"status":"ok","timestamp":1720679529907,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"UgMvt6dIZBrM","outputId":"ce37581c-fd26-46c2-ad87-d933d99f68f7"},"outputs":[{"name":"stdout","output_type":"stream","text":["Python 3.11.9\n","Name: torch\n","Version: 2.4.0\n","Summary: Tensors and Dynamic neural networks in Python with strong GPU acceleration\n","Home-page: https://pytorch.org/\n","Author: PyTorch Team\n","Author-email: packages@pytorch.org\n","License: BSD-3\n","Location: /Users/inflaton/anaconda3/envs/rapget/lib/python3.11/site-packages\n","Requires: filelock, fsspec, jinja2, networkx, sympy, typing-extensions\n","Required-by: accelerate, peft, torchaudio, torchvision\n","---\n","Name: transformers\n","Version: 4.43.3\n","Summary: State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow\n","Home-page: https://github.com/huggingface/transformers\n","Author: The Hugging Face team (past and future) with the help of all our contributors (https://github.com/huggingface/transformers/graphs/contributors)\n","Author-email: transformers@huggingface.co\n","License: Apache 2.0 License\n","Location: /Users/inflaton/anaconda3/envs/rapget/lib/python3.11/site-packages\n","Requires: filelock, huggingface-hub, numpy, packaging, pyyaml, regex, requests, safetensors, tokenizers, tqdm\n","Required-by: peft\n","CPU times: user 7.39 ms, sys: 11.5 ms, total: 18.9 ms\n","Wall time: 1.73 s\n"]}],"source":["%%time\n","os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n","\n","!python --version\n","!pip show torch transformers"]},{"cell_type":"code","execution_count":15,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":1685,"status":"ok","timestamp":1720679531591,"user":{"displayName":"HUANG DONGHAO _","userId":"00977795705617022768"},"user_tz":-480},"id":"ZuS_FsLyZBrN","outputId":"2cba0105-c505-4395-afbd-2f2fee6581d0"},"outputs":[{"name":"stdout","output_type":"stream","text":["MPS is available\n"]}],"source":["from llm_toolkit.llm_utils import *\n","from llm_toolkit.translation_utils import *\n","\n","device = check_gpu()"]},{"cell_type":"code","execution_count":16,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 1133 entries, 0 to 1132\n","Data columns (total 25 columns):\n"," #   Column                                           Non-Null Count  Dtype \n","---  ------                                           --------------  ----- \n"," 0   chinese                                          1133 non-null   object\n"," 1   english                                          1133 non-null   object\n"," 2   Qwen/Qwen2-7B-Instruct/rpp-1.00                  1133 non-null   object\n"," 3   Qwen/Qwen2-7B-Instruct/rpp-1.02                  1133 non-null   object\n"," 4   Qwen/Qwen2-7B-Instruct/rpp-1.04                  1133 non-null   object\n"," 5   Qwen/Qwen2-7B-Instruct/rpp-1.06                  1133 non-null   object\n"," 6   Qwen/Qwen2-7B-Instruct/rpp-1.08                  1133 non-null   object\n"," 7   Qwen/Qwen2-7B-Instruct/rpp-1.10                  1133 non-null   object\n"," 8   Qwen/Qwen2-7B-Instruct/rpp-1.12                  1133 non-null   object\n"," 9   Qwen/Qwen2-7B-Instruct/rpp-1.14                  1133 non-null   object\n"," 10  Qwen/Qwen2-7B-Instruct/rpp-1.16                  1133 non-null   object\n"," 11  Qwen/Qwen2-7B-Instruct/rpp-1.18                  1133 non-null   object\n"," 12  Qwen/Qwen2-7B-Instruct/rpp-1.20                  1133 non-null   object\n"," 13  Qwen/Qwen2-7B-Instruct/rpp-1.22                  1133 non-null   object\n"," 14  Qwen/Qwen2-7B-Instruct/rpp-1.24                  1133 non-null   object\n"," 15  Qwen/Qwen2-7B-Instruct/rpp-1.26                  1133 non-null   object\n"," 16  Qwen/Qwen2-7B-Instruct/rpp-1.28                  1133 non-null   object\n"," 17  Qwen/Qwen2-7B-Instruct/rpp-1.30                  1133 non-null   object\n"," 18  internlm/internlm2_5-7b-chat-1m/rpp-1.00         1133 non-null   object\n"," 19  internlm/internlm2_5-7b-chat-1m/rpp-1.02         1133 non-null   object\n"," 20  Qwen/Qwen2-72B-Instruct/rpp-1.00                 1133 non-null   object\n"," 21  shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.00  1133 non-null   object\n"," 22  shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.02  1133 non-null   object\n"," 23  shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.04  1133 non-null   object\n"," 24  shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.06  1133 non-null   object\n","dtypes: object(25)\n","memory usage: 221.4+ KB\n"]}],"source":["import pandas as pd\n","\n","df = pd.read_csv(results_path)\n","df.info()"]},{"cell_type":"code","execution_count":18,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Qwen/Qwen2-7B-Instruct/rpp-1.00: {'meteor': 0.3757937058055942, 'bleu_scores': {'bleu': 0.11257687997946404, 'precisions': [0.4221057489451477, 0.15152552819915763, 0.07046669041681511, 0.03563738956121464], 'brevity_penalty': 1.0, 'length_ratio': 1.004836038423319, 'translation_length': 30336, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.42398582083049496, 'rouge2': 0.17397502646147225, 'rougeL': 0.364447549509206, 'rougeLsum': 0.36430686603130313}, 'accuracy': 0.00088261253309797, 'correct_ids': [364]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.02: {'meteor': 0.3768162203335968, 'bleu_scores': {'bleu': 0.11553860771639841, 'precisions': [0.421923611570795, 0.15446511467968776, 0.07288535852297123, 0.03751491646778043], 'brevity_penalty': 1.0, 'length_ratio': 1.0007949652202717, 'translation_length': 30214, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.42716298071770475, 'rouge2': 0.17952709653204155, 'rougeL': 0.3693774755784234, 'rougeLsum': 0.3693387884292203}, 'accuracy': 0.00176522506619594, 'correct_ids': [364, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.04: {'meteor': 0.3715147429622351, 'bleu_scores': {'bleu': 0.11311605625702598, 'precisions': [0.41758205508824014, 0.15180590775135358, 0.07144639737602053, 0.036148159155923766], 'brevity_penalty': 1.0, 'length_ratio': 1.0041404438555812, 'translation_length': 30315, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.41883378037175867, 'rouge2': 0.17369723843307666, 'rougeL': 0.3635071379933642, 'rougeLsum': 0.3633620898391615}, 'accuracy': 0.00264783759929391, 'correct_ids': [240, 364, 533]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.06: {'meteor': 0.3721614566005243, 'bleu_scores': {'bleu': 0.10986034422062402, 'precisions': [0.41770767752410615, 0.14848860428286167, 0.06846272346218608, 0.03435399551904406], 'brevity_penalty': 0.9996355745538857, 'length_ratio': 0.9996356409407089, 'translation_length': 30179, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4184198489487805, 'rouge2': 0.17057728266140748, 'rougeL': 0.36097843223543047, 'rougeLsum': 0.36075561041667037}, 'accuracy': 0.00264783759929391, 'correct_ids': [240, 364, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.08: {'meteor': 0.3712966405354824, 'bleu_scores': {'bleu': 0.10809530671609749, 'precisions': [0.41541684679591634, 0.14717672264842077, 0.06768566804531559, 0.033518296340731855], 'brevity_penalty': 0.9960505187431468, 'length_ratio': 0.9960582974494866, 'translation_length': 30071, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4183654239604653, 'rouge2': 0.17226686931692942, 'rougeL': 0.3595185974188336, 'rougeLsum': 0.3594858567645154}, 'accuracy': 0.00264783759929391, 'correct_ids': [240, 364, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.10: {'meteor': 0.3713527017404089, 'bleu_scores': {'bleu': 0.10809698094017595, 'precisions': [0.4147023571713943, 0.145728817077812, 0.06795102628736047, 0.03393775575327552], 'brevity_penalty': 0.9948859408394681, 'length_ratio': 0.9948989731699238, 'translation_length': 30036, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4176394858436876, 'rouge2': 0.17211294909066005, 'rougeL': 0.3594913529676633, 'rougeLsum': 0.35931511942666405}, 'accuracy': 0.00264783759929391, 'correct_ids': [364, 533, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.12: {'meteor': 0.36820419885143935, 'bleu_scores': {'bleu': 0.10505573355971856, 'precisions': [0.4098240955857949, 0.14277339035072595, 0.06492248062015504, 0.03232202311922487], 'brevity_penalty': 0.9980106107363413, 'length_ratio': 0.9980125869493209, 'translation_length': 30130, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.41546245333232956, 'rouge2': 0.16831623851349498, 'rougeL': 0.35882706056796865, 'rougeLsum': 0.3582240777128928}, 'accuracy': 0.00176522506619594, 'correct_ids': [364, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.14: {'meteor': 0.36307746488229864, 'bleu_scores': {'bleu': 0.10051614663163566, 'precisions': [0.4013952416992991, 0.13692917692097348, 0.06165771788216051, 0.030122267506483884], 'brevity_penalty': 1.0, 'length_ratio': 1.0065915866180855, 'translation_length': 30389, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4081754711991447, 'rouge2': 0.16174347273683704, 'rougeL': 0.3496505103802724, 'rougeLsum': 0.3495083811992814}, 'accuracy': 0.00264783759929391, 'correct_ids': [240, 364, 658]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.16: {'meteor': 0.36060381551154586, 'bleu_scores': {'bleu': 0.09572351387840275, 'precisions': [0.3943648240226187, 0.13195897159052566, 0.05795474478161726, 0.027838667251205613], 'brevity_penalty': 1.0, 'length_ratio': 1.019244783040742, 'translation_length': 30771, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.40692843140324875, 'rouge2': 0.16073905715363146, 'rougeL': 0.3505553548589415, 'rougeLsum': 0.3504151270630069}, 'accuracy': 0.00176522506619594, 'correct_ids': [364, 533]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.18: {'meteor': 0.36078545841521914, 'bleu_scores': {'bleu': 0.09571300097111912, 'precisions': [0.3949360480292352, 0.13088260206674573, 0.05813543795363258, 0.027927630371756763], 'brevity_penalty': 1.0, 'length_ratio': 1.0151705862868499, 'translation_length': 30648, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.404993454925694, 'rouge2': 0.15801114156213147, 'rougeL': 0.34414503988984757, 'rougeLsum': 0.3442205706273794}, 'accuracy': 0.00264783759929391, 'correct_ids': [240, 364, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.20: {'meteor': 0.3567548354175595, 'bleu_scores': {'bleu': 0.0912485469982839, 'precisions': [0.3872236189002772, 0.12631719800622218, 0.05570236439499304, 0.025445200521210368], 'brevity_penalty': 1.0, 'length_ratio': 1.0276912885061278, 'translation_length': 31026, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4039670988772276, 'rouge2': 0.15731351800856888, 'rougeL': 0.34455475680525216, 'rougeLsum': 0.34447802187069654}, 'accuracy': 0.00176522506619594, 'correct_ids': [364, 658]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.22: {'meteor': 0.3510044718361491, 'bleu_scores': {'bleu': 0.08350689777294566, 'precisions': [0.3702997530709843, 0.11766040181464679, 0.050021865644027316, 0.02231237322515213], 'brevity_penalty': 1.0, 'length_ratio': 1.059721762172905, 'translation_length': 31993, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.39607308776257855, 'rouge2': 0.15304260749559023, 'rougeL': 0.3369480429607284, 'rougeLsum': 0.3372136099554942}, 'accuracy': 0.00264783759929391, 'correct_ids': [364, 658, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.24: {'meteor': 0.3465600044661264, 'bleu_scores': {'bleu': 0.07954262823239741, 'precisions': [0.3656348982343902, 0.11231059374390323, 0.04652104925559569, 0.020954720954720955], 'brevity_penalty': 1.0, 'length_ratio': 1.056210665783372, 'translation_length': 31887, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3937695251160657, 'rouge2': 0.1489191506658582, 'rougeL': 0.3334167496962085, 'rougeLsum': 0.3332312145948097}, 'accuracy': 0.00264783759929391, 'correct_ids': [364, 658, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.26: {'meteor': 0.3435165661403993, 'bleu_scores': {'bleu': 0.07858780987337025, 'precisions': [0.35780525502318394, 0.1090751833936637, 0.04563887780880202, 0.02141475545730865], 'brevity_penalty': 1.0, 'length_ratio': 1.0715468698244452, 'translation_length': 32350, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.3903804895791132, 'rouge2': 0.14817852121497105, 'rougeL': 0.3301363834057721, 'rougeLsum': 0.3304187467803172}, 'accuracy': 0.00353045013239188, 'correct_ids': [240, 364, 658, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.28: {'meteor': 0.34053363547339577, 'bleu_scores': {'bleu': 0.07203840378380885, 'precisions': [0.3451020592757862, 0.10142348754448399, 0.0418756674541277, 0.018374202216996975], 'brevity_penalty': 1.0, 'length_ratio': 1.0986088108645247, 'translation_length': 33167, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.38637262142257134, 'rouge2': 0.14316756859492308, 'rougeL': 0.32596401712525114, 'rougeLsum': 0.32697850887033386}, 'accuracy': 0.00264783759929391, 'correct_ids': [364, 658, 659]}\n","Qwen/Qwen2-7B-Instruct/rpp-1.30: {'meteor': 0.33446931317267503, 'bleu_scores': {'bleu': 0.062148408497464926, 'precisions': [0.3152004454342984, 0.08905625664759824, 0.035419266654781005, 0.015004765858008178], 'brevity_penalty': 1.0, 'length_ratio': 1.1897979463398476, 'translation_length': 35920, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.382440527497649, 'rouge2': 0.14278568056233967, 'rougeL': 0.32198406741272634, 'rougeLsum': 0.3222889578426552}, 'accuracy': 0.00264783759929391, 'correct_ids': [240, 364, 659]}\n","internlm/internlm2_5-7b-chat-1m/rpp-1.00: {'meteor': 0.3715346402699926, 'bleu_scores': {'bleu': 0.1059772684959813, 'precisions': [0.39683339104158144, 0.1431975453714584, 0.06656950140663662, 0.03334508283397956], 'brevity_penalty': 1.0, 'length_ratio': 1.0523020867837032, 'translation_length': 31769, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4198375474164172, 'rouge2': 0.170026985455455, 'rougeL': 0.36316099332481133, 'rougeLsum': 0.3632892543893713}, 'accuracy': 0.0, 'correct_ids': []}\n","internlm/internlm2_5-7b-chat-1m/rpp-1.02: {'meteor': 0.352901317633597, 'bleu_scores': {'bleu': 0.08697903417673139, 'precisions': [0.3666595931730682, 0.11979657185910718, 0.05260074213918365, 0.024771882392700235], 'brevity_penalty': 1.0, 'length_ratio': 1.0926465717124876, 'translation_length': 32987, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.39949551448928944, 'rouge2': 0.1514617059581041, 'rougeL': 0.3397137691185899, 'rougeLsum': 0.3410587722727241}, 'accuracy': 0.00088261253309797, 'correct_ids': [511]}\n","Qwen/Qwen2-72B-Instruct/rpp-1.00: {'meteor': 0.39496912014495184, 'bleu_scores': {'bleu': 0.12294894050451377, 'precisions': [0.42391407360606537, 0.1626695498329074, 0.079349416448331, 0.041761041902604754], 'brevity_penalty': 1.0, 'length_ratio': 1.048526001987413, 'translation_length': 31655, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4427657230030544, 'rouge2': 0.19160274619019976, 'rougeL': 0.38399413150314754, 'rougeLsum': 0.3840398064172499}, 'accuracy': 0.0, 'correct_ids': []}\n","shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.00: {'meteor': 0.38168584246814397, 'bleu_scores': {'bleu': 0.11518296996672078, 'precisions': [0.42672642762284196, 0.15593196950357058, 0.07280560043080236, 0.036672529281892005], 'brevity_penalty': 0.9976786612989592, 'length_ratio': 0.9976813514408744, 'translation_length': 30120, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.425030070021637, 'rouge2': 0.17614767100972545, 'rougeL': 0.3704736516621092, 'rougeLsum': 0.370091658387699}, 'accuracy': 0.00176522506619594, 'correct_ids': [77, 531]}\n","shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.02: {'meteor': 0.381084663579427, 'bleu_scores': {'bleu': 0.11434064727385712, 'precisions': [0.42645298576938423, 0.15516705248246554, 0.07212973283952392, 0.03635818433974287], 'brevity_penalty': 0.996216776830359, 'length_ratio': 0.9962239152037098, 'translation_length': 30076, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.42429799890968667, 'rouge2': 0.17527380455760494, 'rougeL': 0.36974142291224055, 'rougeLsum': 0.3694755062707584}, 'accuracy': 0.00176522506619594, 'correct_ids': [77, 531]}\n","shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.04: {'meteor': 0.38019108433175514, 'bleu_scores': {'bleu': 0.11353152954579881, 'precisions': [0.42572246637368494, 0.15441303670899215, 0.0716574844262, 0.03599984984421337], 'brevity_penalty': 0.9948859408394681, 'length_ratio': 0.9948989731699238, 'translation_length': 30036, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.42368222183126075, 'rouge2': 0.17486315604625235, 'rougeL': 0.36929728545856716, 'rougeLsum': 0.36885053449143124}, 'accuracy': 0.00176522506619594, 'correct_ids': [77, 531]}\n","shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.06: {'meteor': 0.37862157681270814, 'bleu_scores': {'bleu': 0.11220469680226439, 'precisions': [0.42524207011686144, 0.15293056182114723, 0.07094094274878093, 0.03547621737656762], 'brevity_penalty': 0.9920186657513808, 'length_ratio': 0.9920503477972838, 'translation_length': 29950, 'reference_length': 30190}, 'rouge_scores': {'rouge1': 0.4233090909108499, 'rouge2': 0.17383021735666143, 'rougeL': 0.3687369102186561, 'rougeLsum': 0.3683741409468205}, 'accuracy': 0.00176522506619594, 'correct_ids': [77, 531]}\n"]},{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>model</th>\n","      <th>rpp</th>\n","      <th>meteor</th>\n","      <th>bleu_1</th>\n","      <th>rouge_l</th>\n","      <th>ews_score</th>\n","      <th>repetition_score</th>\n","      <th>total_repetitions</th>\n","      <th>num_entries_with_max_output_tokens</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.00</td>\n","      <td>0.375794</td>\n","      <td>0.112577</td>\n","      <td>0.364448</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.02</td>\n","      <td>0.376816</td>\n","      <td>0.115539</td>\n","      <td>0.369377</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.04</td>\n","      <td>0.371515</td>\n","      <td>0.113116</td>\n","      <td>0.363507</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.06</td>\n","      <td>0.372161</td>\n","      <td>0.109860</td>\n","      <td>0.360978</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.08</td>\n","      <td>0.371297</td>\n","      <td>0.108095</td>\n","      <td>0.359519</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.10</td>\n","      <td>0.371353</td>\n","      <td>0.108097</td>\n","      <td>0.359491</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.12</td>\n","      <td>0.368204</td>\n","      <td>0.105056</td>\n","      <td>0.358827</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.14</td>\n","      <td>0.363077</td>\n","      <td>0.100516</td>\n","      <td>0.349651</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.16</td>\n","      <td>0.360604</td>\n","      <td>0.095724</td>\n","      <td>0.350555</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.18</td>\n","      <td>0.360785</td>\n","      <td>0.095713</td>\n","      <td>0.344145</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.20</td>\n","      <td>0.356755</td>\n","      <td>0.091249</td>\n","      <td>0.344555</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.22</td>\n","      <td>0.351004</td>\n","      <td>0.083507</td>\n","      <td>0.336948</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.24</td>\n","      <td>0.346560</td>\n","      <td>0.079543</td>\n","      <td>0.333417</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.26</td>\n","      <td>0.343517</td>\n","      <td>0.078588</td>\n","      <td>0.330136</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.28</td>\n","      <td>0.340534</td>\n","      <td>0.072038</td>\n","      <td>0.325964</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>Qwen/Qwen2-7B-Instruct</td>\n","      <td>1.30</td>\n","      <td>0.334469</td>\n","      <td>0.062148</td>\n","      <td>0.321984</td>\n","      <td>0.005296</td>\n","      <td>0.005296</td>\n","      <td>0.005296</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>internlm/internlm2_5-7b-chat-1m</td>\n","      <td>1.00</td>\n","      <td>0.371535</td>\n","      <td>0.105977</td>\n","      <td>0.363161</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>internlm/internlm2_5-7b-chat-1m</td>\n","      <td>1.02</td>\n","      <td>0.352901</td>\n","      <td>0.086979</td>\n","      <td>0.339714</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>Qwen/Qwen2-72B-Instruct</td>\n","      <td>1.00</td>\n","      <td>0.394969</td>\n","      <td>0.122949</td>\n","      <td>0.383994</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>shenzhi-wang/Llama3.1-70B-Chinese-Chat</td>\n","      <td>1.00</td>\n","      <td>0.381686</td>\n","      <td>0.115183</td>\n","      <td>0.370474</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>shenzhi-wang/Llama3.1-70B-Chinese-Chat</td>\n","      <td>1.02</td>\n","      <td>0.381085</td>\n","      <td>0.114341</td>\n","      <td>0.369741</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>shenzhi-wang/Llama3.1-70B-Chinese-Chat</td>\n","      <td>1.04</td>\n","      <td>0.380191</td>\n","      <td>0.113532</td>\n","      <td>0.369297</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>shenzhi-wang/Llama3.1-70B-Chinese-Chat</td>\n","      <td>1.06</td>\n","      <td>0.378622</td>\n","      <td>0.112205</td>\n","      <td>0.368737</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0.000000</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                     model   rpp    meteor    bleu_1  \\\n","0                   Qwen/Qwen2-7B-Instruct  1.00  0.375794  0.112577   \n","1                   Qwen/Qwen2-7B-Instruct  1.02  0.376816  0.115539   \n","2                   Qwen/Qwen2-7B-Instruct  1.04  0.371515  0.113116   \n","3                   Qwen/Qwen2-7B-Instruct  1.06  0.372161  0.109860   \n","4                   Qwen/Qwen2-7B-Instruct  1.08  0.371297  0.108095   \n","5                   Qwen/Qwen2-7B-Instruct  1.10  0.371353  0.108097   \n","6                   Qwen/Qwen2-7B-Instruct  1.12  0.368204  0.105056   \n","7                   Qwen/Qwen2-7B-Instruct  1.14  0.363077  0.100516   \n","8                   Qwen/Qwen2-7B-Instruct  1.16  0.360604  0.095724   \n","9                   Qwen/Qwen2-7B-Instruct  1.18  0.360785  0.095713   \n","10                  Qwen/Qwen2-7B-Instruct  1.20  0.356755  0.091249   \n","11                  Qwen/Qwen2-7B-Instruct  1.22  0.351004  0.083507   \n","12                  Qwen/Qwen2-7B-Instruct  1.24  0.346560  0.079543   \n","13                  Qwen/Qwen2-7B-Instruct  1.26  0.343517  0.078588   \n","14                  Qwen/Qwen2-7B-Instruct  1.28  0.340534  0.072038   \n","15                  Qwen/Qwen2-7B-Instruct  1.30  0.334469  0.062148   \n","16         internlm/internlm2_5-7b-chat-1m  1.00  0.371535  0.105977   \n","17         internlm/internlm2_5-7b-chat-1m  1.02  0.352901  0.086979   \n","18                 Qwen/Qwen2-72B-Instruct  1.00  0.394969  0.122949   \n","19  shenzhi-wang/Llama3.1-70B-Chinese-Chat  1.00  0.381686  0.115183   \n","20  shenzhi-wang/Llama3.1-70B-Chinese-Chat  1.02  0.381085  0.114341   \n","21  shenzhi-wang/Llama3.1-70B-Chinese-Chat  1.04  0.380191  0.113532   \n","22  shenzhi-wang/Llama3.1-70B-Chinese-Chat  1.06  0.378622  0.112205   \n","\n","     rouge_l  ews_score  repetition_score  total_repetitions  \\\n","0   0.364448   0.000000          0.000000           0.000000   \n","1   0.369377   0.000000          0.000000           0.000000   \n","2   0.363507   0.000000          0.000000           0.000000   \n","3   0.360978   0.000000          0.000000           0.000000   \n","4   0.359519   0.000000          0.000000           0.000000   \n","5   0.359491   0.000000          0.000000           0.000000   \n","6   0.358827   0.000000          0.000000           0.000000   \n","7   0.349651   0.000000          0.000000           0.000000   \n","8   0.350555   0.000000          0.000000           0.000000   \n","9   0.344145   0.000000          0.000000           0.000000   \n","10  0.344555   0.000000          0.000000           0.000000   \n","11  0.336948   0.000000          0.000000           0.000000   \n","12  0.333417   0.000000          0.000000           0.000000   \n","13  0.330136   0.000000          0.000000           0.000000   \n","14  0.325964   0.000000          0.000000           0.000000   \n","15  0.321984   0.005296          0.005296           0.005296   \n","16  0.363161   0.000000          0.000000           0.000000   \n","17  0.339714   0.000000          0.000000           0.000000   \n","18  0.383994   0.000000          0.000000           0.000000   \n","19  0.370474   0.000000          0.000000           0.000000   \n","20  0.369741   0.000000          0.000000           0.000000   \n","21  0.369297   0.000000          0.000000           0.000000   \n","22  0.368737   0.000000          0.000000           0.000000   \n","\n","    num_entries_with_max_output_tokens  \n","0                                    0  \n","1                                    0  \n","2                                    0  \n","3                                    0  \n","4                                    0  \n","5                                    0  \n","6                                    0  \n","7                                    0  \n","8                                    0  \n","9                                    0  \n","10                                   0  \n","11                                   0  \n","12                                   0  \n","13                                   0  \n","14                                   0  \n","15                                   1  \n","16                                   0  \n","17                                   0  \n","18                                   0  \n","19                                   0  \n","20                                   0  \n","21                                   0  \n","22                                   0  "]},"execution_count":18,"metadata":{},"output_type":"execute_result"}],"source":["metrics_df = get_metrics(df)\n","metrics_df"]},{"cell_type":"code","execution_count":19,"metadata":{},"outputs":[],"source":["tokenizers = {\n","    model: load_tokenizer(model) for model in metrics_df[\"model\"].unique()\n","}"]},{"cell_type":"code","execution_count":20,"metadata":{},"outputs":[],"source":["col = \"Qwen/Qwen2-7B-Instruct/rpp-1.30\"\n","df[[\"ews_score\", \"repetition_score\", \"total_repetitions\"]] = df[col].apply(\n","    detect_scores\n",")\n","df[\"output_tokens\"] = df[col].apply(\n","    lambda x: len(tokenizers[col.split(\"/rpp\")[0]](x)[\"input_ids\"])\n",")\n"]},{"cell_type":"code","execution_count":21,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>chinese</th>\n","      <th>english</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.00</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.02</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.04</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.06</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.08</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.10</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.12</th>\n","      <th>Qwen/Qwen2-7B-Instruct/rpp-1.14</th>\n","      <th>...</th>\n","      <th>internlm/internlm2_5-7b-chat-1m/rpp-1.02</th>\n","      <th>Qwen/Qwen2-72B-Instruct/rpp-1.00</th>\n","      <th>shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.00</th>\n","      <th>shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.02</th>\n","      <th>shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.04</th>\n","      <th>shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.06</th>\n","      <th>ews_score</th>\n","      <th>repetition_score</th>\n","      <th>total_repetitions</th>\n","      <th>output_tokens</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>531</th>\n","      <td>14.分娩</td>\n","      <td>Childbirth</td>\n","      <td>14. Labor</td>\n","      <td>14. Childbirth</td>\n","      <td>14. Childbirth</td>\n","      <td>14. Labor</td>\n","      <td>14. Labor</td>\n","      <td>14. Labor</td>\n","      <td>14. Childbirth</td>\n","      <td>14. Childbirth</td>\n","      <td>...</td>\n","      <td>Translation: Delivery\\n\\nThe word \"分娩\" is a co...</td>\n","      <td>14. Childbirth</td>\n","      <td>Childbirth</td>\n","      <td>Childbirth</td>\n","      <td>Childbirth</td>\n","      <td>Childbirth</td>\n","      <td>6</td>\n","      <td>42</td>\n","      <td>48</td>\n","      <td>2048</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>1 rows × 29 columns</p>\n","</div>"],"text/plain":["    chinese     english Qwen/Qwen2-7B-Instruct/rpp-1.00  \\\n","531   14.分娩  Childbirth                       14. Labor   \n","\n","    Qwen/Qwen2-7B-Instruct/rpp-1.02 Qwen/Qwen2-7B-Instruct/rpp-1.04  \\\n","531                  14. Childbirth                  14. Childbirth   \n","\n","    Qwen/Qwen2-7B-Instruct/rpp-1.06 Qwen/Qwen2-7B-Instruct/rpp-1.08  \\\n","531                       14. Labor                       14. Labor   \n","\n","    Qwen/Qwen2-7B-Instruct/rpp-1.10 Qwen/Qwen2-7B-Instruct/rpp-1.12  \\\n","531                       14. Labor                  14. Childbirth   \n","\n","    Qwen/Qwen2-7B-Instruct/rpp-1.14  ...  \\\n","531                  14. Childbirth  ...   \n","\n","              internlm/internlm2_5-7b-chat-1m/rpp-1.02  \\\n","531  Translation: Delivery\\n\\nThe word \"分娩\" is a co...   \n","\n","    Qwen/Qwen2-72B-Instruct/rpp-1.00  \\\n","531                   14. Childbirth   \n","\n","    shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.00  \\\n","531                                      Childbirth   \n","\n","    shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.02  \\\n","531                                      Childbirth   \n","\n","    shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.04  \\\n","531                                      Childbirth   \n","\n","    shenzhi-wang/Llama3.1-70B-Chinese-Chat/rpp-1.06 ews_score  \\\n","531                                      Childbirth         6   \n","\n","    repetition_score total_repetitions output_tokens  \n","531               42                48          2048  \n","\n","[1 rows x 29 columns]"]},"execution_count":21,"metadata":{},"output_type":"execute_result"}],"source":["rows = df.query(\"ews_score > 0\")\n","rows"]},{"cell_type":"code","execution_count":22,"metadata":{},"outputs":[],"source":["row = rows.iloc[0]"]},{"cell_type":"code","execution_count":23,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["14.分娩\n"]}],"source":["print(row[\"chinese\"])"]},{"cell_type":"code","execution_count":24,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Childbirth\n"]}],"source":["print(row[\"english\"])"]},{"cell_type":"code","execution_count":25,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["Title: Birth\n","\n","Translation:\n","\n","Birth is a fundamental life event that marks the transition from dependency on another organism (usually through parental care) for survival to independence as part of human society's reproductive cycle.\n","\n","The process typically involves several key stages:\n","- **Fertilization**: The fusion of sperm with egg cells inside or outside the female body leads to conception.\n","  \n","  - This can occur naturally during sexual intercourse when both partners contribute their gametes towards fertilizing each other’s eggs/ sperms respectively leading to pregnancy formation within about two weeks after ovulation if conditions allow successful implantation onto endometrial lining post-fertilisation phase where it then begins growing until birth occurs usually around nine months later due primarily biological reasons related to gestational development needed by fetus before delivery possible without significant harm inflicted upon either mother nor baby involved thereby ensuring healthy offspring capable surviving independently thereafter once born alive at full term under ideal circumstances provided no complications arise which may necessitate medical intervention depending severity & nature thereof thus prioritising maternal health alongside child welfare throughout prenatal period especially considering unique individual differences amongst pregnant women including age physical condition emotional state lifestyle choices dietary habits etcetera influencing outcomes significantly hence requiring tailored approaches encompassing comprehensive healthcare support systems designed specifically targeting specific needs based off empirical evidence clinical guidelines research studies patient feedback among others fostering optimal growth environment nurturing potential while mitigating risks associated therein aiming ultimately toward achieving best outcome feasible given current available resources technologies practices expertise knowledge base pertaining subject matter area accordingly taking necessary precautions preventing adverse effects whenever practical enhancing overall quality experience maximizing benefits minimizing harms benefiting all stakeholders concerned equally across spectrum promoting equity fairness justice accessibility affordability sustainability long-term well-being inclusive participation sustainable progress societal harmony environmental stewardship economic prosperity social cohesion strengthening community bonds enriching lives improving global public health status reducing disparities increasing access opportunities empowering individuals families communities nations regions worldwide striving together collectively towards shared goals common good collective advancement humanity improvement quality enhancement longevity extension happiness maximization satisfaction achievement equitable distribution resource utilization optimization efficient allocation leveraging advancements harness technological innovations optimizing workflows streamlining processes automating tasks enhancing productivity efficiency effectiveness safety security privacy protection personal autonomy dignity rights freedom enabling people thrive flourish reach one’s fullest potential overcome challenges obstacles adversity live meaningful purposeful fulfilling satisfying lives contributing positively making valuable contributions shaping world future generations building resilient adaptable societies thriving amidst rapid changes uncertainties embracing innovation adaptability flexibility resilience cultivating creativity critical thinking skills problem-solving abilities leadership qualities teamwork collaboration spirit empathy compassion kindness gratitude mindfulness self-awareness self-regulation lifelong learning capacity adapting evolving continuously transforming oneself society whole expanding horizons opening doors possibilities exploring uncharted territories discovering new frontiers uncovering hidden gems unlocking mysteries solving puzzles unraveling complex issues finding solutions overcoming barriers facilitating breakthroughs advancing fields pioneering discoveries setting precedents establishing norms creating value adding meaning generating ideas insights sparking imagination inspiration driving change disruption revolution transformation elevating standards excellence performance metrics benchmarks reaching milestones achievements recognizing accomplishments celebrating successes rewarding efforts acknowledging hard work dedication commitment passion pursuing dreams aspirations ambitions ideals values beliefs principles ethics morals spirituality faith hope courage perseverance patience optimism resilience determination diligence discipline foresight strategic planning execution monitoring evaluating adjusting refining iterating perfecting implementing scaling replicating disseminating sharing spreading awareness educating informing inspiring mobilizing action engaging citizens participating democracy governance decision-making policy formulation implementation evaluation accountability transparency openness inclusivity diversity equality empowerment civic engagement volunteering philanthropy activism advocacy lobbying campaigning fundraising investing supporting causes initiatives programs projects campaigns movements platforms networks ecosystems partnerships collaborations alliances synergies coalitions coordination cooperation coalition-building alliance-forming networking connecting bridging gaps closing divides uniting fragmented groups aligning interests objectives visions missions rallying behind common cause championing progressive agendas advocating policies reforms legislation regulations standards frameworks protocols procedures rules laws charters constitutions treaties agreements conventions accords memorandums understandings commitments pledges resolutions declarations endorsements certifications acknowledgments appreciations honors awards recognitions commendations congratulatory messages accolades tributes salutations greetings expressions thanks appreciation respect admiration recognition celebration acknowledgement praise congratulations felicitations gratitudes blessings wishes prayers goodwill peace love joy generosity warmth comfort relief encouragement strength wisdom guidance healing restoration balance equilibrium stability unity solidarity harmony coexistence mutual understanding acceptance tolerance forgiveness reconciliation peaceful resolution conflict prevention management deescalation diplomacy negotiation mediation arbitration conciliation settlement dispute resolution compromise concession surrender yielding acquiescence compliance agreement accommodation adjustment adaptation assimilation integration harmonious relations friendly ties cooperative partnership collaborative effort joint venture shared responsibility communitarianism collectivism altruistic behavior moral conduct ethical integrity honesty trustworthiness benevolence kindness helpfulness courtesy politeness graciousness humility modesty grace elegance sophistication refinement eloquence articulateness clarity precision brevity succinctness fluency idiomatic expression metaphors analogies symbolism imagery figures speech poetic language literary devices rhetorical strategies persuasive arguments compelling narratives powerful storytelling vivid descriptions evocative phrasing emotive tone impactful messaging influential communication effective persuasion argumentation logical reasoning sound judgment discernment insight perception intuition cognition cognitive processing analytical synthesis creative ideation divergent convergent thought reflection contemplation introspection metacognition goal-setting intentionality focus concentration motivation ambition aspiration drive desire enthusiasm zeal fervor ardor excitement anticipation eagerness willingness capability competence proficiency skillfulness mastery accomplishment success attainment realization fulfillment embodiment manifestation creation invention discovery exploration expansion enlightenment awakening liberation emancipation elevation transcendental spiritual journey soul-searching existential quest inner voyage transformative experience profound impact lasting legacy positive influence constructive contribution innovative solution addressing pressing concerns tackling major problems resolving crucial matters bringing about beneficial transformations impacting countless lives reshaping futures laying foundations paving paths forging ahead steering destinies navigating treacherous waters conquering formidable odds triumphantly emerging victorious persevering against all odds succeeding beyond expectations surpassing limitations boundaries constraints attaining heights previously unimaginable experiencing moments wonder awe amazement delight pleasure bliss ecstasy serenity calm tranquility peace joy satisfaction pride ownership control agency authority power influence effect efficacy potency versatility robustness endurance fortitude tenacity persistence vigor dynamism energy momentum surge propulsion impetus forward movement progression evolution emergence ascension culmination climax apex pinnacle zenith horizon edge boundary threshold frontier limit ceiling floor ground level ascending descending orbit trajectory path direction route destination endpoint origin return loop sequence series continuum flow rhythm pattern melody music dance choreography synchronization timing pacing tempo beat pulse resonance vibration oscillation fluctuation variation mutation divergence convergence symmetry asymmetry chaos order organization structure system architecture blueprint design layout framework outline map diagram illustration graphic representation visualization conceptual model theoretical construct mathematical equation formula algorithm procedure protocol guideline standard practice routine habit ritual custom tradition folklore myth legend story narrative poem song ballad ode epic allegory parable fable proverb riddle joke anecdote tale account record documentation report analysis assessment critique commentary review summary overview conclusion recommendation proposal plan strategy tactic approach method technique tool instrument mechanism device apparatus appliance software hardware infrastructure logistics supply chain inventory procurement finance accounting budget expenditure revenue profit loss financial statement fiscal year annual forecast projection scenario modeling simulation forecasting predicting estimating measuring calculating quantifying analyzing interpreting visualizing presenting communicating explaining clarifying answering questions providing information facts data statistics details context background explanation justification reason motive objective subjective perspective viewpoint stance attitude belief conviction certainty uncertainty risk reward trade-off opportunity cost marginal benefit incremental gain leverage advantage superiority over competition differentiation distinctiveness uniqueness novelty originality authenticity credibility reliability dependability predictability consistency coherence plausibility truth falsity validity accuracy completeness relevance timeliness availability scalability interoperability modularity extensibility portability maintainability usability learnability user-friendly intuitive simplicity complexity comprehensibility readability navigatability discoverability findabilty ease-of-use multifunctional multi-purpose versatile applicability wide-ranging scope broadening capabilities widening application domains industry sectors vertical markets horizontal markets cross-industry applications interdisciplinary field inter-disciplinary study transdisciplinary inquiry integrated solutions holistic view multidisciplinary perspectives systemic thinking interconnected networked ecosystem dynamic adaptive responsive flexible scalable modular distributed parallel computing cloud-based services platform-as-a-service infrastructure-as-a-service software-defined everything service-oriented architectures microservices containers orchestration automation code-driven deployment continuous integration testing automated QA machine learning AI deep neural nets natural language processing robotics IoT blockchain cryptography cybersecurity digital identity authentication authorization encryption decryption secure coding practices vulnerability mitigation threat detection response proactive defense cyber hygiene zero-day exploits ransomware malware phishing attacks denial-of-service DOS DDoS man-in-the-middle MITM replay attack SQL injection XSS CSRF buffer overflow rootkit trojan horse spyware adware botnet zombie node honeypot trapdoor backdoors vulnerabilities exploitation patching firmware updates patches hotfixes bug fixes upgrades maintenance operations ITIL DevOps agile methodology project management time-to-market competitive speed agility innovation experimentation iteration prototyping A/B testing split testing multivariate tests conversion rate optimization CRO landing pages bounce rates click-through-rate CTAs form submissions sales conversions customer acquisition churn retention upselling downselling product pricing marketing ROI KPIs GA tracking analytics dashboards reports business intelligence BI big-data predictive models statistical analyses econometrics simulations forecasting algorithms machine-learning models clustering classification regression anomaly-detection reinforcement-learning Q-Learning SARSA temporal difference TD active-learning semi-supervised supervised unsupervised ensemble methods random forests gradient boosting XGBoost Catboost AdaBoost naive-Bayesian SVM kernel-SVM LDA PCA ICA EM DBSCAN k-means hierarchical-clustering Gaussian-mixture-models t-Distributed Stochastic Neighbor Embedding UMAP dimension-reduction techniques feature-extraction vector-space representations word embeddings semantic similarity cosine-similarity Jaccard-index TF-IDF bag-of-words document-topic matrix factorization recommender-systems association-rules item-collaborative-filtering latent-factor-analysis sparse matrices linear-algebra numerical-methods calculus probability theory graph theory combinatorics logic formal languages programming paradigms object-oriented functional declarative imperative low-level high-level interpreted compiled just-in-time JIT virtual machines runtime environments operating systems APIs libraries frameworks modules dependencies version-control Git repositories remote-work tools team-management collaboration soft-skills technical-wrangling debugging profiling tuning benchmarking validation verification test cases smoke-tests sanity-checks load-testing stress-testing concurrency threading race-condition deadlocks livelocks starvation atomic transactions database normalization ACID properties replication sharding partitioning indexing caching NoSQL relational databases RDBMS MySQL PostgreSQL MongoDB Redis Elasticsearch Kubernetes Docker AWS Azure Google Cloud Platform GCP OpenShift Jenkins CircleCI Travis CI CodePipeline GitHub Bitbucket GitLab Atlassian Bamboo Trello Asana Jira Confluence Slack Microsoft Teams Zoom WebEx BlueJeans Hangouts Meet GoToMeeting TeamViewer Skype Outlook calendar scheduling meetings web-conferencing online-platforms videoconferencing file-sharing storage backup disaster recovery DR cloud-storage NAS SAN deduplicated replicated encrypted compressed optimized streaming real-time collaboration workflow automation task-tracking issue-tracker pull-request reviews commit-history merge\n","----detect excessive whitespaces----\n","removed excessive whitespaces: 6\n","----detect text repetitions----\n","<re.Match object; span=(7590, 7610), match='ition cognition cogn'>\n","Group 1 found at 7590-7600: `ition cogn`\n","Group 2 found at 7600-7610: `ition cogn`\n","Group 3 found at 7600-7610: `ition cogn`\n","<re.Match object; span=(13637, 13659), match='supervised supervised '>\n","Group 1 found at 13637-13648: `supervised `\n","Group 2 found at 13648-13659: `supervised `\n","Group 3 found at 13648-13659: `supervised `\n","(6, 42, 48)\n"]},{"data":{"text/plain":["(6, 42, 48)"]},"execution_count":25,"metadata":{},"output_type":"execute_result"}],"source":["output = row[\"Qwen/Qwen2-7B-Instruct/rpp-1.30\"]\n","print(row[\"Qwen/Qwen2-7B-Instruct/rpp-1.30\"])\n","detect_repetitions(output, debug=True)"]}],"metadata":{"accelerator":"GPU","application/vnd.databricks.v1+notebook":{"dashboards":[],"environmentMetadata":null,"language":"python","notebookMetadata":{"mostRecentlyExecutedCommandWithImplicitDF":{"commandId":-1,"dataframes":["_sqldf"]},"pythonIndentUnit":4},"notebookName":"10_eval-lf-medium-py3.11","widgets":{}},"colab":{"gpuType":"L4","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.9"}},"nbformat":4,"nbformat_minor":0}

results/mac-results_greedy_decoding.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:863cc947747d50db0011637b38b3cb6ca73c491bf2eb51fdd8391d23191afbe7
+oid sha256:e139f594ce8899f8e2651868965a2105ed14565eb4af07313c73071cee461c76
 size 3814560