{ "cells": [ { "cell_type": "code", "execution_count": 9, "id": "0b15c2b0-69f1-4201-a937-36bd89b97d3d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", "Requirement already satisfied: transformers in /home/ubuntu/.local/lib/python3.10/site-packages (4.40.0)\n", "Requirement already satisfied: datasets in /home/ubuntu/.local/lib/python3.10/site-packages (2.19.0)\n", "Requirement already satisfied: sentencepiece in /home/ubuntu/.local/lib/python3.10/site-packages (0.2.0)\n", "Requirement already satisfied: accelerate in /home/ubuntu/.local/lib/python3.10/site-packages (0.29.3)\n", "Requirement already satisfied: sacrebleu in /home/ubuntu/.local/lib/python3.10/site-packages (2.4.2)\n", "Requirement already satisfied: huggingface_hub in /home/ubuntu/.local/lib/python3.10/site-packages (0.22.2)\n", "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers) (3.6.0)\n", "Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (1.25.2)\n", "Requirement already satisfied: packaging>=20.0 in /usr/lib/python3/dist-packages (from transformers) (21.3)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers) (5.4.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2024.4.16)\n", "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2.31.0)\n", "Requirement already satisfied: tokenizers<0.20,>=0.19 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.19.1)\n", "Requirement already satisfied: safetensors>=0.4.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.4.3)\n", "Requirement already satisfied: tqdm>=4.27 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (4.66.1)\n", "Requirement already satisfied: pyarrow>=12.0.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (16.0.0)\n", "Requirement already satisfied: pyarrow-hotfix in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.6)\n", "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.3.8)\n", "Requirement already satisfied: pandas in /usr/lib/python3/dist-packages (from datasets) (1.3.5)\n", "Requirement already satisfied: xxhash in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.4.1)\n", "Requirement already satisfied: multiprocess in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.70.16)\n", "Requirement already satisfied: fsspec<=2024.3.1,>=2023.1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from fsspec[http]<=2024.3.1,>=2023.1.0->datasets) (2024.3.1)\n", "Requirement already satisfied: aiohttp in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.9.5)\n", "Requirement already satisfied: psutil in /usr/lib/python3/dist-packages (from accelerate) (5.9.0)\n", "Requirement already satisfied: torch>=1.10.0 in /usr/lib/python3/dist-packages (from accelerate) (2.0.1)\n", "Requirement already satisfied: portalocker in /home/ubuntu/.local/lib/python3.10/site-packages (from sacrebleu) (2.8.2)\n", "Requirement already satisfied: tabulate>=0.8.9 in /home/ubuntu/.local/lib/python3.10/site-packages (from sacrebleu) (0.9.0)\n", "Requirement already satisfied: colorama in /usr/lib/python3/dist-packages (from sacrebleu) (0.4.4)\n", "Requirement already satisfied: lxml in /usr/lib/python3/dist-packages (from sacrebleu) (4.8.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface_hub) (4.8.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.5)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (3.3)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/lib/python3/dist-packages (from requests->transformers) (1.26.5)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers) (2020.6.20)\n", "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n", "\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n" ] } ], "source": [ "! pip install transformers datasets sentencepiece accelerate sacrebleu huggingface_hub" ] }, { "cell_type": "code", "execution_count": 2, "id": "5ce67d29-b717-48a1-9bb1-6436e2a700e2", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2024-04-23 05:47:46.579485: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", "2024-04-23 05:47:46.810151: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", "To enable the following instructions: AVX512F AVX512_VNNI, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "/usr/lib/python3/dist-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.17.3 and <1.25.0 is required for this version of SciPy (detected version 1.25.2\n", " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n" ] } ], "source": [ "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline\n", "from datasets import load_dataset, load_metric\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 3, "id": "249efb1d-4723-4544-98c1-4f75dea692a4", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/marian/tokenization_marian.py:175: UserWarning: Recommended: pip install sacremoses.\n", " warnings.warn(\"Recommended: pip install sacremoses.\")\n" ] } ], "source": [ "model_path = 'opus-mt-zh-en-finetuned-zhen-checkpoints' # 模型保存路径\n", "model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n", "tokenizer = AutoTokenizer.from_pretrained(model_path)" ] }, { "cell_type": "code", "execution_count": 4, "id": "747b0aff-1eb3-42cc-84df-0f6c11cd9fcb", "metadata": {}, "outputs": [], "source": [ "def translate(text, model, tokenizer):\n", " encoded_text = tokenizer(text, return_tensors=\"pt\", padding=True, truncation=True)\n", " translated = model.generate(**encoded_text)\n", " translated_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]\n", " return translated_text" ] }, { "cell_type": "code", "execution_count": 7, "id": "ba6141cf-ab92-407c-b7de-2da6f8ad7573", "metadata": {}, "outputs": [], "source": [ "input_text = \"Scipy 是 Python 的一个核心科学计算库，它基于 NumPy，并为我们提供了许多用户友好的数学、科学以及工程方面的功能。Scipy 的功能涵盖了多个子模块，如优化、信号处理、图像处理、统计等。\"" ] }, { "cell_type": "code", "execution_count": 8, "id": "748b9ae1-145d-47aa-86c8-7ea924906921", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Translated Text: ['Scippy, a core scientific computing base of Python, is based on NumPy and provides us with many user-friendly mathematics, science, engineering functions. It covers several sub-modules such as optimization, signal processing, image processing, statistics, etc.']\n" ] } ], "source": [ "output_text = translate(input_text, model, tokenizer)\n", "print(\"Translated Text:\", output_text)" ] }, { "cell_type": "code", "execution_count": null, "id": "120a56ac-f6d7-4bb6-893e-bfec6eadd0e8", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.12" } }, "nbformat": 4, "nbformat_minor": 5 }