Hugging Face's logo

Varine
/
opus-mt-zh-en-model

Text2Text Generation
Transformers
TensorBoard
Safetensors
Chinese
English
marian
Inference Endpoints
Model card Files Metrics Community
opus-mt-zh-en-model
File size: 10,752 Bytes 
ad05dff
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
 
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0b15c2b0-69f1-4201-a937-36bd89b97d3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: transformers in /home/ubuntu/.local/lib/python3.10/site-packages (4.40.0)\n",
      "Requirement already satisfied: datasets in /home/ubuntu/.local/lib/python3.10/site-packages (2.19.0)\n",
      "Requirement already satisfied: sentencepiece in /home/ubuntu/.local/lib/python3.10/site-packages (0.2.0)\n",
      "Requirement already satisfied: accelerate in /home/ubuntu/.local/lib/python3.10/site-packages (0.29.3)\n",
      "Requirement already satisfied: sacrebleu in /home/ubuntu/.local/lib/python3.10/site-packages (2.4.2)\n",
      "Requirement already satisfied: huggingface_hub in /home/ubuntu/.local/lib/python3.10/site-packages (0.22.2)\n",
      "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers) (3.6.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (1.25.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/lib/python3/dist-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers) (5.4.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2024.4.16)\n",
      "Requirement already satisfied: requests in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.19.1)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (0.4.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/ubuntu/.local/lib/python3.10/site-packages (from transformers) (4.66.1)\n",
      "Requirement already satisfied: pyarrow>=12.0.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (16.0.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/lib/python3/dist-packages (from datasets) (1.3.5)\n",
      "Requirement already satisfied: xxhash in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.4.1)\n",
      "Requirement already satisfied: multiprocess in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.3.1,>=2023.1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from fsspec[http]<=2024.3.1,>=2023.1.0->datasets) (2024.3.1)\n",
      "Requirement already satisfied: aiohttp in /home/ubuntu/.local/lib/python3.10/site-packages (from datasets) (3.9.5)\n",
      "Requirement already satisfied: psutil in /usr/lib/python3/dist-packages (from accelerate) (5.9.0)\n",
      "Requirement already satisfied: torch>=1.10.0 in /usr/lib/python3/dist-packages (from accelerate) (2.0.1)\n",
      "Requirement already satisfied: portalocker in /home/ubuntu/.local/lib/python3.10/site-packages (from sacrebleu) (2.8.2)\n",
      "Requirement already satisfied: tabulate>=0.8.9 in /home/ubuntu/.local/lib/python3.10/site-packages (from sacrebleu) (0.9.0)\n",
      "Requirement already satisfied: colorama in /usr/lib/python3/dist-packages (from sacrebleu) (0.4.4)\n",
      "Requirement already satisfied: lxml in /usr/lib/python3/dist-packages (from sacrebleu) (4.8.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ubuntu/.local/lib/python3.10/site-packages (from huggingface_hub) (4.8.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/ubuntu/.local/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ubuntu/.local/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/lib/python3/dist-packages (from requests->transformers) (1.26.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers) (2020.6.20)\n",
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3 -m pip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "! pip install transformers datasets sentencepiece accelerate sacrebleu huggingface_hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5ce67d29-b717-48a1-9bb1-6436e2a700e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-23 05:47:46.579485: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-04-23 05:47:46.810151: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX512F AVX512_VNNI, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "/usr/lib/python3/dist-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.17.3 and <1.25.0 is required for this version of SciPy (detected version 1.25.2\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline\n",
    "from datasets import load_dataset, load_metric\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "249efb1d-4723-4544-98c1-4f75dea692a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/transformers/models/marian/tokenization_marian.py:175: UserWarning: Recommended: pip install sacremoses.\n",
      "  warnings.warn(\"Recommended: pip install sacremoses.\")\n"
     ]
    }
   ],
   "source": [
    "model_path = 'opus-mt-zh-en-finetuned-zhen-checkpoints'  # 模型保存路径\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_path)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "747b0aff-1eb3-42cc-84df-0f6c11cd9fcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(text, model, tokenizer):\n",
    "    encoded_text = tokenizer(text, return_tensors=\"pt\", padding=True, truncation=True)\n",
    "    translated = model.generate(**encoded_text)\n",
    "    translated_text = [tokenizer.decode(t, skip_special_tokens=True) for t in translated]\n",
    "    return translated_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ba6141cf-ab92-407c-b7de-2da6f8ad7573",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_text = \"Scipy 是 Python 的一个核心科学计算库，它基于 NumPy，并为我们提供了许多用户友好的数学、科学以及工程方面的功能。Scipy 的功能涵盖了多个子模块，如优化、信号处理、图像处理、统计等。\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "748b9ae1-145d-47aa-86c8-7ea924906921",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Translated Text: ['Scippy, a core scientific computing base of Python, is based on NumPy and provides us with many user-friendly mathematics, science, engineering functions. It covers several sub-modules such as optimization, signal processing, image processing, statistics, etc.']\n"
     ]
    }
   ],
   "source": [
    "output_text = translate(input_text, model, tokenizer)\n",
    "print(\"Translated Text:\", output_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "120a56ac-f6d7-4bb6-893e-bfec6eadd0e8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}