File size: 47,899 Bytes
dd89ecf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 |
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"id": "0paOn0yhDB63"
},
"outputs": [],
"source": [
"from hybrid_pipe import HybridQAPipeline\n",
"from transformers import pipeline\n",
"from transformers.pipelines import PIPELINE_REGISTRY\n",
"\n",
"from transformers import AutoModelForQuestionAnswering, TFAutoModelForQuestionAnswering\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"id": "DuwOF8yjDB66"
},
"outputs": [],
"source": [
"# Register new pipe\n",
"PIPELINE_REGISTRY.register_pipeline(\n",
" \"hybrid-qa\",\n",
" pipeline_class=HybridQAPipeline,\n",
" pt_model=AutoModelForQuestionAnswering,\n",
" tf_model=TFAutoModelForQuestionAnswering\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"id": "pf_tBYQsDB67",
"colab": {
"base_uri": "https://localhost:8080/"
},
"outputId": "2d75ec1b-a844-441b-ca84-7859dd8eedc5"
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n"
]
}
],
"source": [
"# Create pipe instance\n",
"# Note: the model specified here does not matter, we just need to\n",
"# pass something valid to satisfy the pipeline class=\n",
"hybrid_pipe = pipeline(\"hybrid-qa\", model='datarpit/distilbert-base-uncased-finetuned-natural-questions')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "KKv6ZS2LDB67",
"outputId": "58f78991-1204-4714-af1c-bad70d120118"
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1141: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
" warnings.warn(\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'guess': 'Oslo', 'confidence': 2.0940363768613864e-14}"
]
},
"metadata": {},
"execution_count": 5
}
],
"source": [
"# Inference testing!\n",
"hybrid_pipe(question=\"What is the capital of Norway?\",context=\"The capital of Norway is Oslo\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 53
},
"id": "sgrDgs9-DB68",
"outputId": "7fe9f733-f19b-43cb-e68c-33e302b2be43"
},
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"CommitInfo(commit_url='https://huggingface.co/justinhl/hybrid-qa/commit/7019d3e4971d6c754e9529b5a3de9a0425c3cccf', commit_message='Upload HybridQAPipeline', commit_description='', oid='7019d3e4971d6c754e9529b5a3de9a0425c3cccf', pr_url=None, pr_revision=None, pr_num=None)"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
}
},
"metadata": {},
"execution_count": 6
}
],
"source": [
"# Pushing to hub\n",
"hybrid_pipe.push_to_hub(\"hybrid-qa\")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "PPOf6vUhDB68",
"outputId": "1fa601e1-dfc7-4128-c430-44652916aa87"
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Some weights of the model checkpoint at justinhl/hybrid-qa were not used when initializing DistilBertForQuestionAnswering: ['model_extractive.distilbert.embeddings.LayerNorm.bias', 'model_extractive.distilbert.embeddings.LayerNorm.weight', 'model_extractive.distilbert.embeddings.position_embeddings.weight', 'model_extractive.distilbert.embeddings.word_embeddings.weight', 'model_extractive.distilbert.transformer.layer.0.attention.k_lin.bias', 'model_extractive.distilbert.transformer.layer.0.attention.k_lin.weight', 'model_extractive.distilbert.transformer.layer.0.attention.out_lin.bias', 'model_extractive.distilbert.transformer.layer.0.attention.out_lin.weight', 'model_extractive.distilbert.transformer.layer.0.attention.q_lin.bias', 'model_extractive.distilbert.transformer.layer.0.attention.q_lin.weight', 'model_extractive.distilbert.transformer.layer.0.attention.v_lin.bias', 'model_extractive.distilbert.transformer.layer.0.attention.v_lin.weight', 'model_extractive.distilbert.transformer.layer.0.ffn.lin1.bias', 'model_extractive.distilbert.transformer.layer.0.ffn.lin1.weight', 'model_extractive.distilbert.transformer.layer.0.ffn.lin2.bias', 'model_extractive.distilbert.transformer.layer.0.ffn.lin2.weight', 'model_extractive.distilbert.transformer.layer.0.output_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.0.output_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.0.sa_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.0.sa_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.1.attention.k_lin.bias', 'model_extractive.distilbert.transformer.layer.1.attention.k_lin.weight', 'model_extractive.distilbert.transformer.layer.1.attention.out_lin.bias', 'model_extractive.distilbert.transformer.layer.1.attention.out_lin.weight', 'model_extractive.distilbert.transformer.layer.1.attention.q_lin.bias', 'model_extractive.distilbert.transformer.layer.1.attention.q_lin.weight', 'model_extractive.distilbert.transformer.layer.1.attention.v_lin.bias', 'model_extractive.distilbert.transformer.layer.1.attention.v_lin.weight', 'model_extractive.distilbert.transformer.layer.1.ffn.lin1.bias', 'model_extractive.distilbert.transformer.layer.1.ffn.lin1.weight', 'model_extractive.distilbert.transformer.layer.1.ffn.lin2.bias', 'model_extractive.distilbert.transformer.layer.1.ffn.lin2.weight', 'model_extractive.distilbert.transformer.layer.1.output_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.1.output_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.1.sa_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.1.sa_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.2.attention.k_lin.bias', 'model_extractive.distilbert.transformer.layer.2.attention.k_lin.weight', 'model_extractive.distilbert.transformer.layer.2.attention.out_lin.bias', 'model_extractive.distilbert.transformer.layer.2.attention.out_lin.weight', 'model_extractive.distilbert.transformer.layer.2.attention.q_lin.bias', 'model_extractive.distilbert.transformer.layer.2.attention.q_lin.weight', 'model_extractive.distilbert.transformer.layer.2.attention.v_lin.bias', 'model_extractive.distilbert.transformer.layer.2.attention.v_lin.weight', 'model_extractive.distilbert.transformer.layer.2.ffn.lin1.bias', 'model_extractive.distilbert.transformer.layer.2.ffn.lin1.weight', 'model_extractive.distilbert.transformer.layer.2.ffn.lin2.bias', 'model_extractive.distilbert.transformer.layer.2.ffn.lin2.weight', 'model_extractive.distilbert.transformer.layer.2.output_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.2.output_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.2.sa_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.2.sa_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.3.attention.k_lin.bias', 'model_extractive.distilbert.transformer.layer.3.attention.k_lin.weight', 'model_extractive.distilbert.transformer.layer.3.attention.out_lin.bias', 'model_extractive.distilbert.transformer.layer.3.attention.out_lin.weight', 'model_extractive.distilbert.transformer.layer.3.attention.q_lin.bias', 'model_extractive.distilbert.transformer.layer.3.attention.q_lin.weight', 'model_extractive.distilbert.transformer.layer.3.attention.v_lin.bias', 'model_extractive.distilbert.transformer.layer.3.attention.v_lin.weight', 'model_extractive.distilbert.transformer.layer.3.ffn.lin1.bias', 'model_extractive.distilbert.transformer.layer.3.ffn.lin1.weight', 'model_extractive.distilbert.transformer.layer.3.ffn.lin2.bias', 'model_extractive.distilbert.transformer.layer.3.ffn.lin2.weight', 'model_extractive.distilbert.transformer.layer.3.output_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.3.output_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.3.sa_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.3.sa_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.4.attention.k_lin.bias', 'model_extractive.distilbert.transformer.layer.4.attention.k_lin.weight', 'model_extractive.distilbert.transformer.layer.4.attention.out_lin.bias', 'model_extractive.distilbert.transformer.layer.4.attention.out_lin.weight', 'model_extractive.distilbert.transformer.layer.4.attention.q_lin.bias', 'model_extractive.distilbert.transformer.layer.4.attention.q_lin.weight', 'model_extractive.distilbert.transformer.layer.4.attention.v_lin.bias', 'model_extractive.distilbert.transformer.layer.4.attention.v_lin.weight', 'model_extractive.distilbert.transformer.layer.4.ffn.lin1.bias', 'model_extractive.distilbert.transformer.layer.4.ffn.lin1.weight', 'model_extractive.distilbert.transformer.layer.4.ffn.lin2.bias', 'model_extractive.distilbert.transformer.layer.4.ffn.lin2.weight', 'model_extractive.distilbert.transformer.layer.4.output_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.4.output_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.4.sa_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.4.sa_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.5.attention.k_lin.bias', 'model_extractive.distilbert.transformer.layer.5.attention.k_lin.weight', 'model_extractive.distilbert.transformer.layer.5.attention.out_lin.bias', 'model_extractive.distilbert.transformer.layer.5.attention.out_lin.weight', 'model_extractive.distilbert.transformer.layer.5.attention.q_lin.bias', 'model_extractive.distilbert.transformer.layer.5.attention.q_lin.weight', 'model_extractive.distilbert.transformer.layer.5.attention.v_lin.bias', 'model_extractive.distilbert.transformer.layer.5.attention.v_lin.weight', 'model_extractive.distilbert.transformer.layer.5.ffn.lin1.bias', 'model_extractive.distilbert.transformer.layer.5.ffn.lin1.weight', 'model_extractive.distilbert.transformer.layer.5.ffn.lin2.bias', 'model_extractive.distilbert.transformer.layer.5.ffn.lin2.weight', 'model_extractive.distilbert.transformer.layer.5.output_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.5.output_layer_norm.weight', 'model_extractive.distilbert.transformer.layer.5.sa_layer_norm.bias', 'model_extractive.distilbert.transformer.layer.5.sa_layer_norm.weight', 'model_extractive.qa_outputs.bias', 'model_extractive.qa_outputs.weight', 'model_generative.decoder.block.0.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.0.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.0.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight', 'model_generative.decoder.block.0.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.0.layer.0.layer_norm.weight', 'model_generative.decoder.block.0.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.0.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.0.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.0.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.0.layer.1.layer_norm.weight', 'model_generative.decoder.block.0.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.0.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.0.layer.2.layer_norm.weight', 'model_generative.decoder.block.1.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.1.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.1.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.1.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.1.layer.0.layer_norm.weight', 'model_generative.decoder.block.1.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.1.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.1.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.1.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.1.layer.1.layer_norm.weight', 'model_generative.decoder.block.1.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.1.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.1.layer.2.layer_norm.weight', 'model_generative.decoder.block.10.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.10.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.10.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.10.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.10.layer.0.layer_norm.weight', 'model_generative.decoder.block.10.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.10.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.10.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.10.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.10.layer.1.layer_norm.weight', 'model_generative.decoder.block.10.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.10.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.10.layer.2.layer_norm.weight', 'model_generative.decoder.block.11.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.11.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.11.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.11.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.11.layer.0.layer_norm.weight', 'model_generative.decoder.block.11.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.11.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.11.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.11.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.11.layer.1.layer_norm.weight', 'model_generative.decoder.block.11.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.11.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.11.layer.2.layer_norm.weight', 'model_generative.decoder.block.2.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.2.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.2.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.2.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.2.layer.0.layer_norm.weight', 'model_generative.decoder.block.2.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.2.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.2.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.2.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.2.layer.1.layer_norm.weight', 'model_generative.decoder.block.2.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.2.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.2.layer.2.layer_norm.weight', 'model_generative.decoder.block.3.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.3.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.3.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.3.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.3.layer.0.layer_norm.weight', 'model_generative.decoder.block.3.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.3.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.3.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.3.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.3.layer.1.layer_norm.weight', 'model_generative.decoder.block.3.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.3.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.3.layer.2.layer_norm.weight', 'model_generative.decoder.block.4.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.4.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.4.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.4.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.4.layer.0.layer_norm.weight', 'model_generative.decoder.block.4.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.4.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.4.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.4.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.4.layer.1.layer_norm.weight', 'model_generative.decoder.block.4.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.4.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.4.layer.2.layer_norm.weight', 'model_generative.decoder.block.5.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.5.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.5.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.5.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.5.layer.0.layer_norm.weight', 'model_generative.decoder.block.5.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.5.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.5.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.5.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.5.layer.1.layer_norm.weight', 'model_generative.decoder.block.5.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.5.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.5.layer.2.layer_norm.weight', 'model_generative.decoder.block.6.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.6.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.6.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.6.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.6.layer.0.layer_norm.weight', 'model_generative.decoder.block.6.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.6.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.6.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.6.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.6.layer.1.layer_norm.weight', 'model_generative.decoder.block.6.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.6.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.6.layer.2.layer_norm.weight', 'model_generative.decoder.block.7.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.7.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.7.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.7.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.7.layer.0.layer_norm.weight', 'model_generative.decoder.block.7.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.7.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.7.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.7.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.7.layer.1.layer_norm.weight', 'model_generative.decoder.block.7.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.7.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.7.layer.2.layer_norm.weight', 'model_generative.decoder.block.8.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.8.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.8.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.8.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.8.layer.0.layer_norm.weight', 'model_generative.decoder.block.8.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.8.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.8.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.8.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.8.layer.1.layer_norm.weight', 'model_generative.decoder.block.8.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.8.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.8.layer.2.layer_norm.weight', 'model_generative.decoder.block.9.layer.0.SelfAttention.k.weight', 'model_generative.decoder.block.9.layer.0.SelfAttention.o.weight', 'model_generative.decoder.block.9.layer.0.SelfAttention.q.weight', 'model_generative.decoder.block.9.layer.0.SelfAttention.v.weight', 'model_generative.decoder.block.9.layer.0.layer_norm.weight', 'model_generative.decoder.block.9.layer.1.EncDecAttention.k.weight', 'model_generative.decoder.block.9.layer.1.EncDecAttention.o.weight', 'model_generative.decoder.block.9.layer.1.EncDecAttention.q.weight', 'model_generative.decoder.block.9.layer.1.EncDecAttention.v.weight', 'model_generative.decoder.block.9.layer.1.layer_norm.weight', 'model_generative.decoder.block.9.layer.2.DenseReluDense.wi.weight', 'model_generative.decoder.block.9.layer.2.DenseReluDense.wo.weight', 'model_generative.decoder.block.9.layer.2.layer_norm.weight', 'model_generative.decoder.embed_tokens.weight', 'model_generative.decoder.final_layer_norm.weight', 'model_generative.encoder.block.0.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.0.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.0.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.0.layer.0.SelfAttention.relative_attention_bias.weight', 'model_generative.encoder.block.0.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.0.layer.0.layer_norm.weight', 'model_generative.encoder.block.0.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.0.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.0.layer.1.layer_norm.weight', 'model_generative.encoder.block.1.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.1.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.1.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.1.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.1.layer.0.layer_norm.weight', 'model_generative.encoder.block.1.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.1.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.1.layer.1.layer_norm.weight', 'model_generative.encoder.block.10.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.10.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.10.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.10.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.10.layer.0.layer_norm.weight', 'model_generative.encoder.block.10.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.10.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.10.layer.1.layer_norm.weight', 'model_generative.encoder.block.11.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.11.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.11.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.11.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.11.layer.0.layer_norm.weight', 'model_generative.encoder.block.11.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.11.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.11.layer.1.layer_norm.weight', 'model_generative.encoder.block.2.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.2.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.2.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.2.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.2.layer.0.layer_norm.weight', 'model_generative.encoder.block.2.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.2.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.2.layer.1.layer_norm.weight', 'model_generative.encoder.block.3.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.3.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.3.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.3.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.3.layer.0.layer_norm.weight', 'model_generative.encoder.block.3.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.3.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.3.layer.1.layer_norm.weight', 'model_generative.encoder.block.4.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.4.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.4.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.4.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.4.layer.0.layer_norm.weight', 'model_generative.encoder.block.4.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.4.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.4.layer.1.layer_norm.weight', 'model_generative.encoder.block.5.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.5.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.5.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.5.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.5.layer.0.layer_norm.weight', 'model_generative.encoder.block.5.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.5.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.5.layer.1.layer_norm.weight', 'model_generative.encoder.block.6.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.6.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.6.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.6.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.6.layer.0.layer_norm.weight', 'model_generative.encoder.block.6.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.6.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.6.layer.1.layer_norm.weight', 'model_generative.encoder.block.7.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.7.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.7.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.7.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.7.layer.0.layer_norm.weight', 'model_generative.encoder.block.7.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.7.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.7.layer.1.layer_norm.weight', 'model_generative.encoder.block.8.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.8.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.8.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.8.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.8.layer.0.layer_norm.weight', 'model_generative.encoder.block.8.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.8.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.8.layer.1.layer_norm.weight', 'model_generative.encoder.block.9.layer.0.SelfAttention.k.weight', 'model_generative.encoder.block.9.layer.0.SelfAttention.o.weight', 'model_generative.encoder.block.9.layer.0.SelfAttention.q.weight', 'model_generative.encoder.block.9.layer.0.SelfAttention.v.weight', 'model_generative.encoder.block.9.layer.0.layer_norm.weight', 'model_generative.encoder.block.9.layer.1.DenseReluDense.wi.weight', 'model_generative.encoder.block.9.layer.1.DenseReluDense.wo.weight', 'model_generative.encoder.block.9.layer.1.layer_norm.weight', 'model_generative.encoder.embed_tokens.weight', 'model_generative.encoder.final_layer_norm.weight', 'model_generative.lm_head.weight', 'model_generative.shared.weight']\n",
"- This IS expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
"- This IS NOT expected if you are initializing DistilBertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
"Some weights of DistilBertForQuestionAnswering were not initialized from the model checkpoint at justinhl/hybrid-qa and are newly initialized: ['embeddings.LayerNorm.bias', 'embeddings.LayerNorm.weight', 'embeddings.position_embeddings.weight', 'embeddings.word_embeddings.weight', 'qa_outputs.bias', 'qa_outputs.weight', 'transformer.layer.0.attention.k_lin.bias', 'transformer.layer.0.attention.k_lin.weight', 'transformer.layer.0.attention.out_lin.bias', 'transformer.layer.0.attention.out_lin.weight', 'transformer.layer.0.attention.q_lin.bias', 'transformer.layer.0.attention.q_lin.weight', 'transformer.layer.0.attention.v_lin.bias', 'transformer.layer.0.attention.v_lin.weight', 'transformer.layer.0.ffn.lin1.bias', 'transformer.layer.0.ffn.lin1.weight', 'transformer.layer.0.ffn.lin2.bias', 'transformer.layer.0.ffn.lin2.weight', 'transformer.layer.0.output_layer_norm.bias', 'transformer.layer.0.output_layer_norm.weight', 'transformer.layer.0.sa_layer_norm.bias', 'transformer.layer.0.sa_layer_norm.weight', 'transformer.layer.1.attention.k_lin.bias', 'transformer.layer.1.attention.k_lin.weight', 'transformer.layer.1.attention.out_lin.bias', 'transformer.layer.1.attention.out_lin.weight', 'transformer.layer.1.attention.q_lin.bias', 'transformer.layer.1.attention.q_lin.weight', 'transformer.layer.1.attention.v_lin.bias', 'transformer.layer.1.attention.v_lin.weight', 'transformer.layer.1.ffn.lin1.bias', 'transformer.layer.1.ffn.lin1.weight', 'transformer.layer.1.ffn.lin2.bias', 'transformer.layer.1.ffn.lin2.weight', 'transformer.layer.1.output_layer_norm.bias', 'transformer.layer.1.output_layer_norm.weight', 'transformer.layer.1.sa_layer_norm.bias', 'transformer.layer.1.sa_layer_norm.weight', 'transformer.layer.2.attention.k_lin.bias', 'transformer.layer.2.attention.k_lin.weight', 'transformer.layer.2.attention.out_lin.bias', 'transformer.layer.2.attention.out_lin.weight', 'transformer.layer.2.attention.q_lin.bias', 'transformer.layer.2.attention.q_lin.weight', 'transformer.layer.2.attention.v_lin.bias', 'transformer.layer.2.attention.v_lin.weight', 'transformer.layer.2.ffn.lin1.bias', 'transformer.layer.2.ffn.lin1.weight', 'transformer.layer.2.ffn.lin2.bias', 'transformer.layer.2.ffn.lin2.weight', 'transformer.layer.2.output_layer_norm.bias', 'transformer.layer.2.output_layer_norm.weight', 'transformer.layer.2.sa_layer_norm.bias', 'transformer.layer.2.sa_layer_norm.weight', 'transformer.layer.3.attention.k_lin.bias', 'transformer.layer.3.attention.k_lin.weight', 'transformer.layer.3.attention.out_lin.bias', 'transformer.layer.3.attention.out_lin.weight', 'transformer.layer.3.attention.q_lin.bias', 'transformer.layer.3.attention.q_lin.weight', 'transformer.layer.3.attention.v_lin.bias', 'transformer.layer.3.attention.v_lin.weight', 'transformer.layer.3.ffn.lin1.bias', 'transformer.layer.3.ffn.lin1.weight', 'transformer.layer.3.ffn.lin2.bias', 'transformer.layer.3.ffn.lin2.weight', 'transformer.layer.3.output_layer_norm.bias', 'transformer.layer.3.output_layer_norm.weight', 'transformer.layer.3.sa_layer_norm.bias', 'transformer.layer.3.sa_layer_norm.weight', 'transformer.layer.4.attention.k_lin.bias', 'transformer.layer.4.attention.k_lin.weight', 'transformer.layer.4.attention.out_lin.bias', 'transformer.layer.4.attention.out_lin.weight', 'transformer.layer.4.attention.q_lin.bias', 'transformer.layer.4.attention.q_lin.weight', 'transformer.layer.4.attention.v_lin.bias', 'transformer.layer.4.attention.v_lin.weight', 'transformer.layer.4.ffn.lin1.bias', 'transformer.layer.4.ffn.lin1.weight', 'transformer.layer.4.ffn.lin2.bias', 'transformer.layer.4.ffn.lin2.weight', 'transformer.layer.4.output_layer_norm.bias', 'transformer.layer.4.output_layer_norm.weight', 'transformer.layer.4.sa_layer_norm.bias', 'transformer.layer.4.sa_layer_norm.weight', 'transformer.layer.5.attention.k_lin.bias', 'transformer.layer.5.attention.k_lin.weight', 'transformer.layer.5.attention.out_lin.bias', 'transformer.layer.5.attention.out_lin.weight', 'transformer.layer.5.attention.q_lin.bias', 'transformer.layer.5.attention.q_lin.weight', 'transformer.layer.5.attention.v_lin.bias', 'transformer.layer.5.attention.v_lin.weight', 'transformer.layer.5.ffn.lin1.bias', 'transformer.layer.5.ffn.lin1.weight', 'transformer.layer.5.ffn.lin2.bias', 'transformer.layer.5.ffn.lin2.weight', 'transformer.layer.5.output_layer_norm.bias', 'transformer.layer.5.output_layer_norm.weight', 'transformer.layer.5.sa_layer_norm.bias', 'transformer.layer.5.sa_layer_norm.weight']\n",
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
]
}
],
"source": [
"# Importing from remote\n",
"imported_pipe = pipeline(\"hybrid-qa\", model=\"justinhl/hybrid-qa\", trust_remote_code=True)"
]
},
{
"cell_type": "code",
"source": [
"# Inference testing!\n",
"imported_pipe(question=\"What is the capital of Norway?\",context=\"The capital of Norway is Oslo\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "sQsoT-UpPp0O",
"outputId": "dd922309-bd21-4684-caee-c4d4499bf69b"
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
"/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py:1141: UserWarning: Using the model-agnostic default `max_length` (=20) to control the generation length. We recommend setting `max_new_tokens` to control the maximum length of the generation.\n",
" warnings.warn(\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"{'guess': 'Oslo', 'confidence': 2.0940363768613864e-14}"
]
},
"metadata": {},
"execution_count": 8
}
]
},
{
"cell_type": "code",
"source": [
"print(\"Model loaded:\", imported_pipe.model)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "GEmtld6OVT7W",
"outputId": "93217a25-668e-4a46-8fc9-9db440693a1c"
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Model loaded: HybridQAModel(\n",
" (model_extractive): DistilBertForQuestionAnswering(\n",
" (distilbert): DistilBertModel(\n",
" (embeddings): Embeddings(\n",
" (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
" (position_embeddings): Embedding(512, 768)\n",
" (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (transformer): Transformer(\n",
" (layer): ModuleList(\n",
" (0-5): 6 x TransformerBlock(\n",
" (attention): MultiHeadSelfAttention(\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
" (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
" (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
" (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
" )\n",
" (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" (ffn): FFN(\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
" (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
" (activation): GELUActivation()\n",
" )\n",
" (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (qa_outputs): Linear(in_features=768, out_features=2, bias=True)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (model_generative): T5ForConditionalGeneration(\n",
" (shared): Embedding(32128, 768)\n",
" (encoder): T5Stack(\n",
" (embed_tokens): Embedding(32128, 768)\n",
" (block): ModuleList(\n",
" (0): T5Block(\n",
" (layer): ModuleList(\n",
" (0): T5LayerSelfAttention(\n",
" (SelfAttention): T5Attention(\n",
" (q): Linear(in_features=768, out_features=768, bias=False)\n",
" (k): Linear(in_features=768, out_features=768, bias=False)\n",
" (v): Linear(in_features=768, out_features=768, bias=False)\n",
" (o): Linear(in_features=768, out_features=768, bias=False)\n",
" (relative_attention_bias): Embedding(32, 12)\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (1): T5LayerFF(\n",
" (DenseReluDense): T5DenseActDense(\n",
" (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
" (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (act): ReLU()\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" (1-11): 11 x T5Block(\n",
" (layer): ModuleList(\n",
" (0): T5LayerSelfAttention(\n",
" (SelfAttention): T5Attention(\n",
" (q): Linear(in_features=768, out_features=768, bias=False)\n",
" (k): Linear(in_features=768, out_features=768, bias=False)\n",
" (v): Linear(in_features=768, out_features=768, bias=False)\n",
" (o): Linear(in_features=768, out_features=768, bias=False)\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (1): T5LayerFF(\n",
" (DenseReluDense): T5DenseActDense(\n",
" (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
" (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (act): ReLU()\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (final_layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (decoder): T5Stack(\n",
" (embed_tokens): Embedding(32128, 768)\n",
" (block): ModuleList(\n",
" (0): T5Block(\n",
" (layer): ModuleList(\n",
" (0): T5LayerSelfAttention(\n",
" (SelfAttention): T5Attention(\n",
" (q): Linear(in_features=768, out_features=768, bias=False)\n",
" (k): Linear(in_features=768, out_features=768, bias=False)\n",
" (v): Linear(in_features=768, out_features=768, bias=False)\n",
" (o): Linear(in_features=768, out_features=768, bias=False)\n",
" (relative_attention_bias): Embedding(32, 12)\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (1): T5LayerCrossAttention(\n",
" (EncDecAttention): T5Attention(\n",
" (q): Linear(in_features=768, out_features=768, bias=False)\n",
" (k): Linear(in_features=768, out_features=768, bias=False)\n",
" (v): Linear(in_features=768, out_features=768, bias=False)\n",
" (o): Linear(in_features=768, out_features=768, bias=False)\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (2): T5LayerFF(\n",
" (DenseReluDense): T5DenseActDense(\n",
" (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
" (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (act): ReLU()\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" (1-11): 11 x T5Block(\n",
" (layer): ModuleList(\n",
" (0): T5LayerSelfAttention(\n",
" (SelfAttention): T5Attention(\n",
" (q): Linear(in_features=768, out_features=768, bias=False)\n",
" (k): Linear(in_features=768, out_features=768, bias=False)\n",
" (v): Linear(in_features=768, out_features=768, bias=False)\n",
" (o): Linear(in_features=768, out_features=768, bias=False)\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (1): T5LayerCrossAttention(\n",
" (EncDecAttention): T5Attention(\n",
" (q): Linear(in_features=768, out_features=768, bias=False)\n",
" (k): Linear(in_features=768, out_features=768, bias=False)\n",
" (v): Linear(in_features=768, out_features=768, bias=False)\n",
" (o): Linear(in_features=768, out_features=768, bias=False)\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (2): T5LayerFF(\n",
" (DenseReluDense): T5DenseActDense(\n",
" (wi): Linear(in_features=768, out_features=3072, bias=False)\n",
" (wo): Linear(in_features=3072, out_features=768, bias=False)\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" (act): ReLU()\n",
" )\n",
" (layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" )\n",
" )\n",
" )\n",
" (final_layer_norm): T5LayerNorm()\n",
" (dropout): Dropout(p=0.1, inplace=False)\n",
" )\n",
" (lm_head): Linear(in_features=768, out_features=32128, bias=False)\n",
" )\n",
")\n"
]
}
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
},
"colab": {
"provenance": []
}
},
"nbformat": 4,
"nbformat_minor": 0
} |