kuotient commited on
Commit
12fd89a
1 Parent(s): 31569fe

Upload folder using huggingface_hub

Browse files
Files changed (3) hide show
  1. README.md +50 -5
  2. generation_config.json +1 -1
  3. tokenizer_config.json +1 -1
README.md CHANGED
@@ -186,7 +186,6 @@ extra_gated_fields:
186
  extra_gated_description: The information you provide will be collected, stored, processed and shared in accordance with the [Meta Privacy Policy](https://www.facebook.com/privacy/policy/).
187
  extra_gated_button_content: Submit
188
  ---
189
- # Not include _./original_ from original repo.
190
 
191
  ## Model Details
192
 
@@ -274,7 +273,9 @@ This repository contains two versions of Meta-Llama-3-8B-Instruct, for use with
274
 
275
  ### Use with transformers
276
 
277
- See the snippet below for usage with Transformers:
 
 
278
 
279
  ```python
280
  import transformers
@@ -286,7 +287,7 @@ pipeline = transformers.pipeline(
286
  "text-generation",
287
  model=model_id,
288
  model_kwargs={"torch_dtype": torch.bfloat16},
289
- device="cuda",
290
  )
291
 
292
  messages = [
@@ -301,8 +302,8 @@ prompt = pipeline.tokenizer.apply_chat_template(
301
  )
302
 
303
  terminators = [
304
- tokenizer.eos_token_id,
305
- tokenizer.convert_tokens_to_ids("<|eot_id|>")
306
  ]
307
 
308
  outputs = pipeline(
@@ -316,6 +317,50 @@ outputs = pipeline(
316
  print(outputs[0]["generated_text"][len(prompt):])
317
  ```
318
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
319
  ### Use with `llama3`
320
 
321
  Please, follow the instructions in the [repository](https://github.com/meta-llama/llama3)
 
186
  extra_gated_description: The information you provide will be collected, stored, processed and shared in accordance with the [Meta Privacy Policy](https://www.facebook.com/privacy/policy/).
187
  extra_gated_button_content: Submit
188
  ---
 
189
 
190
  ## Model Details
191
 
 
273
 
274
  ### Use with transformers
275
 
276
+ You can run conversational inference using the Transformers pipeline abstraction, or by leveraging the Auto classes with the `generate()` function. Let's see examples of both.
277
+
278
+ #### Transformers pipeline
279
 
280
  ```python
281
  import transformers
 
287
  "text-generation",
288
  model=model_id,
289
  model_kwargs={"torch_dtype": torch.bfloat16},
290
+ device="auto",
291
  )
292
 
293
  messages = [
 
302
  )
303
 
304
  terminators = [
305
+ pipeline.tokenizer.eos_token_id,
306
+ pipeline.tokenizer.convert_tokens_to_ids("<|eot_id|>")
307
  ]
308
 
309
  outputs = pipeline(
 
317
  print(outputs[0]["generated_text"][len(prompt):])
318
  ```
319
 
320
+ #### Transformers AutoModelForCausalLM
321
+
322
+ ```python
323
+ from transformers import AutoTokenizer, AutoModelForCausalLM
324
+ import torch
325
+
326
+ model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
327
+
328
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
329
+ model = AutoModelForCausalLM.from_pretrained(
330
+ model_id,
331
+ torch_dtype=torch.bfloat16,
332
+ device_map="auto",
333
+ )
334
+
335
+ messages = [
336
+ {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
337
+ {"role": "user", "content": "Who are you?"},
338
+ ]
339
+
340
+ input_ids = tokenizer.apply_chat_template(
341
+ messages,
342
+ add_generation_prompt=True,
343
+ return_tensors="pt"
344
+ ).to(model.device)
345
+
346
+ terminators = [
347
+ tokenizer.eos_token_id,
348
+ tokenizer.convert_tokens_to_ids("<|eot_id|>")
349
+ ]
350
+
351
+ outputs = model.generate(
352
+ input_ids,
353
+ max_new_tokens=256,
354
+ eos_token_id=terminators,
355
+ do_sample=True,
356
+ temperature=0.6,
357
+ top_p=0.9,
358
+ )
359
+ response = outputs[0][input_ids.shape[-1]:]
360
+ print(tokenizer.decode(response, skip_special_tokens=True))
361
+ ```
362
+
363
+
364
  ### Use with `llama3`
365
 
366
  Please, follow the instructions in the [repository](https://github.com/meta-llama/llama3)
generation_config.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "_from_model_config": true,
3
  "bos_token_id": 128000,
4
- "eos_token_id": 128001,
5
  "transformers_version": "4.40.0.dev0"
6
  }
 
1
  {
2
  "_from_model_config": true,
3
  "bos_token_id": 128000,
4
+ "eos_token_id": [128001, 128009],
5
  "transformers_version": "4.40.0.dev0"
6
  }
tokenizer_config.json CHANGED
@@ -2050,7 +2050,7 @@
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
- "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}",
2054
  "clean_up_tokenization_spaces": true,
2055
  "eos_token": "<|end_of_text|>",
2056
  "model_input_names": [
 
2050
  }
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
+ "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
2054
  "clean_up_tokenization_spaces": true,
2055
  "eos_token": "<|end_of_text|>",
2056
  "model_input_names": [