from datasets import load_dataset dataset = load_dataset("yelp_review_full") dataset["train"][100] #creating the dataset from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") def tokenize_function(examples): return tokenizer(examples["text"], padding="max_length", truncation=True) #mapping the values: tokenized_datasets = dataset.map(tokenize_function, batched=True) #small Datasets: small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000)) small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000)) #Loading pretrained Model: from transformers import AutoModelForSequenceClassification model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5) ### from transformers import TrainingArguments training_args = TrainingArguments(output_dir="test_trainer") #Evaluate def compute_metrics(eval_pred): logits, labels = eval_pred predictions = np.argmax(logits, axis=-1) return metric.compute(predictions=predictions, references=labels) #Training Argumnents and importing Trainer: from transformers import TrainingArguments, Trainer training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch") #Defining Hyperparameters for Trainer: trainer = Trainer( model=model, args=training_args, train_dataset=small_train_dataset, eval_dataset=small_eval_dataset, compute_metrics=compute_metrics, ) #Execute the training: trainer.train() #Predictions: predictions = trainer.predict(small_eval_dataset) print(predictions.predictions.shape,predictions.label_ids.shape) print(predictions) #Gradio import gradio as gr def greet(name): return "Hello " + name + "!" demo = gr.Interface(fn=greet, inputs="text", outputs="text") demo.launch()