t5-cnn-dailymail-summarizer

import torch from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification, TrainingArguments, Trainer from torch.utils.data import Dataset, DataLoader import numpy as np

--- Example: Using bert-base-multilingual-cased for Embeddings ---

model_name_embeddings = "bert-base-multilingual-cased" tokenizer_embeddings = AutoTokenizer.from_pretrained(model_name_embeddings) model_embeddings = AutoModel.from_pretrained(model_name_embeddings)

text_embeddings = "Hello, how are you? ¡Hola, cómo estás? Bonjour, comment ça va?" tokens_embeddings = tokenizer_embeddings(text_embeddings, return_tensors="pt", padding=True, truncation=True)

with torch.no_grad(): outputs_embeddings = model_embeddings(**tokens_embeddings)

embeddings = outputs_embeddings.last_hidden_state print("Embeddings Shape:", embeddings.shape)

--- Example: Using bert-base-multilingual-cased for Sequence Classification ---

model_name_classification = "bert-base-multilingual-cased" tokenizer_classification = AutoTokenizer.from_pretrained(model_name_classification) model_classification = AutoModelForSequenceClassification.from_pretrained(model_name_classification, num_labels=3) # Example: 3 labels

text_classification = "This is a positive sentence." tokens_classification = tokenizer_classification(text_classification, return_tensors="pt", padding=True, truncation=True)

with torch.no_grad(): outputs_classification = model_classification(**tokens_classification)

logits = outputs_classification.logits predicted_class = torch.argmax(logits, dim=-1) print("Predicted Class:", predicted_class)

--- Example: Fine-tuning (Simplified) ---

Sample Dataset (replace with your actual dataset)

class SimpleDataset(Dataset): def init(self, texts, labels, tokenizer, max_len): self.texts = texts self.labels = labels self.tokenizer = tokenizer self.max_len = max_len

def __len__(self):
    return len(self.texts)

def __getitem__(self, idx):
    text = str(self.texts[idx])
    label = self.labels[idx]
    encoding = self.tokenizer(
        text,
        add_special_tokens=True,
        max_length=self.max_len,
        return_token_type_ids=False,
        padding='max_length',
        return_attention_mask=True,
        return_tensors='pt',
        truncation=True
    )
    return {
        'input_ids': encoding['input_ids'].flatten(),
        'attention_mask': encoding['attention_mask'].flatten(),
        'labels': torch.tensor(label, dtype=torch.long)
    }

Sample data

train_texts = ["This is good", "This is bad", "Neutral sentence"] train_labels = [1, 0, 2] # 1: positive, 0: negative, 2: neutral

Parameters

model_name_finetune = "bert-base-multilingual-cased" tokenizer_finetune = AutoTokenizer.from_pretrained(model_name_finetune) model_finetune = AutoModelForSequenceClassification.from_pretrained(model_name_finetune, num_labels=3) max_len = 128

Create dataset

train_dataset = SimpleDataset(train_texts, train_labels, tokenizer_finetune, max_len)

Training arguments

training_args = TrainingArguments( output_dir="./results", num_train_epochs=3, per_device_train_batch_size=16, per_device_eval_batch_size=64, warmup_steps=500, weight_decay=0.01, logging_dir="./logs", logging_steps=10, )

Trainer

trainer = Trainer( model=model_finetune, args=training_args, train_dataset=train_dataset, )

Train the model

trainer.train()

#Example of inference after fine tuning. test_text = "This is very good" test_tokens = tokenizer_finetune(test_text, return_tensors="pt", padding=True, truncation=True) with torch.no_grad(): test_outputs = model_finetune(**test_tokens) test_logits = test_outputs.logits test_predicted_class = torch.argmax(test_logits, dim=-1) print(f"Test prediction after fine tuning: {test_predicted_class}")