report-summarization_model

import numpy as np import evaluate from datasets import load_dataset from transformers import AutoTokenizer from transformers import DataCollatorForSeq2Seq from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer from huggingface_hub import notebook_login from huggingface_hub import HfApi

api = HfApi() api.upload_file( path_or_fileobj="\Users\nandh\Desktop\StudyBuzz\newmodel.py", path_in_repo="README.md", repo_id="nandhag29/report-summarization_model", repo_type="model", )

govreps = load_dataset("ccdv/govreport-summarization", split="train") govreps = govreps.train_test_split(test_size=0.2)

checkpoint = "t5-small" tokenizer = AutoTokenizer.from_pretrained(checkpoint)

prefix = "summarize: "

def preprocess_function(examples): inputs = [prefix + doc for doc in examples["report"]] model_inputs = tokenizer(inputs, max_length=1024, truncation=True)

labels = tokenizer(text_target=examples["summary"], max_length=128, truncation=True)

model_inputs["labels"] = labels["input_ids"]
return model_inputs

tokenized_govreps = govreps.map(preprocess_function, batched=True)

data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=checkpoint)

rouge = evaluate.load("rouge")

def compute_metrics(eval_pred): predictions, labels = eval_pred decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True) labels = np.where(labels != -100, labels, tokenizer.pad_token_id) decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)

result = rouge.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)

prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]
result["gen_len"] = np.mean(prediction_lens)

return {k: round(v, 4) for k, v in result.items()}

model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)

training_args = Seq2SeqTrainingArguments( output_dir="report-summarization_model", evaluation_strategy="epoch", learning_rate=2e-5, per_device_train_batch_size=16, per_device_eval_batch_size=16, weight_decay=0.01, save_total_limit=3, num_train_epochs=4, predict_with_generate=True, fp16=False, push_to_hub=True, )

trainer = Seq2SeqTrainer( model=model, args=training_args, train_dataset=tokenized_govreps["train"], eval_dataset=tokenized_govreps["test"], tokenizer=tokenizer, data_collator=data_collator, compute_metrics=compute_metrics, )

trainer.train()