BanglaBert_with_TFModel

Test Code

import tensorflow as tf
from transformers import TFAutoModelForPreTraining, AutoTokenizer
from normalizer import normalize
import numpy as np

model = TFAutoModelForPreTraining.from_pretrained("SarwarShafee/BanglaBert_with_TFModel", from_pt=True)
tokenizer = AutoTokenizer.from_pretrained("SarwarShafee/BanglaBert_with_TFModel")

original_sentence = "আমি কৃতজ্ঞ কারণ আপনি আমার জন্য অনেক কিছু করেছেন।"
fake_sentence = "আমি হতাশ কারণ আপনি আমার জন্য অনেক কিছু করেছেন।"
fake_sentence = normalize(fake_sentence)  # this normalization step is required before tokenizing the text

fake_tokens = tokenizer.tokenize(fake_sentence)
fake_inputs = tokenizer.encode(fake_sentence, return_tensors="tf")
discriminator_outputs = model(fake_inputs)[0]
predictions = tf.round((tf.sign(discriminator_outputs) + 1) / 2)

# Convert the predictions to a Python list and then to integers
predictions_list = predictions.numpy().squeeze().tolist()
integer_predictions = [int(prediction[0]) for prediction in predictions_list[1:-1]]

print(" ".join(fake_tokens))
print("-" * 50)
print(" ".join([str(prediction) for prediction in integer_predictions]))
print("-" * 50)