Gemma4_meme_classification

Finetuned Gemma 4 based Hate Detection in Arabic MultiModal Memes

The rise of social media and online communication platforms has led to the spread of Arabic memes as a key form of digital expression. While these contents can be humorous and informative, they are also increasingly being used to spread offensive language and hate speech. Consequently, there is a growing demand for precise analysis of content in Arabic memes.

This work used Gemma 4 with its vision capability to effectively identify hate content within Arabic memes. The evaluation is conducted using a dataset of Arabic memes proposed in the ArabicNLP ArGuard 2026 challenge. The results underscore the capacity of unsloth/gemma-4-E4B-it fine-tuned with Arabic memes, to deliver the superior performance.

The proposed solutions offer a more nuanced understanding of memes for accurate and efficient Arabic content moderation systems.

Examples of Arabic Memes from ArabicNLP ArGuard 2026 challenge

Examples

Finetuned Gemma 4 Embedding Model with mean pooling

import os
import torch

# 1. Create a dummy pass-through decorator to replace torch.compile
def dummy_compile(fn=None, *args, **kwargs):
    if fn is None:
        return lambda x: x
    return fn

# 2. Patch torch.compile BEFORE unsloth imports
torch.compile = dummy_compile
os.environ["UNSLOTH_FUSED_FORWARD"] = "0"
os.environ["UNSLOTH_DISABLE_AUTO_UPDATES"] = "1"

# 3. Import Unsloth safely now
from unsloth import FastVisionModel

print("SUCCESS: Unsloth loaded smoothly without compiler errors!")


import numpy as np
import torch
import torch._dynamo
from tqdm import tqdm  # Progress bar library
from unsloth import FastVisionModel


from datasets import load_dataset

instruction = "classify meme into Hateful or Not"

def convert_to_conversation(sample):
    
    
    lis=[]
    lis.append({"type": "text", "text": sample["text"]})
    lis.append({"type": "image", "image": sample["image"]})
            
            
    conversation = [
        {
            "role": "system",
            "content": instruction,
        },
        {
            "role": "user",
            "content": lis,
        },
        {"role": "assistant", "content": [{"type": "text", "text": sample["label"]}]},
    ]
    return {"messages": conversation}
pass

dataset = load_dataset("QCRI/ArGuard-Task1",  split="train")

converted_dataset = [convert_to_conversation(sample) for sample in dataset]


# 2. Load your fine-tuned model and processor
model_path = "NYUAD-ComNets/Gemma4_meme_classification"
model, processor = FastVisionModel.from_pretrained(
    model_path,  device_map = {"": 0},
    load_in_4bit = True,token = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
)
FastVisionModel.for_inference(model)

# Instruction context
instruction = "classify meme into Hateful or Not"

all_embeddings = []
labels_list = []  

num_iterations = len(converted_dataset)

print(f"Starting embedding extraction for {num_iterations} items...")

for idx in tqdm(range(num_iterations), desc="Extracting Embeddings"):
    try:
        sample = converted_dataset[idx]
        
        # Pull text, image, and ground-truth label
        sample_text = sample["messages"][1]["content"][0]["text"]
        sample_image = sample["messages"][1]["content"][1]["image"]
        sample_label = sample["messages"][2]["content"][0]["text"] # From training format

        # Setup multimodal conversation payload
        conversation = [
            {"role": "system", "content": instruction},
            {"role": "user", "content": [{"type": "text", "text": sample_text}, {"type": "image", "image": sample_image}]},
        ]

        # 4. Process inputs normally
        templated_text = processor.apply_chat_template(conversation, tokenize=False)
        inputs = processor(text=templated_text, images=sample_image, return_tensors="pt").to("cuda")

        # 5. Forward Pass
        with torch.no_grad():
            with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
                outputs = model(**inputs, output_hidden_states=True, return_dict=True)

        # 6. Extract final layer and attention mask
        last_hidden_states = outputs.hidden_states[-1]  # [batch_size, seq_len, hidden_dim]
        attention_mask = inputs["attention_mask"]       # [batch_size, seq_len]

        # 7. Masked Mean Pooling (Ignores padding tokens entirely)
        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_states.size()).float()
        sum_embeddings = torch.sum(last_hidden_states * input_mask_expanded, dim=1)
        sum_mask = torch.clamp(input_mask_expanded.sum(dim=1), min=1e-9)
        
        all_embedding = (sum_embeddings / sum_mask).squeeze(0).float()
        
        all_embeddings.append(all_embedding.cpu().numpy())


        labels_list.append(sample_label)

    except Exception as e:
        print(f"\nSkipping row {idx} due to an error: {e}")
        continue


embedding_matrix = np.vstack(all_embeddings)

print("Final Concatenated Array Shape:", embedding_matrix.shape)

np.save("train_gemma4_mean_embeddings.npy", embedding_matrix)

Finetuned Gemma 4 for Inference

import pandas as pd
import torch

from datasets import load_dataset
dataset = load_dataset("QCRI/ArGuard-Task1")

instruction = "Classify meme into Hateful or not"

def convert_to_conversation(sample):
    conversation = [
        {
            "role": "user",
            "content": [
                {"type": "text", "text": instruction},
                {"type": "text", "text": sample["text"]},
                {"type": "image", "image": sample["image"]},
            ],
        },
        {"role": "assistant", "content": [{"type": "text", "text": sample["label"]}]},
    ]
    return {"messages": conversation}
pass

converted_dataset_dev = [convert_to_conversation(sample) for sample in dataset['dev']]

from unsloth import FastVisionModel

model, processor = FastVisionModel.from_pretrained(
    model_name = "NYUAD-ComNets/Gemma4_meme_classification", # Load clean base
    load_in_4bit = True,
)

FastVisionModel.for_inference(model)

lis=[]
pred=[]

for k in range(len(converted_dataset_dev)):

        sample=converted_dataset_dev[k]['messages'][0]['content']
        
        messages = [
            {
                "role": "user",
                "content": [
                     {"type": "text", "text": sample[0]['text']},
                    {
                        "type": "text",
                        "text": sample[1]['text'],
                    },
                    {
                        "type": "image",
                        "image":sample[2]['image'].convert("RGB")
                    },
                ],
            },
        ]
        input_text = processor.apply_chat_template(messages, add_generation_prompt = True)
        inputs = processor(
            sample[2]['image'].convert("RGB"),
            input_text,
            add_special_tokens = False,
            return_tensors = "pt",
        ).to("cuda")
        
        from transformers import TextStreamer
        
        text_streamer = TextStreamer(processor.tokenizer, skip_prompt = True)
        result = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 4,
                           use_cache = True, temperature = 0.1, top_p = 0.95, top_k = 64)

        lab=dataset['dev'][k]['label']
    
        clean_result = result[result != 258880]
        res=processor.tokenizer.decode(clean_result, skip_special_tokens=True).split("model\n")[-1].strip()
        lis.append(lab)
        pred.append(res)
        d=pd.DataFrame({'lab':lis,'pred':pred})
        print(sum(d.lab==d.pred))

We used Low-Rank Adaptation (LoRA) as the Parameter-Efficient Fine-Tuning (PEFT) method for fine-tuning utilizing the unsloth framework.

BibTeX entry and citation info

@misc{aldahoul,
      title={NYUAD at ArGuard Shared Task: Multimodal Embedding Models for
Detecting Arabic Hateful Memes and Unsafe Prompts}, 
      author={Nouar AlDahoul and Yasir Zaki},
      year={2026},
      eprint={},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={}, 
}