BERTopic-Phi-3

BERTopic-Phi-3

This is a BERTopic model. BERTopic is a flexible and modular topic modeling framework that allows for the generation of easily interpretable topics from large datasets.

Usage

To use this model, please install BERTopic:

pip install -U bertopic

You can use the model as follows:

from bertopic import BERTopic
topic_model = BERTopic.load("Tsunnami/BERTopic-Phi-3")

topic_model.get_topic_info()

Topic overview

• Number of topics: 370
• Number of training documents: 19550

<details> <summary>Click here for an overview of all topics.</summary> </details>

Training Procedure

The model was trained as follows:

from bertopic import BERTopic

from sentence_transformers import SentenceTransformer

from umap import UMAP
from hdbscan import HDBSCAN
from sklearn.feature_extraction.text import CountVectorizer
from bertopic.representation import PartOfSpeech, KeyBERTInspired, MaximalMarginalRelevance, ZeroShotClassification

embedding_model = SentenceTransformer("all-mpnet-base-v2")
umap_model = UMAP(n_neighbors=25, n_components=5, min_dist=0.0, metric='cosine', random_state=42, verbose=True) #change n_neightbor, n_components, metric
hdbscan_model = HDBSCAN(min_cluster_size=20, min_samples=20, metric='euclidean', cluster_selection_method='eom', prediction_data=True) #change min_cluster_size, min_samples
vectorizer_model = CountVectorizer(stop_words="english", ngram_range=(1, 3), min_df=5)


representation_models = {
    "POS": PartOfSpeech("en_core_web_lg"),
    "KeyBERTInspired": KeyBERTInspired(),
    "MMR": MaximalMarginalRelevance(diversity=0.3),
    "KeyBERT + MMR": [KeyBERTInspired(), MaximalMarginalRelevance(diversity=0.3)],
    "Summarization": summarization, # Own Prompted Model as to Summarize.
}

topic_model = BERTopic(
    language="english", 
    embedding_model=embedding_model,
    umap_model=umap_model,
    hdbscan_model=hdbscan_model,
    vectorizer_model=vectorizer_model,
    representation_model=representation_models,
    verbose=True,
)
topics, probs = topic_model.fit_transform(docs)

Create Own Representation Model

Using microsoft/Phi-3-mini-128k-instruct for its lightweightness

Defined Summarization

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline

torch.random.manual_seed(42)

summarization_model = AutoModelForCausalLM.from_pretrained(
    "microsoft/Phi-3-mini-128k-instruct", 
    device_map="cuda", 
    torch_dtype="auto", 
    trust_remote_code=True, 
)
summarization_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct")

def summarize_with_model(text):
    question = f"""
        I have a document of which abstract and title are given.
        The following documents are a small but representative subset of all documents in the topic:
        {text}

        Based on the information above, please give a description topic in the following keyword format:
        topic: <description>
    """
    messages = [
        {"role": "user", "content": question},
    ]
    pipe = pipeline(
        "text-generation",
        model=summarization_model,
        tokenizer=summarization_tokenizer,
    )
    generation_args = {
        "max_new_tokens": 128,
        "return_full_text": False,
        "temperature": 0.0,
        "do_sample": False,
    }
    output = pipe(messages, **generation_args)
    return output[0]['generated_text']

Prompt Used,

question = f"""
    I have a document of which abstract and title are given.
    The following documents are a small but representative subset of all documents in the topic:
    {text}

    Based on the information above, please give a description topic in the following keyword format:
    topic: <description>
"""

NOTE: Persuation with other better propmt is recommended

Mounted on Base-Representation

from bertopic.representation._base import BaseRepresentation
from typing import List, Mapping, Tuple

class SummarizationRepresentation(BaseRepresentation):
    def __init__(self, summarization_model, summarization_tokenizer):
        self.summarization_model = summarization_model
        self.summarization_tokenizer = summarization_tokenizer

    def extract_topics(self, topic_model, documents, c_tf_idf, topics
                      ) -> Mapping[str, List[Tuple[str, float]]]:
        updated_topics = {}
        for topic_id, words in topics.items():
            words_only = [word[0] for word in words]
            text = " ".join(words_only)
            summary = summarize_with_model(text)
            updated_topics[topic_id] = [(summary, 1.0)]
        return updated_topics

summarization = SummarizationRepresentation(summarization_model, summarization_tokenizer)