fineweb-1.5b-checkpoints

FineWeb 1.5B

Custom GPT Decoder Model

Usage

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
from peft import PeftModel

base_model_id = "Adit1Sharma/fineweb-1.5b-checkpoints"
adapter_id = "Adit1Sharma/fineweb-1.5b-final-sft-adapter"

# 1. Load the tokenizer
tokenizer = AutoTokenizer.from_pretrained("gpt2")

# 2. Load the base model
print("Loading base model...")
base_model = AutoModelForCausalLM.from_pretrained(
    base_model_id, 
    trust_remote_code=True, 
    device_map="auto"
)

# 3. Apply the SFT Adapter dynamically
print("Applying SFT adapter...")
model = PeftModel.from_pretrained(
    base_model, 
    adapter_id
)

# 4. Set up the pipeline with sampling parameters
model.generation_config.max_length = None  # kill the stale max_length=20

pipe = pipeline(
    task="text-generation",
    model=model,
    tokenizer=tokenizer,
    max_new_tokens=200,
    pad_token_id=tokenizer.eos_token_id,
    do_sample=True,
    temperature=0.6,
    top_p=0.6,
    top_k=40,
    repetition_penalty=1.1,
    no_repeat_ngram_size=3,
)
# 5. Generate Text
prompt = "Artificial intelligence is"
output = pipe(prompt)
print(output[0]['generated_text'])