pmc-falcon-7b

This model is built as a chatbot for breast cancer related queries, trained only on data samples relating to breast cancer. falcon-7b-instruct-sharded model fine-tuned using QLoRA. github

Loading the model for inference:

import torch
from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig


PEFT_MODEL = "manik1080/pmc-falcon-7b"

config = PeftConfig.from_pretrained(PEFT_MODEL)
model_ = AutoModelForCausalLM.from_pretrained(
    config.base_model_name_or_path,
    return_dict=True,
    quantization_config=bnb_config,
    device_map="auto",
    trust_remote_code=True
)

tokenizer=AutoTokenizer.from_pretrained(config.base_model_name_or_path)
tokenizer.pad_token = tokenizer.eos_token

model_ = PeftModel.from_pretrained(model_, PEFT_MODEL)

Example inference:

device = "cuda:0" if torch.cuda.is_available() else "cpu"

prompt = """
<human>: Is a prior diagnosis of breast cancer a risk factor for breast cancer in BRCA1 and BRCA2 carriers?
<assistant>:
""".strip()

encoding = tokenizer(prompt, return_tensors="pt").to(device)
with torch.inference_mode():
  outputs = model_.generate(
      input_ids = encoding.input_ids,
      attention_mask = encoding.attention_mask,
      generation_config = generation_config
  )

outp = tokenizer.decode(outputs[0], skip_special_tokens=True)

print("User Query: ", prompt.strip('<assistant>: ')[0].lstrip('<human>'))
print("Bot Response: ", outp.strip('<assistant>: ')[1].rstrip('User'))