Axion-0.5B-GGUF

GGUF Q4 Model (Quantized 4Bit)

To Try this model using Python

import os
from llama_cpp import Llama

# Set this to either a Hugging Face repo ID or a local .gguf model path.
MODEL_PATH_OR_REPO = "Rajiv714/Axion-0.5B-GGUF"
MODEL_FILENAME = "Axion-0.5B-Q4_K_M.gguf"
N_CTX = 4096
N_THREADS = 8
N_GPU_LAYERS = 0
MAX_TOKENS = 1500
TEMPERATURE = 0.3
TOP_P = 0.9
SYSTEM_PROMPT = " "
USER_PROMPT = "I am having a fever for the past 10 days. What should I do?"

def load_model(model_path_or_repo):
    if os.path.isfile(model_path_or_repo):
        return Llama(
            model_path=model_path_or_repo,
            n_ctx=N_CTX,
            n_threads=N_THREADS,
            n_gpu_layers=N_GPU_LAYERS,
            verbose=False,
        )
    print(f"Loading {MODEL_FILENAME} from {model_path_or_repo}...")

    return Llama.from_pretrained(
        repo_id=model_path_or_repo,
        filename=MODEL_FILENAME,
        n_ctx=N_CTX,
        n_threads=N_THREADS,
        n_gpu_layers=N_GPU_LAYERS,
        verbose=False,
    )

def main():
    llm = load_model(MODEL_PATH_OR_REPO)
    print("--- MODEL RESPONSE ---")

    response = llm.create_chat_completion(
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": USER_PROMPT},
        ],
        max_tokens=MAX_TOKENS,
        temperature=TEMPERATURE,
        top_p=TOP_P,
    )
    print(response["choices"][0]["message"]["content"])
    print("\n--- END RESPONSE ---")

if __name__ == "__main__":
    main()