Bonsai-8B-1bit-GGUF-colab-prebuilt-cpu

Bonsai-8B (GGUF) – Run in 30 Seconds on CPU 🚀

⚡ Quick Start

🔧 Setup (run once)

# ============================================================
# CELL 1: INSTANT SETUP (Run this once)
# Downloads pre-built CPU files from Zlib2's Hugging Face repo
# ============================================================

import os
import tarfile
import subprocess

print("📦 Installing dependencies...")
!pip install -q huggingface_hub
from huggingface_hub import hf_hub_download

# Your Hugging Face Repo (CPU Version)
HF_REPO = "Zlib2/Bonsai-8B-1bit-GGUF-colab-prebuilt-cpu"

print("\n📥 1/3 Downloading pre-built llama.cpp (CPU) (Fast)...")
llama_zip = hf_hub_download(repo_id=HF_REPO, filename="llama_cpp_prebuilt_cpu.tar.gz")

print("📥 2/3 Downloading Bonsai-8B.gguf model (Large file, takes 1-2 mins)...")
model_path = hf_hub_download(repo_id=HF_REPO, filename="Bonsai-8B.gguf")

print("📦 3/3 Extracting files and setting permissions...")
!mkdir -p /content/llama.cpp
with tarfile.open(llama_zip, "r:gz") as tar:
    tar.extractall(path="/content/llama.cpp")

# Make the compiled binaries executable
!chmod +x /content/llama.cpp/build/bin/llama-cli
!chmod +x /content/llama.cpp/build/bin/llama-server 2>/dev/null

# Fix shared library path
print("🔗 Setting up shared libraries...")
so_dirs = subprocess.run(
    ['find', '/content/llama.cpp/build', '-name', '*.so*', '-exec', 'dirname', '{}', ';'],
    capture_output=True, text=True
).stdout.strip().split('\n')

so_dirs = list(set([d for d in so_dirs if d]))

if so_dirs:
    lib_path = ':'.join(so_dirs)
    os.environ['LD_LIBRARY_PATH'] = lib_path + ':' + os.environ.get('LD_LIBRARY_PATH', '')
    print(f"   ✅ Library path set:")
    for d in so_dirs:
        print(f"      → {d}")
else:
    print("   ⚠️ No shared libraries found (may be statically linked)")

print("\n" + "="*70)
print("🎉 SETUP COMPLETE! You are ready to run inference (CPU).")
print("="*70)

---

🤖 Run Inference

# ============================================================
# CELL 2: RUN INFERENCE (CPU ONLY)
# Change the USER_PROMPT to ask different questions
# ============================================================

# ⚠️ CHANGE YOUR QUESTION HERE ⚠️
USER_PROMPT = "Explain quantum computing in simple terms."

SYSTEM_PROMPT = "You are a helpful assistant"

print("🚀 Running inference (CPU only)...\n")

!/content/llama.cpp/build/bin/llama-cli \
    -m "{model_path}" \
    --system-prompt "{SYSTEM_PROMPT}" \
    -p "{USER_PROMPT}" \
    -n 4096 \
    --temp 0.5 \
    --top-p 0.85 \
    --top-k 20