mlc-chat-TinyLlama-1.1B-1T-OpenOrca-q4f32_1

TinyLlama-1.1B-1T-OpenOrca for Web-LLM q4f32_1

This is a compiled version of TinyLlama-1.1B-1T-OpenOrca for MLC Web-LLM, using q4f32_1 quantization.

Usage

import * as webLLM from "@mlc-ai/web-llm";

const modelId = "TinyLlama-1.1B-1T-OpenOrca-q4f32_1";

const appConfig = {
  model_list: [
    {
      model_url:
        "https://huggingface.co/Felladrin/mlc-chat-TinyLlama-1.1B-1T-OpenOrca-q4f32_1/resolve/main/params/",
      local_id: modelId,
    },
  ],
  model_lib_map: {
    [modelId]:
      "https://huggingface.co/Felladrin/mlc-chat-TinyLlama-1.1B-1T-OpenOrca-q4f32_1/resolve/main/TinyLlama-1.1B-1T-OpenOrca-q4f32_1-webgpu.wasm",
  },
};

const chatConfig = {
  temperature: 0.7,
  repetition_penalty: 1.2,
  top_p: 0.95
};

async function main() {
  const chat = new webLLM.ChatModule();

  await chat.reload(modelId, chatConfig, appConfig);

  let lastResponse = "";

  const generateProgressCallback = (_, message = "") => {
    if (message.length === 0) return chat.interruptGenerate();

    lastResponse = message;

    console.log(`Partial response: ${lastResponse}`);
  };

  const fistPrompt = "Hi! I have a question.";
  await chat.generate(fistPrompt, generateProgressCallback);
  console.log(`Complete response: ${lastResponse}`);

  const secondPrompt =
    "How to get in a good university?";
  await chat.generate(secondPrompt, generateProgressCallback);
  console.log(`Complete response: ${lastResponse}`);

  console.info(await chat.runtimeStatsText());
}

main();