DeepSeek-R1-Distill-Qwen-1.5B-ONNX-INT4-CPU

DeepSeek-R1-Distill-Qwen-1.5B-INT4-ONNX

<b><span style="text-decoration:underline">Note: This is unoffical version,just for test and dev.</span></b>

Sample

import onnxruntime_genai as og
import numpy as np
import os


model_folder = "Your DeepSeek-R1-Distill-Qwen-1.5B-INT4-ONNX location"


model = og.Model(model_folder)


tokenizer = og.Tokenizer(model)
tokenizer_stream = tokenizer.create_stream()


search_options = {}
search_options['max_length'] = 2048
search_options['past_present_share_buffer'] = False


chat_template = "<|im_start|><|user|>{input}<|im_end|>\n<|assistant|>"


text = """ Explain 'The goat grazing problem'"""


prompt = f'{chat_template.format(input=text)}'


input_tokens = tokenizer.encode(prompt)


params = og.GeneratorParams(model)


params.set_search_options(**search_options)
params.input_ids = input_tokens


generator = og.Generator(model, params)


while not generator.is_done():
      generator.compute_logits()
      generator.generate_next_token()

      new_token = generator.get_next_tokens()[0]
      print(tokenizer_stream.decode(new_token), end='', flush=True)