flame_waterfall_7b

We open-sourced Flame-Waterfall-7B, a model built by connecting DeepSeek-Coder-7B-Instruct and the SigLIP vision encoder with a 2-layer MLP, and instruction-tuned on the Flame-Code-VLM/Flame-Waterfall-React dataset. This model is released to showcase the value of the synthesized dataset. However, it is not intended for general-purpose tasks. Please use it with caution.

Generation

The following is the sample code for inference.

# pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git
# Replace the corresponding code files in the original repository with those in https://github.com/Flame-Code-VLM/Flame-Code-VLM/tree/main/model
# export PYTHONPATH="/your_path_to_LLaVA-NeXT_repo:$PYTHONPATH"

from llava.model.builder import load_pretrained_model
from llava.mm_utils import process_images, tokenizer_image_token
from llava.constants import DEFAULT_IMAGE_TOKEN

from PIL import Image
import torch
import warnings

warnings.filterwarnings("ignore")

pretrained = "Flame-Code-VLM/flame_waterfall_7b"

model_name = "flame"
device = "cuda"
device_map = "auto"
llava_model_args = {
    "multimodal": True,
    "attn_implementation": None,
}
tokenizer, model, image_processor, max_length = load_pretrained_model(pretrained, None, model_name, device_map=device_map,**llava_model_args)
model.config.tokenizer_padding_side = 'left'  # Use left padding for batch processing
model.eval()

url = "path_to_your_screenshot_image_file"
image = Image.open(url) 
image_tensor = process_images([image], image_processor, model.config)
image_tensor = [_image.to(dtype=torch.float16, device=device) for _image in image_tensor]

prompt = "Below is an image of the page to create. Generate React code and styles to replicate the design, including layout, typography, and styling. Format your response as follows:'// CSS\n[CSS/SCSS code]\n\n// [React Implementation (JS/TS/JSX/TSX)]\n[Component code]'.\n\n ### Input Image:\n{image}\n\n### Response:\n"

input_ids = tokenizer_image_token(prompt, tokenizer, return_tensors='pt')
input_ids = input_ids.unsqueeze(0)
input_ids=input_ids.to(device)
image_sizes = [image.size]
modalities = ["image"] 

cont = model.generate(
    input_ids,
    images=image_tensor,
    image_sizes=image_sizes,
    modalities=modalities,      # Added this line with the modalities
    do_sample=True, 
    temperature=0.1,
    max_new_tokens=4096,
    top_p=0.95,
    repetition_penalty=1.05
)

text_outputs = tokenizer.batch_decode(cont, skip_special_tokens=True)