Flux-ControlNet-ArtisticText

Color-Patette-Flux_dev

Inference

<Gallery />

import torch
import cv2
from PIL import Image
import numpy as np
from diffusers.utils import load_image
from diffusers.pipelines.flux.pipeline_flux_controlnet import FluxControlNetPipeline
from diffusers.models.controlnet_flux import FluxControlNetModel

controlnet_model_path = './flux_controlnet_artistic_text'
controlnet = FluxControlNetModel.from_pretrained(controlnet_model, torch_dtype=torch.bfloat16)
pipe = FluxControlNetPipeline.from_pretrained('black-forest-labs/FLUX.1-dev',
                                              controlnet=controlnet,
                                              torch_dtype=torch.bfloat16).to("cuda")


font_mask_pil = Image.open("pictures/A.png").convert("RGB")
font_mask_npy = np.array(font_mask_pil)

prompt = "Vibrant, multicolored lettering against a soft, pastel background, with the letters appearing to be made of delicate petals and blooming flowers, giving a sense of freshness and natural beauty. The texture should mimic the intricate layers and velvety surfaces of various blossoms, with subtle gradients and occasional dewdrops enhancing the lifelike appearance."
image = pipe(prompt,
             control_image=font_mask_pil,
             controlnet_conditioning_scale=0.6,
             num_inference_steps=30,
             guidance_scale=3.5,
             generator=torch.Generator("cuda").manual_seed(42)).images[0]
rgba = Image.fromarray(np.concatenate([np.array(image), cv2.resize(font_mask_npy, (1024, 1024))[..., :1]], axis=-1))
rgba.save("./{}.png".format(datetime.now().strftime("%Y%m%d%H%M%S")))

Training

Training was done using https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet_flux.py