实战 | 猫猫、少女、FLUX、ControlNet

git clone https://github.com/modelscope/DiffSynth-Studio.git
cd DiffSynth-Studio
pip install -e .

python examples/ControlNet/flux_controlnet.py

from diffsynth import ModelManager, FluxImagePipeline, ControlNetConfigUnit, download_customized_models
import torch
from PIL import Image
import numpy as np
model_manager = ModelManager(torch_dtype=torch.bfloat16, model_id_list=[
    "FLUX.1-dev",
    "InstantX/FLUX.1-dev-Controlnet-Union-alpha",
    "alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta",
    "jasperai/Flux.1-dev-Controlnet-Upscaler",
])
pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[
    ControlNetConfigUnit(
        processor_id="inpaint",
        model_path="models/ControlNet/alimama-creative/FLUX.1-dev-Controlnet-Inpainting-Beta/diffusion_pytorch_model.safetensors",
        scale=0.9
    ),
    ControlNetConfigUnit(
        processor_id="canny",
        model_path="models/ControlNet/InstantX/FLUX.1-dev-Controlnet-Union-alpha/diffusion_pytorch_model.safetensors",
        scale=0.5
    ),
])
image_1 = pipe(
    prompt="a beautiful Asian woman and a cat on a chair. The woman wears a T-shirt.",
    height=1024, width=1024,
    seed=1000
)
image_1.save("image_13.jpg")
mask_global = np.zeros((1024, 1024, 3), dtype=np.uint8)
mask_global = Image.fromarray(mask_global)
mask_global.save("mask_13_global.jpg")
mask_1 = np.zeros((1024, 1024, 3), dtype=np.uint8)
mask_1[400: -10, 50: 440] = 255
mask_1 = Image.fromarray(mask_1)
mask_1.save("mask_13_1.jpg")
mask_2 = np.zeros((1024, 1024, 3), dtype=np.uint8)
mask_2[500:, 440: -140] = 255
mask_2 = Image.fromarray(mask_2)
mask_2.save("mask_13_2.jpg")
image_2 = pipe(
    prompt="a beautiful Asian woman and a cat on a chair. The woman wears a T-shirt.",
    controlnet_image=image_1, controlnet_inpaint_mask=mask_global,
    local_prompts=["an gray cat, highly detailed", "a girl wearing a pink T-shirt, pure color pink T-shirt"], masks=[mask_1, mask_2], mask_scales=[10.0, 100.0],
    height=1024, width=1024,
    seed=1010
)
image_2.save("image_14.jpg")
model_manager.load_lora("models/lora/FLUX-dev-lora-AntiBlur.safetensors", lora_alpha=1)
image_3 = pipe(
    prompt="a beautiful Asian woman wearing a pink T-shirt and an gray cat on a chair. clear background.",
    negative_prompt="blur, blurry",
    input_image=image_2, denoising_strength=0.7,
    height=1024, width=1024,
    cfg_scale=2.0, num_inference_steps=50,
    seed=10201
)
image_3.save("image_15.jpg")
pipe = FluxImagePipeline.from_model_manager(model_manager, controlnet_config_units=[
    ControlNetConfigUnit(
        processor_id="tile",
        model_path="models/ControlNet/jasperai/Flux.1-dev-Controlnet-Upscaler/diffusion_pytorch_model.safetensors",
        scale=0.7
    ),
])
image_4 = pipe(
    prompt="a beautiful Asian woman wearing a pink T-shirt and an gray cat on a chair. highly detailed, delicate skin texture, clear background.",
    controlnet_image=image_3.resize((2048, 2048)),
    input_image=image_3.resize((2048, 2048)), denoising_strength=0.99,
    height=2048, width=2048, tiled=True,
    seed=103
)
image_4.save("image_16.jpg")
image_5 = pipe(
    prompt="a beautiful Asian woman wearing a pink T-shirt and an gray cat on a chair. highly detailed, delicate skin texture, clear background.",
    controlnet_image=image_4.resize((4096, 4096)),
    input_image=image_4.resize((4096, 4096)), denoising_strength=0.99,
    height=4096, width=4096, tiled=True,
    seed=104
)
image_5.save("image_17.jpg")