Multimodal Stable Diffusion 高级用法

# Stable Diffusion 高级用法指南 ## 自定义工作流 ### 从组件构建 python from diffusers import ( UNet2DConditionModel, AutoencoderKL, DDPMScheduler, StableDiffusionPipeline ) from transformers import CLIPTextModel, CLIPTokenizer import torch # 单独加载组件 unet = UNet2DConditionModel.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", subfolder="unet" ) vae = AutoencoderKL.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", subfolder="vae" ) text_encoder = CLIPTextModel.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", subfolder="text_encoder" ) tokenizer = CLIPTokenizer.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", subfolder="tokenizer" ) scheduler = DDPMScheduler.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", subfolder="scheduler" ) # 组装工作流 pipe = StableDiffusionPipeline( unet=unet, vae=vae, text_encoder=text_encoder, tokenizer=tokenizer, scheduler=scheduler, safety_checker=None, feature_extractor=None, requires_safety_checker=False ) ### 自定义去噪循环 python from diffusers import DDIMScheduler, AutoencoderKL, UNet2DConditionModel from transformers import CLIPTextModel, CLIPTokenizer import torch def custom_generate( prompt: str, num_steps: int = 50, guidance_scale: float = 7.5, height: int = 512, width: int = 512 ): # 加载组件 tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") text_encoder = CLIPTextModel.from_pretrained("openai/clip-vit-large-patch14") unet = UNet2DConditionModel.from_pretrained("sd-model", subfolder="unet") vae = AutoencoderKL.from_pretrained("sd-model", subfolder="vae") scheduler = DDIMScheduler.from_pretrained("sd-model", subfolder="scheduler") device = "cuda" text_encoder.to(device) unet.to(device) vae.to(device) # 编码提示词 text_input = tokenizer( prompt, padding="max_length", max_length=77, truncation=True, return_tensors="pt" ) text_embeddings = text_encoder(text_input.input_ids.to(device))[0] # 用于无分类器引导的无条件嵌入 uncond_input = tokenizer( "", padding="max_length", max_length=77, return_tensors="pt" ) uncond_embeddings = text_encoder(uncond_input.input_ids.to(dev