Skip to content

feat: Add explicit mmproj input to GGUF CLIP loaders #391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sebagallo wants to merge 1 commit into
base: main
Choose a base branch
from
+68 −47
Open

feat: Add explicit mmproj input to GGUF CLIP loaders #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 5 additions & 31 deletions loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,34 +219,8 @@ def strip_quant_suffix(name):

def gguf_mmproj_loader(path):
# Reverse version of Qwen2VLVisionModel.modify_tensors
logging.info("Attenpting to find mmproj file for text encoder...")

# get name to match w/o quant suffix
tenc_fname = os.path.basename(path)
tenc = os.path.splitext(tenc_fname)[0].lower()
tenc = strip_quant_suffix(tenc)

# try and find matching mmproj
target = []
root = os.path.dirname(path)
for fname in os.listdir(root):
name, ext = os.path.splitext(fname)
if ext.lower() != ".gguf":
continue
if "mmproj" not in name.lower():
continue
if tenc in name.lower():
target.append(fname)

if len(target) == 0:
logging.error(f"Error: Can't find mmproj file for '{tenc_fname}' (matching:'{tenc}')! Qwen-Image-Edit will be broken!")
return {}
if len(target) > 1:
logging.error(f"Ambiguous mmproj for text encoder '{tenc_fname}', will use first match.")

logging.info(f"Using mmproj '{target[0]}' for text encoder '{tenc_fname}'.")
target = os.path.join(root, target[0])
vsd = gguf_sd_loader(target, is_text_model=True)
logging.info(f"Loading mmproj from {path}")
vsd = gguf_sd_loader(path, is_text_model=True)

# concat 4D to 5D
if "v.patch_embd.weight.1" in vsd:
Expand Down Expand Up @@ -374,7 +348,7 @@ def gguf_tekken_tokenizer_loader(path, temb_shape):
del reader
return torch.ByteTensor(list(json.dumps(data).encode('utf-8')))

def gguf_clip_loader(path):
def gguf_clip_loader(path, mmproj_path=None):
sd, arch = gguf_sd_loader(path, return_arch=True, is_text_model=True)
if arch in {"t5", "t5encoder"}:
temb_key = "token_embd.weight"
Expand All @@ -398,8 +372,8 @@ def gguf_clip_loader(path):
sd = sd_map_replace(sd, LLAMA_SD_MAP)
if arch == "llama":
sd = llama_permute(sd, 32, 8) # L3 / Mistral
if arch == "qwen2vl":
vsd = gguf_mmproj_loader(path)
if arch == "qwen2vl" and mmproj_path:
vsd = gguf_mmproj_loader(mmproj_path)
sd.update(vsd)
else:
pass
Expand Down
79 changes: 63 additions & 16 deletions nodes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ def INPUT_TYPES(s):
"required": {
"clip_name": (s.get_filename_list(),),
"type": base["required"]["type"],
"mmproj_name": (["none"] + s.get_mmproj_list(),),
}
}

Expand All @@ -213,11 +214,25 @@ def get_filename_list(s):
files += folder_paths.get_filename_list("clip_gguf")
return sorted(files)

def load_data(self, ckpt_paths):
@classmethod
def get_mmproj_list(s):
files = []
for x in folder_paths.get_filename_list("clip"):
if "mmproj" in x.lower():
files.append(x)
for x in folder_paths.get_filename_list("clip_gguf"):
if "mmproj" in x.lower():
files.append(x)
return sorted(files)

def load_data(self, ckpt_paths, mmproj_paths=None):
if mmproj_paths is None:
mmproj_paths = [None] * len(ckpt_paths)

clip_data = []
for p in ckpt_paths:
for p, mp in zip(ckpt_paths, mmproj_paths):
if p.endswith(".gguf"):
sd = gguf_clip_loader(p)
sd = gguf_clip_loader(p, mmproj_path=mp)
else:
sd = comfy.utils.load_torch_file(p, safe_load=True)
if "scaled_fp8" in sd: # NOTE: Scaled FP8 would require different custom ops, but only one can be active
Expand All @@ -238,10 +253,12 @@ def load_patcher(self, clip_paths, clip_type, clip_data):
clip.patcher = GGUFModelPatcher.clone(clip.patcher)
return clip

def load_clip(self, clip_name, type="stable_diffusion"):
def load_clip(self, clip_name, type="stable_diffusion", mmproj_name="none"):
if mmproj_name == "none": mmproj_name = None
clip_path = folder_paths.get_full_path("clip", clip_name)
mmproj_path = folder_paths.get_full_path("clip", mmproj_name) if mmproj_name is not None else None
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)
return (self.load_patcher([clip_path], clip_type, self.load_data([clip_path])),)
return (self.load_patcher([clip_path], clip_type, self.load_data([clip_path], mmproj_paths=[mmproj_path])),)

class DualCLIPLoaderGGUF(CLIPLoaderGGUF):
@classmethod
Expand All @@ -251,19 +268,26 @@ def INPUT_TYPES(s):
return {
"required": {
"clip_name1": file_options,
"mmproj_name1": (["none"] + s.get_mmproj_list(),),
"clip_name2": file_options,
"mmproj_name2": (["none"] + s.get_mmproj_list(),),
"type": base["required"]["type"],
}
}

TITLE = "DualCLIPLoader (GGUF)"

def load_clip(self, clip_name1, clip_name2, type):
def load_clip(self, clip_name1, clip_name2, type, mmproj_name1="none", mmproj_name2="none"):
if mmproj_name1 == "none": mmproj_name1 = None
if mmproj_name2 == "none": mmproj_name2 = None
clip_path1 = folder_paths.get_full_path("clip", clip_name1)
clip_path2 = folder_paths.get_full_path("clip", clip_name2)
clip_paths = (clip_path1, clip_path2)
mmproj_path1 = folder_paths.get_full_path("clip", mmproj_name1) if mmproj_name1 is not None else None
mmproj_path2 = folder_paths.get_full_path("clip", mmproj_name2) if mmproj_name2 is not None else None
mmproj_paths = (mmproj_path1, mmproj_path2)
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)
return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),)
return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths, mmproj_paths=mmproj_paths)),)

class TripleCLIPLoaderGGUF(CLIPLoaderGGUF):
@classmethod
Expand All @@ -272,44 +296,67 @@ def INPUT_TYPES(s):
return {
"required": {
"clip_name1": file_options,
"mmproj_name1": (["none"] + s.get_mmproj_list(),),
"clip_name2": file_options,
"mmproj_name2": (["none"] + s.get_mmproj_list(),),
"clip_name3": file_options,
"mmproj_name3": (["none"] + s.get_mmproj_list(),),
}
}

TITLE = "TripleCLIPLoader (GGUF)"

def load_clip(self, clip_name1, clip_name2, clip_name3, type="sd3"):
def load_clip(self, clip_name1, clip_name2, clip_name3, type="sd3", mmproj_name1="none", mmproj_name2="none", mmproj_name3="none"):
if mmproj_name1 == "none": mmproj_name1 = None
if mmproj_name2 == "none": mmproj_name2 = None
if mmproj_name3 == "none": mmproj_name3 = None
clip_path1 = folder_paths.get_full_path("clip", clip_name1)
clip_path2 = folder_paths.get_full_path("clip", clip_name2)
clip_path3 = folder_paths.get_full_path("clip", clip_name3)
clip_paths = (clip_path1, clip_path2, clip_path3)
mmproj_path1 = folder_paths.get_full_path("clip", mmproj_name1) if mmproj_name1 is not None else None
mmproj_path2 = folder_paths.get_full_path("clip", mmproj_name2) if mmproj_name2 is not None else None
mmproj_path3 = folder_paths.get_full_path("clip", mmproj_name3) if mmproj_name3 is not None else None
mmproj_paths = (mmproj_path1, mmproj_path2, mmproj_path3)
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)
return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),)
return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths, mmproj_paths=mmproj_paths)),)

class QuadrupleCLIPLoaderGGUF(CLIPLoaderGGUF):
@classmethod
def INPUT_TYPES(s):
file_options = (s.get_filename_list(), )
return {
"required": {
"clip_name1": file_options,
"clip_name2": file_options,
"clip_name3": file_options,
"clip_name4": file_options,
"clip_name1": file_options,
"mmproj_name1": (["none"] + s.get_mmproj_list(),),
"clip_name2": file_options,
"mmproj_name2": (["none"] + s.get_mmproj_list(),),
"clip_name3": file_options,
"mmproj_name3": (["none"] + s.get_mmproj_list(),),
"clip_name4": file_options,
"mmproj_name4": (["none"] + s.get_mmproj_list(),),
}
}
}

TITLE = "QuadrupleCLIPLoader (GGUF)"

def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4, type="stable_diffusion"):
def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4, type="stable_diffusion", mmproj_name1="none", mmproj_name2="none", mmproj_name3="none", mmproj_name4="none"):
if mmproj_name1 == "none": mmproj_name1 = None
if mmproj_name2 == "none": mmproj_name2 = None
if mmproj_name3 == "none": mmproj_name3 = None
if mmproj_name4 == "none": mmproj_name4 = None
clip_path1 = folder_paths.get_full_path("clip", clip_name1)
clip_path2 = folder_paths.get_full_path("clip", clip_name2)
clip_path3 = folder_paths.get_full_path("clip", clip_name3)
clip_path4 = folder_paths.get_full_path("clip", clip_name4)
clip_paths = (clip_path1, clip_path2, clip_path3, clip_path4)
mmproj_path1 = folder_paths.get_full_path("clip", mmproj_name1) if mmproj_name1 is not None else None
mmproj_path2 = folder_paths.get_full_path("clip", mmproj_name2) if mmproj_name2 is not None else None
mmproj_path3 = folder_paths.get_full_path("clip", mmproj_name3) if mmproj_name3 is not None else None
mmproj_path4 = folder_paths.get_full_path("clip", mmproj_name4) if mmproj_name4 is not None else None
mmproj_paths = (mmproj_path1, mmproj_path2, mmproj_path3, mmproj_path4)
clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION)
return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),)
return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths, mmproj_paths=mmproj_paths)),)

NODE_CLASS_MAPPINGS = {
"UnetLoaderGGUF": UnetLoaderGGUF,
Expand Down