From af94203d1b461d934ca1c44211ca0f71a5d05d48 Mon Sep 17 00:00:00 2001 From: melMass Date: Tue, 10 Oct 2023 11:44:14 +0200 Subject: [PATCH 01/12] =?UTF-8?q?feat:=20=E2=9C=A8=20add=20batch=20shake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit applies "camera shake" using Brownian Noise --- nodes/batch.py | 296 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 290 insertions(+), 6 deletions(-) diff --git a/nodes/batch.py b/nodes/batch.py index ade068f..108f94d 100644 --- a/nodes/batch.py +++ b/nodes/batch.py @@ -1,3 +1,4 @@ +import math import os from pathlib import Path from typing import List @@ -7,6 +8,7 @@ import numpy as np import torch +from ..log import log from ..utils import apply_easing, pil2tensor from .transform import TransformImage @@ -252,6 +254,57 @@ def set_floats(self, mode, count, min, max, easing): return (keyframes,) +class BatchMerge: + """Merges multiple image batches with different frame counts""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "fusion_mode": (["add", "multiply", "average"], {"default": "average"}), + "fill": (["head", "tail"], {"default": "tail"}), + } + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "merge_batches" + CATEGORY = "mtb/batch" + + def merge_batches(self, fusion_mode, fill, **kwargs): + images = kwargs.values() + max_frames = max(img.shape[0] for img in images) + + adjusted_images = [] + for img in images: + frame_count = img.shape[0] + if frame_count < max_frames: + fill_frame = img[0] if fill == "head" else img[-1] + fill_frames = fill_frame.repeat(max_frames - frame_count, 1, 1, 1) + adjusted_batch = ( + torch.cat((fill_frames, img), dim=0) + if fill == "head" + else torch.cat((img, fill_frames), dim=0) + ) + else: + adjusted_batch = img + adjusted_images.append(adjusted_batch) + + # Merge the adjusted batches + merged_image = None + for img in adjusted_images: + if merged_image is None: + merged_image = img + else: + if fusion_mode == "add": + merged_image += img + elif fusion_mode == "multiply": + merged_image *= img + elif fusion_mode == "average": + merged_image = (merged_image + img) / 2 + + return (merged_image,) + + class Batch2dTransform: """Transform a batch of images using a batch of keyframes""" @@ -279,6 +332,14 @@ def INPUT_TYPES(cls): FUNCTION = "transform_batch" CATEGORY = "mtb/batch" + def get_num_elements(self, param) -> int: + if isinstance(param, torch.Tensor): + return torch.numel(param) + elif isinstance(param, list): + return len(param) + + return 0 + def transform_batch( self, image: torch.Tensor, @@ -290,22 +351,24 @@ def transform_batch( angle=None, shear=None, ): - if not any([x, y, zoom, angle]): + if all( + self.get_num_elements(param) <= 0 for param in [x, y, zoom, angle, shear] + ): raise ValueError("At least one transform parameter must be provided") keyframes = {"x": [], "y": [], "zoom": [], "angle": [], "shear": []} default_vals = {"x": 0, "y": 0, "zoom": 1.0, "angle": 0, "shear": 0} - if x: + if self.get_num_elements(x) > 0: keyframes["x"] = x - if y: + if self.get_num_elements(y) > 0: keyframes["y"] = y - if zoom: + if self.get_num_elements(zoom) > 0: keyframes["zoom"] = zoom - if angle: + if self.get_num_elements(angle) > 0: keyframes["angle"] = angle - if shear: + if self.get_num_elements(shear) > 0: keyframes["shear"] = shear for name, values in keyframes.items(): @@ -334,6 +397,225 @@ def transform_batch( return (torch.cat(res, dim=0),) +DEFAULT_INTERPOLANT = lambda t: t * t * t * (t * (t * 6 - 15) + 10) + + +class BatchShake: + """Applies a shaking effect to batches of images.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "position_amount_x": ("FLOAT", {"default": 1.0}), + "position_amount_y": ("FLOAT", {"default": 1.0}), + "rotation_amount": ("FLOAT", {"default": 10.0}), + "frequency": ("FLOAT", {"default": 1.0}), + "frequency_divider": ("FLOAT", {"default": 1.0}), + "octaves": ("INT", {"default": 2}), + "seed": ("INT", {"default": 0}), + } + } + + RETURN_TYPES = ("IMAGE", "FLOATS", "FLOATS", "FLOATS") + RETURN_NAMES = ("image", "pos_x", "pos_y", "rot") + FUNCTION = "apply_shake" + CATEGORY = "mtb/batch" + + def rehash(self, seed, octaves): + np.random.seed(seed) + self.position_offset = np.random.uniform(-1e3, 1e3, 3) + self.rotation_offset = np.random.uniform(-1e3, 1e3, 3) + self.noise_pattern = self.generate_fractal_noise_2d( + (512, 512), (32, 32), octaves + ) + + # def interpolant(self, t): + # return t * t * t * (t * (t * 6 - 15) + 10) + + def generate_perlin_noise_2d( + self, shape, res, tileable=(False, False), interpolant=None + ): + """Generate a 2D numpy array of perlin noise. + + Args: + shape: The shape of the generated array (tuple of two ints). + This must be a multple of res. + res: The number of periods of noise to generate along each + axis (tuple of two ints). Note shape must be a multiple of + res. + tileable: If the noise should be tileable along each axis + (tuple of two bools). Defaults to (False, False). + interpolant: The interpolation function, defaults to + t*t*t*(t*(t*6 - 15) + 10). + + Returns: + A numpy array of shape shape with the generated noise. + + Raises: + ValueError: If shape is not a multiple of res. + """ + if interpolant is None: + interpolant = DEFAULT_INTERPOLANT + delta = (res[0] / shape[0], res[1] / shape[1]) + d = (shape[0] // res[0], shape[1] // res[1]) + grid = ( + np.mgrid[0 : res[0] : delta[0], 0 : res[1] : delta[1]].transpose(1, 2, 0) + % 1 + ) + # Gradients + angles = 2 * np.pi * np.random.rand(res[0] + 1, res[1] + 1) + gradients = np.dstack((np.cos(angles), np.sin(angles))) + if tileable[0]: + gradients[-1, :] = gradients[0, :] + if tileable[1]: + gradients[:, -1] = gradients[:, 0] + gradients = gradients.repeat(d[0], 0).repeat(d[1], 1) + g00 = gradients[: -d[0], : -d[1]] + g10 = gradients[d[0] :, : -d[1]] + g01 = gradients[: -d[0], d[1] :] + g11 = gradients[d[0] :, d[1] :] + # Ramps + n00 = np.sum(np.dstack((grid[:, :, 0], grid[:, :, 1])) * g00, 2) + n10 = np.sum(np.dstack((grid[:, :, 0] - 1, grid[:, :, 1])) * g10, 2) + n01 = np.sum(np.dstack((grid[:, :, 0], grid[:, :, 1] - 1)) * g01, 2) + n11 = np.sum(np.dstack((grid[:, :, 0] - 1, grid[:, :, 1] - 1)) * g11, 2) + # Interpolation + t = interpolant(grid) + n0 = n00 * (1 - t[:, :, 0]) + t[:, :, 0] * n10 + n1 = n01 * (1 - t[:, :, 0]) + t[:, :, 0] * n11 + return np.sqrt(2) * ((1 - t[:, :, 1]) * n0 + t[:, :, 1] * n1) + + def generate_fractal_noise_2d( + self, + shape, + res, + octaves=1, + persistence=0.5, + lacunarity=2, + tileable=(True, True), + interpolant=None, + ): + """Generate a 2D numpy array of fractal noise. + + Args: + shape: The shape of the generated array (tuple of two ints). + This must be a multiple of lacunarity**(octaves-1)*res. + res: The number of periods of noise to generate along each + axis (tuple of two ints). Note shape must be a multiple of + (lacunarity**(octaves-1)*res). + octaves: The number of octaves in the noise. Defaults to 1. + persistence: The scaling factor between two octaves. + lacunarity: The frequency factor between two octaves. + tileable: If the noise should be tileable along each axis + (tuple of two bools). Defaults to (True,True). + interpolant: The, interpolation function, defaults to + t*t*t*(t*(t*6 - 15) + 10). + + Returns: + A numpy array of fractal noise and of shape shape generated by + combining several octaves of perlin noise. + + Raises: + ValueError: If shape is not a multiple of + (lacunarity**(octaves-1)*res). + """ + if interpolant is None: + interpolant = DEFAULT_INTERPOLANT + noise = np.zeros(shape) + frequency = 1 + amplitude = 1 + for _ in range(octaves): + noise += amplitude * self.generate_perlin_noise_2d( + shape, (frequency * res[0], frequency * res[1]), tileable, interpolant + ) + frequency *= lacunarity + amplitude *= persistence + return noise + + def fbm(self, x, y, octaves): + # noise_2d = self.generate_fractal_noise_2d((256, 256), (8, 8), octaves) + # Now, extract a single noise value based on x and y, wrapping indices if necessary + x_idx = int(x) % 256 + y_idx = int(y) % 256 + return self.noise_pattern[x_idx, y_idx] + + def apply_shake( + self, + images, + position_amount_x, + position_amount_y, + rotation_amount, + frequency, + frequency_divider, + octaves, + seed, + ): + self.rehash(seed, octaves) + + # Assuming frame count is derived from the first dimension of images tensor + frame_count = images.shape[0] + + frequency = frequency / frequency_divider + + # Generate shaking parameters for each frame + x_translations = [] + y_translations = [] + rotations = [] + + for frame_num in range(frame_count): + time = frame_num * frequency + x_idx = (self.position_offset[0] + frame_num) % 256 + y_idx = (self.position_offset[1] + frame_num) % 256 + + np_position = np.array( + [ + self.fbm(x_idx, time, octaves), + self.fbm(y_idx, time, octaves), + ] + ) + + # np_position = np.array( + # [ + # self.fbm(self.position_offset[0] + frame_num, time, octaves), + # self.fbm(self.position_offset[1] + frame_num, time, octaves), + # ] + # ) + # np_rotation = self.fbm(self.rotation_offset[2] + frame_num, time, octaves) + + rot_idx = (self.rotation_offset[2] + frame_num) % 256 + np_rotation = self.fbm(rot_idx, time, octaves) + + x_translations.append(np_position[0] * position_amount_x) + y_translations.append(np_position[1] * position_amount_y) + rotations.append(np_rotation * rotation_amount) + + # Convert lists to tensors + # x_translations = torch.tensor(x_translations, dtype=torch.float32) + # y_translations = torch.tensor(y_translations, dtype=torch.float32) + # rotations = torch.tensor(rotations, dtype=torch.float32) + + # Create an instance of Batch2dTransform + transform = Batch2dTransform() + + log.debug( + f"Applying shaking with parameters: \nposition {position_amount_x}, {position_amount_y}\nrotation {rotation_amount}\nfrequency {frequency}\noctaves {octaves}" + ) + + # Apply shaking transformations to images + shaken_images = transform.transform_batch( + images, + border_handling="edge", # Assuming edge handling as default + constant_color="#000000", # Assuming black as default constant color + x=x_translations, + y=y_translations, + angle=rotations, + )[0] + + return (shaken_images, x_translations, y_translations, rotations) + + __nodes__ = [ BatchFloat, Batch2dTransform, @@ -341,4 +623,6 @@ def transform_batch( BatchMake, BatchFloatAssemble, BatchFloatFill, + BatchMerge, + BatchShake, ] From d7b8ac8e0c98b0d7a2e21889d35aad9f6b093560 Mon Sep 17 00:00:00 2001 From: melMass Date: Tue, 10 Oct 2023 14:31:28 +0200 Subject: [PATCH 02/12] =?UTF-8?q?feat:=20=E2=9C=A8=20Add=20support=20for?= =?UTF-8?q?=20extra=5Fmodel=5Fpaths.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #66 --- nodes/deep_bump.py | 13 +++++++---- nodes/faceenhance.py | 28 +++++++++++------------ nodes/faceswap.py | 44 ++++++++++++++---------------------- nodes/image_interpolation.py | 15 ++++++++---- utils.py | 35 +++++++++++++++++++++++----- 5 files changed, 78 insertions(+), 57 deletions(-) diff --git a/nodes/deep_bump.py b/nodes/deep_bump.py index fcc4ce1..82639f1 100644 --- a/nodes/deep_bump.py +++ b/nodes/deep_bump.py @@ -1,14 +1,15 @@ import tempfile from pathlib import Path +import folder_paths import numpy as np import onnxruntime as ort import torch from PIL import Image +from ..errors import ModelNotFound from ..log import mklog -from ..utils import (models_dir, tensor2pil, tiles_infer, tiles_merge, - tiles_split) +from ..utils import get_model_path, tensor2pil, tiles_infer, tiles_merge, tiles_split # Disable MS telemetry ort.disable_telemetry_events() @@ -54,9 +55,11 @@ def color_to_normals(color_img, overlap, progress_callback, save_temp=False): # Load model log.debug("DeepBump Color β†’ Normals : loading model") - ort_session = ort.InferenceSession( - (models_dir / "deepbump" / "deepbump256.onnx").as_posix() - ) + model = get_model_path("deepbump", "deepbump256.onnx") + if not model or not model.exists(): + raise ModelNotFound(f"deepbump ({model})") + + ort_session = ort.InferenceSession(model) # Predict normal map for each tile log.debug("DeepBump Color β†’ Normals : generating") diff --git a/nodes/faceenhance.py b/nodes/faceenhance.py index 2e81861..dcb1e06 100644 --- a/nodes/faceenhance.py +++ b/nodes/faceenhance.py @@ -1,21 +1,20 @@ -from gfpgan import GFPGANer -import cv2 -import numpy as np import os from pathlib import Path -import folder_paths -from ..utils import pil2tensor, np2tensor, tensor2np +from typing import Tuple +import comfy +import comfy.utils +import cv2 +import folder_paths +import numpy as np +import torch from basicsr.utils import imwrite - - +from comfy import model_management +from gfpgan import GFPGANer from PIL import Image -import torch + from ..log import NullWriter, log -from comfy import model_management -import comfy -import comfy.utils -from typing import Tuple +from ..utils import get_model_path, np2tensor, pil2tensor, tensor2np class LoadFaceEnhanceModel: @@ -26,11 +25,12 @@ def __init__(self) -> None: @classmethod def get_models_root(cls): - fr = Path(folder_paths.models_dir) / "face_restore" + fr = get_model_path("face_restore") + # fr = Path(folder_paths.models_dir) / "face_restore" if fr.exists(): return (fr, None) - um = Path(folder_paths.models_dir) / "upscale_models" + um = get_model_path("upscale_models") return (fr, um) if um.exists() else (None, None) @classmethod diff --git a/nodes/faceswap.py b/nodes/faceswap.py index 0047ba6..2711426 100644 --- a/nodes/faceswap.py +++ b/nodes/faceswap.py @@ -1,21 +1,21 @@ +# Optional face enhance nodes # region imports -import onnxruntime +import sys from pathlib import Path -from PIL import Image -from typing import List, Set, Union, Optional +from typing import List, Optional, Set, Union + +import comfy.model_management as model_management import cv2 -import folder_paths -import glob import insightface import numpy as np -import os +import onnxruntime import torch from insightface.model_zoo.inswapper import INSwapper -from ..utils import pil2tensor, tensor2pil, download_antelopev2 -from ..log import mklog, NullWriter -import sys -import comfy.model_management as model_management +from PIL import Image +from ..errors import ModelNotFound +from ..log import NullWriter, mklog +from ..utils import download_antelopev2, get_model_path, pil2tensor, tensor2pil # endregion @@ -27,15 +27,6 @@ class LoadFaceAnalysisModel: models = [] - @staticmethod - def get_models() -> List[str]: - models_path = os.path.join(folder_paths.models_dir, "insightface/*") - models = glob.glob(models_path) - models = [ - Path(x).name for x in models if x.endswith(".onnx") or x.endswith(".pth") - ] - return models - @classmethod def INPUT_TYPES(cls): return { @@ -57,7 +48,7 @@ def load_model(self, faceswap_model: str): face_analyser = insightface.app.FaceAnalysis( name=faceswap_model, - root=os.path.join(folder_paths.models_dir, "insightface"), + root=get_model_path("insightface"), ) return (face_analyser,) @@ -67,10 +58,8 @@ class LoadFaceSwapModel: @staticmethod def get_models() -> List[Path]: - models_path = os.path.join(folder_paths.models_dir, "insightface/*") - models = glob.glob(models_path) - models = [Path(x) for x in models if x.endswith(".onnx") or x.endswith(".pth")] - return models + models_path = get_model_path("insightface").iterdir() + return [x for x in models_path if x.suffix in [".onnx", ".pth"]] @classmethod def INPUT_TYPES(cls): @@ -88,9 +77,10 @@ def INPUT_TYPES(cls): CATEGORY = "mtb/facetools" def load_model(self, faceswap_model: str): - model_path = os.path.join( - folder_paths.models_dir, "insightface", faceswap_model - ) + model_path = get_model_path("insightface", faceswap_model) + if not model_path or not model_path.exists(): + raise ModelNotFound(f"{faceswap_model} ({model_path})") + log.info(f"Loading model {model_path}") return ( INSwapper( diff --git a/nodes/image_interpolation.py b/nodes/image_interpolation.py index 0774952..9dd07ca 100644 --- a/nodes/image_interpolation.py +++ b/nodes/image_interpolation.py @@ -12,6 +12,9 @@ import torch from frame_interpolation.eval import interpolator, util +from utils import get_model_path + +from ..errors import ModelNotFound from ..log import log @@ -20,10 +23,9 @@ class LoadFilmModel: @staticmethod def get_models() -> List[Path]: - models_path = os.path.join(folder_paths.models_dir, "FILM/*") - models = glob.glob(models_path) - models = [Path(x) for x in models if x.endswith(".onnx") or x.endswith(".pth")] - return models + models_paths = get_model_path("FILM").iterdir() + + return [x for x in models_paths if x.suffix in [".onnx", ".pth"]] @classmethod def INPUT_TYPES(cls): @@ -41,7 +43,10 @@ def INPUT_TYPES(cls): CATEGORY = "mtb/frame iterpolation" def load_model(self, film_model: str): - model_path = Path(folder_paths.models_dir) / "FILM" / film_model + model_path = get_model_path("FILM", film_model) + if not model_path or not model_path.exists(): + raise ModelNotFound(f"FILM ({model_path})") + if not (model_path / "saved_model.pb").exists(): model_path = model_path / "saved_model" diff --git a/utils.py b/utils.py index 85c05c3..0d15bf2 100644 --- a/utils.py +++ b/utils.py @@ -1,17 +1,14 @@ +import contextlib import functools import math import os import shlex import shutil -import signal import socket import subprocess import sys -import threading import uuid -from contextlib import suppress from pathlib import Path -from queue import Empty, Queue from typing import List, Optional, Union import folder_paths @@ -506,12 +503,11 @@ def download_antelopev2(): antelopev2_url = "https://drive.google.com/uc?id=18wEUfMNohBJ4K3Ly5wpTejPfDzp-8fI8" try: - import folder_paths import gdown log.debug("Loading antelopev2 model") - dest = Path(folder_paths.models_dir) / "insightface" + dest = get_model_path("insightface") archive = dest / "antelopev2.zip" final_path = dest / "models" / "antelopev2" if not final_path.exists(): @@ -538,6 +534,33 @@ def download_antelopev2(): raise e +def get_model_path(fam, model=None): + log.debug(f"Requesting {fam} with model {model}") + res = None + if model: + res = folder_paths.get_full_path(fam, model) + else: + # this one can raise errors... + with contextlib.suppress(KeyError): + res = folder_paths.get_folder_paths(fam) + + if res: + if isinstance(res, list): + if len(res) > 1: + log.warning( + f"Found multiple match, we will pick the first {res[0]}\n{res}" + ) + res = res[0] + res = Path(res) + log.debug(f"Resolved model path from folder_paths: {res}") + else: + res = models_dir / fam + if model: + res /= model + + return res + + # endregion From 5af284067c65042bcdfff04a5d5a2360bf9e4af7 Mon Sep 17 00:00:00 2001 From: melMass Date: Tue, 10 Oct 2023 14:40:44 +0200 Subject: [PATCH 03/12] =?UTF-8?q?fix:=20=F0=9F=90=9B=20encoding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #107 --- __init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__init__.py b/__init__.py index f3c8832..33e9907 100644 --- a/__init__.py +++ b/__init__.py @@ -43,7 +43,7 @@ def extract_nodes_from_source(filename): source_code = "" - with open(filename, "r") as file: + with open(filename, "r", encoding="utf8") as file: source_code = file.read() nodes = [] From e6f65026735770df8aced4a3acb75550ff1c84da Mon Sep 17 00:00:00 2001 From: melMass Date: Tue, 10 Oct 2023 14:41:53 +0200 Subject: [PATCH 04/12] =?UTF-8?q?fix:=20=E2=9C=A8=20handle=20malformed=20s?= =?UTF-8?q?tyles.csv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #106 --- nodes/conditions.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/nodes/conditions.py b/nodes/conditions.py index aef9941..1a32b37 100644 --- a/nodes/conditions.py +++ b/nodes/conditions.py @@ -1,9 +1,11 @@ -from ..utils import here -from ..log import log -import folder_paths -from pathlib import Path -import shutil import csv +import shutil +from pathlib import Path + +import folder_paths + +from ..log import log +from ..utils import here class InterpolateClipSequential: @@ -157,7 +159,13 @@ def INPUT_TYPES(cls): parsed = csv.reader(f) for row in parsed: log.debug(f"Adding style {row[0]}") - cls.options[row[0]] = (row[1], row[2]) + try: + cls.options[row[0]] = (row[1], row[2]) + except Exception: + log.warning( + f"There was an error while parsing {file}, make sure it respects A1111 format, i.e 3 columns name, positive, negative" + ) + continue else: log.debug(f"Using cached styles (count: {len(cls.options)})") From 8d12b59844958fbc696d01d51162f97262664ae9 Mon Sep 17 00:00:00 2001 From: melMass Date: Fri, 20 Oct 2023 20:01:28 +0200 Subject: [PATCH 05/12] =?UTF-8?q?fix:=20=F0=9F=90=9B=20wrong=20output=20fo?= =?UTF-8?q?r=20bbox?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nodes/crop.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nodes/crop.py b/nodes/crop.py index 89a397c..ec25cb9 100644 --- a/nodes/crop.py +++ b/nodes/crop.py @@ -1,9 +1,9 @@ -import torch -from ..utils import tensor2pil, pil2tensor, tensor2np, np2tensor -from PIL import Image, ImageFilter, ImageDraw, ImageChops import numpy as np +import torch +from PIL import Image, ImageChops, ImageDraw, ImageFilter from ..log import log +from ..utils import np2tensor, pil2tensor, tensor2np, tensor2pil class Bbox: @@ -32,7 +32,7 @@ def INPUT_TYPES(cls): CATEGORY = "mtb/crop" def do_crop(self, x, y, width, height): # bbox - return (x, y, width, height) + return ((x, y, width, height),) # return bbox From 255ac036bab1d776301857843d0e7a85e9a9dcb8 Mon Sep 17 00:00:00 2001 From: melMass Date: Fri, 20 Oct 2023 20:02:10 +0200 Subject: [PATCH 06/12] =?UTF-8?q?fix:=20=F0=9F=90=9B=20import=20issue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nodes/image_interpolation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nodes/image_interpolation.py b/nodes/image_interpolation.py index 9dd07ca..4b0354b 100644 --- a/nodes/image_interpolation.py +++ b/nodes/image_interpolation.py @@ -12,10 +12,9 @@ import torch from frame_interpolation.eval import interpolator, util -from utils import get_model_path - from ..errors import ModelNotFound from ..log import log +from ..utils import get_model_path class LoadFilmModel: From 049983dbe2dbce6b772908468c4042d2bfde5eb2 Mon Sep 17 00:00:00 2001 From: melMass Date: Fri, 20 Oct 2023 20:03:08 +0200 Subject: [PATCH 07/12] =?UTF-8?q?feat:=20=F0=9F=9A=80=20add=20optional=20i?= =?UTF-8?q?nputs=20to=20colored=20image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nodes/image_processing.py | 48 +++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/nodes/image_processing.py b/nodes/image_processing.py index 8ee5667..29ac47e 100644 --- a/nodes/image_processing.py +++ b/nodes/image_processing.py @@ -354,7 +354,11 @@ def INPUT_TYPES(cls): "color": ("COLOR",), "width": ("INT", {"default": 512, "min": 16, "max": 8160}), "height": ("INT", {"default": 512, "min": 16, "max": 8160}), - } + }, + "optional": { + "foreground_image": ("IMAGE",), + "foreground_mask": ("MASK",), + }, } CATEGORY = "mtb/generate" @@ -363,12 +367,46 @@ def INPUT_TYPES(cls): FUNCTION = "render_img" - def render_img(self, color, width, height): - image = Image.new("RGB", (width, height), color=color) + def render_img( + self, color, width, height, foreground_image=None, foreground_mask=None + ): + image = Image.new("RGBA", (width, height), color=color) + output = [] + if foreground_image is not None: + if foreground_mask is None: + fg_images = tensor2pil(foreground_image) + for img in fg_images: + if image.size != img.size: + raise ValueError( + f"Dimension mismatch: image {image.size}, img {img.size}" + ) + + if img.mode != "RGBA": + raise ValueError( + f"Foreground image must be in 'RGBA' mode when no mask is provided, got {img.mode}" + ) + + output.append(Image.alpha_composite(image, img).convert("RGB")) + + elif foreground_image.size[0] != foreground_mask.size[0]: + raise ValueError("Foreground image and mask must have same batch size") + else: + fg_images = tensor2pil(foreground_image) + fg_masks = tensor2pil(foreground_mask) + output.extend( + Image.composite( + fg_image.convert("RGBA"), + image, + fg_mask, + ).convert("RGB") + for fg_image, fg_mask in zip(fg_images, fg_masks) + ) + elif foreground_mask is not None: + log.warn("Mask ignored because no foreground image is given") - image = pil2tensor(image) + output = pil2tensor(output) - return (image,) + return (output,) class ImagePremultiply: From bcac66508d2e788cc437da289d1ccede19465b8c Mon Sep 17 00:00:00 2001 From: melMass Date: Sat, 21 Oct 2023 03:14:12 +0200 Subject: [PATCH 08/12] =?UTF-8?q?refactor:=20=E2=9A=A1=EF=B8=8F=20small=20?= =?UTF-8?q?local=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit made while writting wiki --- errors.py | 7 ++++ nodes/deep_bump.py | 1 - nodes/faceenhance.py | 7 ++-- nodes/image_processing.py | 85 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 errors.py diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..2f98184 --- /dev/null +++ b/errors.py @@ -0,0 +1,7 @@ +class ModelNotFound(Exception): + def __init__(self, model_name, *args, **kwargs): + super().__init__( + f"The model {model_name} could not be found, make sure to download it using ComfyManager first.\nrepository: https://github.com/ltdrdata/ComfyUI-Manager", + *args, + **kwargs, + ) diff --git a/nodes/deep_bump.py b/nodes/deep_bump.py index 82639f1..c1d54bb 100644 --- a/nodes/deep_bump.py +++ b/nodes/deep_bump.py @@ -1,7 +1,6 @@ import tempfile from pathlib import Path -import folder_paths import numpy as np import onnxruntime as ort import torch diff --git a/nodes/faceenhance.py b/nodes/faceenhance.py index dcb1e06..c308525 100644 --- a/nodes/faceenhance.py +++ b/nodes/faceenhance.py @@ -8,7 +8,6 @@ import folder_paths import numpy as np import torch -from basicsr.utils import imwrite from comfy import model_management from gfpgan import GFPGANer from PIL import Image @@ -247,16 +246,16 @@ def save_intermediate_images(self, cropped_faces, restored_faces, height, width) ): face_id = idx + 1 file = self.get_step_image_path("cropped_faces", face_id) - imwrite(cropped_face, file) + cv2.imwrite(file, cropped_face) file = self.get_step_image_path("cropped_faces_restored", face_id) - imwrite(restored_face, file) + cv2.imwrite(file, restored_face) file = self.get_step_image_path("cropped_faces_compare", face_id) # save comparison image cmp_img = np.concatenate((cropped_face, restored_face), axis=1) - imwrite(cmp_img, file) + cv2.imwrite(file, cmp_img) __nodes__ = [RestoreFace, LoadFaceEnhanceModel] diff --git a/nodes/image_processing.py b/nodes/image_processing.py index 29ac47e..3853a44 100644 --- a/nodes/image_processing.py +++ b/nodes/image_processing.py @@ -22,6 +22,18 @@ # log.warning("cv2.ximgproc.guidedFilter not found, use opencv-contrib-python") +def gaussian_kernel(kernel_size: int, sigma_x: float, sigma_y: float, device=None): + x, y = torch.meshgrid( + torch.linspace(-1, 1, kernel_size, device=device), + torch.linspace(-1, 1, kernel_size, device=device), + indexing="ij", + ) + d_x = x * x / (2.0 * sigma_x * sigma_x) + d_y = y * y / (2.0 * sigma_y * sigma_y) + g = torch.exp(-(d_x + d_y)) + return g / g.sum() + + class ColorCorrect: """Various color correction methods""" @@ -273,6 +285,78 @@ def blur(self, image: torch.Tensor, sigmaX, sigmaY): return (torch.from_numpy(image),) +class Sharpen_: + """Sharpens an image using a Gaussian kernel.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ("IMAGE",), + "sharpen_radius": ( + "INT", + {"default": 1, "min": 1, "max": 31, "step": 1}, + ), + "sigma_x": ( + "FLOAT", + {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}, + ), + "sigma_y": ( + "FLOAT", + {"default": 1.0, "min": 0.1, "max": 10.0, "step": 0.1}, + ), + "alpha": ( + "FLOAT", + {"default": 1.0, "min": 0.0, "max": 5.0, "step": 0.1}, + ), + }, + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "do_sharp" + CATEGORY = "mtb/image processing" + + def do_sharp( + self, + image: torch.Tensor, + sharpen_radius: int, + sigma_x: float, + sigma_y: float, + alpha: float, + ): + if sharpen_radius == 0: + return (image,) + + channels = image.shape[3] + + kernel_size = 2 * sharpen_radius + 1 + kernel = gaussian_kernel(kernel_size, sigma_x, sigma_y) * -(alpha * 10) + + # Modify center of kernel to make it a sharpening kernel + center = kernel_size // 2 + kernel[center, center] = kernel[center, center] - kernel.sum() + 1.0 + + kernel = kernel.repeat(channels, 1, 1).unsqueeze(1) + tensor_image = image.permute(0, 3, 1, 2) + + tensor_image = F.pad( + tensor_image, + (sharpen_radius, sharpen_radius, sharpen_radius, sharpen_radius), + "reflect", + ) + sharpened = F.conv2d(tensor_image, kernel, padding=center, groups=channels) + + # Remove padding + sharpened = sharpened[ + :, :, sharpen_radius:-sharpen_radius, sharpen_radius:-sharpen_radius + ] + + sharpened = sharpened.permute(0, 2, 3, 1) + result = torch.clamp(sharpened, 0, 1) + + return (result,) + + # https://github.com/lllyasviel/AdverseCleaner/blob/main/clean.py # def deglaze_np_img(np_img): # y = np_img.copy() @@ -702,4 +786,5 @@ def tile_image(self, image: torch.Tensor, tiles: int = 2): ImageResizeFactor, SaveImageGrid_, LoadImageFromUrl_, + Sharpen_, ] From c8658dfbdd3a0ca8c3e88cd1adfddc55c7444045 Mon Sep 17 00:00:00 2001 From: melMass Date: Sat, 4 Nov 2023 16:10:52 +0100 Subject: [PATCH 09/12] =?UTF-8?q?fix:=20=F0=9F=90=9B=20fit=20number=20regr?= =?UTF-8?q?ession?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #120 --- examples/04-animation_builder-deforum.json | 2 +- log.py | 4 ++- nodes/graph_utils.py | 29 +++++++++++++--------- utils.py | 3 --- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/examples/04-animation_builder-deforum.json b/examples/04-animation_builder-deforum.json index 5cd3826..fe8d950 100644 --- a/examples/04-animation_builder-deforum.json +++ b/examples/04-animation_builder-deforum.json @@ -1 +1 @@ -{"last_node_id":43,"last_link_id":69,"nodes":[{"id":24,"type":"Note","pos":[-827,406],"size":[233.25148010253906,82.53218841552734],"flags":{},"order":0,"mode":0,"properties":{"text":""},"widgets_values":["On first frame we get the init image, on all subsequent ones the feedback from the previous queue item"],"color":"#223","bgcolor":"#335","shape":1},{"id":10,"type":"LoadImage","pos":[-1409,524],"size":[315,314],"flags":{},"order":1,"mode":0,"outputs":[{"name":"IMAGE","type":"IMAGE","links":[10],"shape":3,"slot_index":0},{"name":"MASK","type":"MASK","links":null,"shape":3}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["example.png","image"],"color":"#432","bgcolor":"#653","shape":1},{"id":35,"type":"CLIPTextEncode","pos":[-118,331],"size":[210,54],"flags":{},"order":7,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":60},{"name":"text","type":"STRING","link":66,"widget":{"name":"text","config":["STRING",{"multiline":true}]}}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[54],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":[""]},{"id":9,"type":"CheckpointLoaderSimple","pos":[-558,114],"size":[301.2330322265625,98],"flags":{},"order":2,"mode":0,"outputs":[{"name":"MODEL","type":"MODEL","links":[62],"shape":3,"slot_index":0},{"name":"CLIP","type":"CLIP","links":[59,60],"shape":3,"slot_index":1},{"name":"VAE","type":"VAE","links":[58,69],"shape":3,"slot_index":2}],"properties":{"Node name for S&R":"CheckpointLoaderSimple"},"widgets_values":["revAnimated_v122.safetensors"]},{"id":37,"type":"VAEEncode","pos":[-125,236],"size":[210,46],"flags":{},"order":16,"mode":0,"inputs":[{"name":"pixels","type":"IMAGE","link":57},{"name":"vae","type":"VAE","link":58,"slot_index":1}],"outputs":[{"name":"LATENT","type":"LATENT","links":[55],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"VAEEncode"}},{"id":34,"type":"CLIPTextEncode","pos":[-124,84],"size":[210,96],"flags":{},"order":6,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":59}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[53],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":["A plastic (skeleton) in a pink dress sleeping, draping, wrinkles, shiny"]},{"id":33,"type":"Text box","pos":[-497,349],"size":[294,95.1284408569336],"flags":{},"order":3,"mode":0,"outputs":[{"name":"STRING","type":"STRING","links":[66],"shape":3,"slot_index":0}],"title":"❌Mel Negatives (general) (Negative)","properties":{"Node name for S&R":"Text box"},"widgets_values":["embedding:EasyNegative, embedding:EasyNegativeV2, watermark, text, deformed, NSFW, Cleavage, Pubic Hair, Nudity, Naked, censored"]},{"id":39,"type":"Reroute","pos":[-156,908],"size":[75,26],"flags":{},"order":11,"mode":0,"inputs":[{"name":"","type":"*","link":67}],"outputs":[{"name":"","type":"INT","links":[68]}],"properties":{"showOutputText":false,"horizontal":false}},{"id":36,"type":"KSampler","pos":[223,262],"size":[315,442],"flags":{},"order":17,"mode":0,"inputs":[{"name":"model","type":"MODEL","link":62,"slot_index":0},{"name":"positive","type":"CONDITIONING","link":53},{"name":"negative","type":"CONDITIONING","link":54},{"name":"latent_image","type":"LATENT","link":55},{"name":"denoise","type":"FLOAT","link":63,"widget":{"name":"denoise","config":["FLOAT",{"default":1,"min":0,"max":1,"step":0.01}]},"slot_index":4},{"name":"seed","type":"INT","link":68,"widget":{"name":"seed","config":["INT",{"default":0,"min":0,"max":18446744073709552000}]}}],"outputs":[{"name":"LATENT","type":"LATENT","links":[56],"shape":3,"slot_index":0,"color":"#FF9CF9"}],"properties":{"Node name for S&R":"KSampler"},"widgets_values":[938170558049910,"randomize",15,8,"euler_ancestral","normal",0.6]},{"id":15,"type":"SaveImage","pos":[782,259],"size":[330.1112365722656,378.1239929199219],"flags":{},"order":19,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":65}],"properties":{},"widgets_values":["ComfyUI"]},{"id":38,"type":"VAEDecode","pos":[556,260],"size":[210,46],"flags":{},"order":18,"mode":0,"inputs":[{"name":"samples","type":"LATENT","link":56},{"name":"vae","type":"VAE","link":69,"slot_index":1}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[65],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"VAEDecode"}},{"id":11,"type":"Get Batch From History (mtb)","pos":[-800,524],"size":[235.1999969482422,126],"flags":{},"order":13,"mode":0,"inputs":[{"name":"passthrough_image","type":"IMAGE","link":10},{"name":"enable","type":"BOOLEAN","link":9,"widget":{"name":"enable","config":["BOOLEAN",{"default":true}]},"slot_index":1}],"outputs":[{"name":"i","type":"IMAGE","links":[26],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Get Batch From History (mtb)"},"widgets_values":[false,1,0,1035,true],"color":"#223","bgcolor":"#335"},{"id":12,"type":"Int To Bool (mtb)","pos":[-1065,765],"size":[210,36.366058349609375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"int","type":"INT","link":34,"widget":{"name":"int","config":["INT",{"default":0}]},"slot_index":0}],"outputs":[{"name":"BOOLEAN","type":"BOOLEAN","links":[9],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Int To Bool (mtb)"},"widgets_values":[29],"color":"#222","bgcolor":"#000"},{"id":22,"type":"Fit Number (mtb)","pos":[-647,882],"size":[232.28509521484375,178],"flags":{},"order":10,"mode":0,"inputs":[{"name":"value","type":"FLOAT","link":27}],"outputs":[{"name":"FLOAT","type":"FLOAT","links":[63],"shape":3,"slot_index":0}],"title":"Fit Number (mtb) - Denoise","properties":{"Node name for S&R":"Fit Number (mtb)"},"widgets_values":[true,0,1,0.4,0.65,"Quart In/Out"]},{"id":14,"type":"Transform Image (mtb)","pos":[-527,520],"size":[315,214],"flags":{},"order":15,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":26}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[57],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Transform Image (mtb)"},"widgets_values":[15,0,0.98,-1,0,"reflect","#930606"],"color":"#223","bgcolor":"#335"},{"id":31,"type":"PrimitiveNode","pos":[-2204,1309],"size":[210,82],"flags":{},"order":4,"mode":0,"outputs":[{"name":"INT","type":"INT","links":[43],"widget":{"name":"total_frames","config":["INT",{"default":100,"min":0}]},"slot_index":0}],"title":"total_frames","properties":{},"widgets_values":[30,"fixed"],"color":"#432","bgcolor":"#653"},{"id":43,"type":"Note","pos":[-2209,1157],"size":[214.22988891601562,107.93704986572266],"flags":{},"order":5,"mode":0,"properties":{"text":""},"widgets_values":["BECAUSE THE ORDER OF OPERATION FROM HOW ANIMATION BUILDER IS DONE, YOU MUST USE ONE MORE FRAME THAN NEEDED HERE"],"color":"#432","bgcolor":"#653"},{"id":18,"type":"Get Batch From History (mtb)","pos":[-960,1257],"size":[235.1999969482422,126],"flags":{},"order":12,"mode":0,"inputs":[{"name":"passthrough_image","type":"IMAGE","link":null},{"name":"enable","type":"BOOLEAN","link":31,"widget":{"name":"enable","config":["BOOLEAN",{"default":true}]},"slot_index":1}],"outputs":[{"name":"i","type":"IMAGE","links":[18],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Get Batch From History (mtb)"},"widgets_values":[true,29,0,1035,false]},{"id":19,"type":"Save Gif (mtb)","pos":[-613,1256],"size":[210,372],"flags":{},"order":14,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":18}],"properties":{"Node name for S&R":"Save Gif (mtb)"},"widgets_values":[12,1,false,false,"nearest","/view?filename=0f83896060.gif&subfolder=&type=output"]},{"id":17,"type":"Animation Builder (mtb)","pos":[-1312,883],"size":[211.60000610351562,294],"flags":{},"order":8,"mode":0,"inputs":[{"name":"total_frames","type":"INT","link":43,"widget":{"name":"total_frames","config":["INT",{"default":100,"min":0}]},"slot_index":0}],"outputs":[{"name":"frame","type":"INT","links":[34],"shape":3,"slot_index":0},{"name":"0-1 (scaled)","type":"FLOAT","links":[27],"shape":3,"slot_index":1},{"name":"count","type":"INT","links":[67],"shape":3,"slot_index":2},{"name":"loop_ended","type":"BOOLEAN","links":[31],"shape":3,"slot_index":3}],"properties":{"Node name for S&R":"Animation Builder (mtb)"},"widgets_values":[30,1,2,0,0,"Idle","Iteration: Idle","reset","queue"],"color":"#232","bgcolor":"#353","shape":1}],"links":[[9,12,0,11,1,"BOOLEAN"],[10,10,0,11,0,"IMAGE"],[18,18,0,19,0,"IMAGE"],[26,11,0,14,0,"IMAGE"],[27,17,1,22,0,"FLOAT"],[31,17,3,18,1,"BOOLEAN"],[34,17,0,12,0,"INT"],[43,31,0,17,0,"INT"],[53,34,0,36,1,"CONDITIONING"],[54,35,0,36,2,"CONDITIONING"],[55,37,0,36,3,"LATENT"],[56,36,0,38,0,"LATENT"],[57,14,0,37,0,"IMAGE"],[58,9,2,37,1,"VAE"],[59,9,1,34,0,"CLIP"],[60,9,1,35,0,"CLIP"],[62,9,0,36,0,"MODEL"],[63,22,0,36,4,"FLOAT"],[65,38,0,15,0,"IMAGE"],[66,33,0,35,1,"STRING"],[67,17,2,39,0,"*"],[68,39,0,36,5,"INT"],[69,9,2,38,1,"VAE"]],"groups":[{"title":"Video Output","bounding":[-702,1161,516,773],"color":"#3f789e","locked":false},{"title":"START THE QUEUE BY CLICKLING HERE πŸ‘†","bounding":[-1612,1219,521,80],"color":"#8A8","locked":false}],"config":{},"extra":{},"version":0.4} \ No newline at end of file +{"last_node_id":44,"last_link_id":71,"nodes":[{"id":24,"type":"Note","pos":[-827,406],"size":[233.25148010253906,82.53218841552734],"flags":{},"order":0,"mode":0,"properties":{"text":""},"widgets_values":["On first frame we get the init image, on all subsequent ones the feedback from the previous queue item"],"color":"#223","bgcolor":"#335","shape":1},{"id":10,"type":"LoadImage","pos":[-1409,524],"size":[315,314],"flags":{},"order":1,"mode":0,"outputs":[{"name":"IMAGE","type":"IMAGE","links":[10],"shape":3,"slot_index":0},{"name":"MASK","type":"MASK","links":null,"shape":3}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["example.png","image"],"color":"#432","bgcolor":"#653","shape":1},{"id":35,"type":"CLIPTextEncode","pos":[-118,331],"size":[210,54],"flags":{},"order":7,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":60},{"name":"text","type":"STRING","link":66,"widget":{"name":"text"}}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[54],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":[""]},{"id":9,"type":"CheckpointLoaderSimple","pos":[-558,114],"size":[301.2330322265625,98],"flags":{},"order":2,"mode":0,"outputs":[{"name":"MODEL","type":"MODEL","links":[62],"shape":3,"slot_index":0},{"name":"CLIP","type":"CLIP","links":[59,60],"shape":3,"slot_index":1},{"name":"VAE","type":"VAE","links":[58,69],"shape":3,"slot_index":2}],"properties":{"Node name for S&R":"CheckpointLoaderSimple"},"widgets_values":["revAnimated_v122.safetensors"]},{"id":37,"type":"VAEEncode","pos":[-125,236],"size":[210,46],"flags":{},"order":16,"mode":0,"inputs":[{"name":"pixels","type":"IMAGE","link":57},{"name":"vae","type":"VAE","link":58,"slot_index":1}],"outputs":[{"name":"LATENT","type":"LATENT","links":[55],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"VAEEncode"}},{"id":34,"type":"CLIPTextEncode","pos":[-124,84],"size":[210,96],"flags":{},"order":6,"mode":0,"inputs":[{"name":"clip","type":"CLIP","link":59}],"outputs":[{"name":"CONDITIONING","type":"CONDITIONING","links":[53],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"CLIPTextEncode"},"widgets_values":["A plastic (skeleton) in a pink dress sleeping, draping, wrinkles, shiny"]},{"id":33,"type":"Text box","pos":[-497,349],"size":[294,95.1284408569336],"flags":{},"order":3,"mode":0,"outputs":[{"name":"STRING","type":"STRING","links":[66],"shape":3,"slot_index":0}],"title":"❌Mel Negatives (general) (Negative)","properties":{"Node name for S&R":"Text box"},"widgets_values":["embedding:EasyNegative, embedding:EasyNegativeV2, watermark, text, deformed, NSFW, Cleavage, Pubic Hair, Nudity, Naked, censored"]},{"id":39,"type":"Reroute","pos":[-156,908],"size":[75,26],"flags":{},"order":11,"mode":0,"inputs":[{"name":"","type":"*","link":67}],"outputs":[{"name":"","type":"INT","links":[68]}],"properties":{"showOutputText":false,"horizontal":false}},{"id":36,"type":"KSampler","pos":[223,262],"size":[315,442],"flags":{},"order":17,"mode":0,"inputs":[{"name":"model","type":"MODEL","link":62,"slot_index":0},{"name":"positive","type":"CONDITIONING","link":53},{"name":"negative","type":"CONDITIONING","link":54},{"name":"latent_image","type":"LATENT","link":55},{"name":"denoise","type":"FLOAT","link":71,"widget":{"name":"denoise"},"slot_index":4},{"name":"seed","type":"INT","link":68,"widget":{"name":"seed"}}],"outputs":[{"name":"LATENT","type":"LATENT","links":[56],"shape":3,"slot_index":0,"color":"#FF9CF9"}],"properties":{"Node name for S&R":"KSampler"},"widgets_values":[286457730243556,"randomize",15,8,"euler_ancestral","normal",0.6]},{"id":15,"type":"SaveImage","pos":[782,259],"size":[330.1112365722656,378.1239929199219],"flags":{},"order":19,"mode":0,"inputs":[{"name":"images","type":"IMAGE","link":65}],"properties":{},"widgets_values":["ComfyUI"]},{"id":38,"type":"VAEDecode","pos":[556,260],"size":[210,46],"flags":{},"order":18,"mode":0,"inputs":[{"name":"samples","type":"LATENT","link":56},{"name":"vae","type":"VAE","link":69,"slot_index":1}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[65],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"VAEDecode"}},{"id":11,"type":"Get Batch From History (mtb)","pos":[-800,524],"size":[235.1999969482422,126],"flags":{},"order":13,"mode":0,"inputs":[{"name":"passthrough_image","type":"IMAGE","link":10},{"name":"enable","type":"BOOLEAN","link":9,"widget":{"name":"enable"},"slot_index":1}],"outputs":[{"name":"i","type":"IMAGE","links":[26],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Get Batch From History (mtb)"},"widgets_values":[false,1,0,1036],"color":"#223","bgcolor":"#335"},{"id":12,"type":"Int To Bool (mtb)","pos":[-1065,765],"size":[210,36.366058349609375],"flags":{},"order":9,"mode":0,"inputs":[{"name":"int","type":"INT","link":34,"widget":{"name":"int"},"slot_index":0}],"outputs":[{"name":"BOOLEAN","type":"BOOLEAN","links":[9],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Int To Bool (mtb)"},"widgets_values":[29],"color":"#222","bgcolor":"#000"},{"id":14,"type":"Transform Image (mtb)","pos":[-527,520],"size":[315,214],"flags":{},"order":15,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":26}],"outputs":[{"name":"IMAGE","type":"IMAGE","links":[57],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Transform Image (mtb)"},"widgets_values":[15,0,0.98,-1,0,"reflect","#930606"],"color":"#223","bgcolor":"#335"},{"id":31,"type":"PrimitiveNode","pos":[-2204,1309],"size":[210,82],"flags":{},"order":4,"mode":0,"outputs":[{"name":"INT","type":"INT","links":[43],"widget":{"name":"total_frames"},"slot_index":0}],"title":"total_frames","properties":{},"widgets_values":[30,"fixed"],"color":"#432","bgcolor":"#653"},{"id":43,"type":"Note","pos":[-2209,1157],"size":[214.22988891601562,107.93704986572266],"flags":{},"order":5,"mode":0,"properties":{"text":""},"widgets_values":["BECAUSE THE ORDER OF OPERATION FROM HOW ANIMATION BUILDER IS DONE, YOU MUST USE ONE MORE FRAME THAN NEEDED HERE"],"color":"#432","bgcolor":"#653"},{"id":18,"type":"Get Batch From History (mtb)","pos":[-960,1257],"size":[235.1999969482422,126],"flags":{},"order":12,"mode":0,"inputs":[{"name":"passthrough_image","type":"IMAGE","link":null},{"name":"enable","type":"BOOLEAN","link":31,"widget":{"name":"enable"},"slot_index":1}],"outputs":[{"name":"i","type":"IMAGE","links":[18],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Get Batch From History (mtb)"},"widgets_values":[true,29,0,1036]},{"id":19,"type":"Save Gif (mtb)","pos":[-613,1256],"size":[210,372],"flags":{},"order":14,"mode":0,"inputs":[{"name":"image","type":"IMAGE","link":18}],"properties":{"Node name for S&R":"Save Gif (mtb)"},"widgets_values":[12,1,false,false,"nearest"]},{"id":44,"type":"Fit Number (mtb)","pos":[-640,883],"size":[315,202],"flags":{},"order":10,"mode":0,"inputs":[{"name":"value","type":"FLOAT","link":70,"widget":{"name":"value"}}],"outputs":[{"name":"FLOAT","type":"FLOAT","links":[71],"shape":3,"slot_index":0}],"properties":{"Node name for S&R":"Fit Number (mtb)"},"widgets_values":[0,false,0,1,0.4,0.65,"Linear"]},{"id":17,"type":"Animation Builder (mtb)","pos":[-1312,883],"size":[211.60000610351562,294],"flags":{},"order":8,"mode":0,"inputs":[{"name":"total_frames","type":"INT","link":43,"widget":{"name":"total_frames"},"slot_index":0}],"outputs":[{"name":"frame","type":"INT","links":[34],"shape":3,"slot_index":0},{"name":"0-1 (scaled)","type":"FLOAT","links":[70],"shape":3,"slot_index":1},{"name":"count","type":"INT","links":[67],"shape":3,"slot_index":2},{"name":"loop_ended","type":"BOOLEAN","links":[31],"shape":3,"slot_index":3}],"properties":{"Node name for S&R":"Animation Builder (mtb)"},"widgets_values":[30,1,2,0,0,"Idle","Iteration: Idle","reset","queue"],"color":"#232","bgcolor":"#353","shape":1}],"links":[[9,12,0,11,1,"BOOLEAN"],[10,10,0,11,0,"IMAGE"],[18,18,0,19,0,"IMAGE"],[26,11,0,14,0,"IMAGE"],[31,17,3,18,1,"BOOLEAN"],[34,17,0,12,0,"INT"],[43,31,0,17,0,"INT"],[53,34,0,36,1,"CONDITIONING"],[54,35,0,36,2,"CONDITIONING"],[55,37,0,36,3,"LATENT"],[56,36,0,38,0,"LATENT"],[57,14,0,37,0,"IMAGE"],[58,9,2,37,1,"VAE"],[59,9,1,34,0,"CLIP"],[60,9,1,35,0,"CLIP"],[62,9,0,36,0,"MODEL"],[65,38,0,15,0,"IMAGE"],[66,33,0,35,1,"STRING"],[67,17,2,39,0,"*"],[68,39,0,36,5,"INT"],[69,9,2,38,1,"VAE"],[70,17,1,44,0,"FLOAT"],[71,44,0,36,4,"FLOAT"]],"groups":[{"title":"Video Output","bounding":[-702,1161,516,773],"color":"#3f789e","font_size":24,"locked":false},{"title":"START THE QUEUE BY CLICKLING HERE πŸ‘†","bounding":[-1612,1219,521,80],"color":"#8A8","font_size":24,"locked":false}],"config":{},"extra":{},"version":0.4} \ No newline at end of file diff --git a/log.py b/log.py index 977c113..271e814 100644 --- a/log.py +++ b/log.py @@ -1,6 +1,6 @@ import logging -import re import os +import re base_log_level = logging.DEBUG if os.environ.get("MTB_DEBUG") else logging.INFO @@ -75,5 +75,7 @@ def cyan_text(text): def get_label(label): + if label.startswith("MTB_"): + label = label[4:] words = re.findall(r"(?:^|[A-Z])[a-z]*", label) return " ".join(words).strip() diff --git a/nodes/graph_utils.py b/nodes/graph_utils.py index 7c35ec9..7bd49aa 100644 --- a/nodes/graph_utils.py +++ b/nodes/graph_utils.py @@ -179,10 +179,10 @@ def INPUT_TYPES(cls): "required": { "value": ("FLOAT", {"default": 0, "forceInput": True}), "clamp": ("BOOLEAN", {"default": False}), - "source_min": ("FLOAT", {"default": 0.0}), - "source_max": ("FLOAT", {"default": 1.0}), - "target_min": ("FLOAT", {"default": 0.0}), - "target_max": ("FLOAT", {"default": 1.0}), + "source_min": ("FLOAT", {"default": 0.0, "step": 0.01}), + "source_max": ("FLOAT", {"default": 1.0, "step": 0.01}), + "target_min": ("FLOAT", {"default": 0.0, "step": 0.01}), + "target_max": ("FLOAT", {"default": 1.0, "step": 0.01}), "easing": ( [ "Linear", @@ -228,19 +228,18 @@ def set_range( target_max: float, easing: str, ): - normalized_value = (value - source_min) / (source_max - source_min) + if source_min == source_max: + normalized_value = 0 + else: + normalized_value = (value - source_min) / (source_max - source_min) + if clamp: + normalized_value = max(min(normalized_value, 1), 0) eased_value = apply_easing(normalized_value, easing) # - Convert the eased value to the target range res = target_min + (target_max - target_min) * eased_value - if clamp: - if target_min > target_max: - res = max(min(res, target_min), target_max) - else: - res = max(min(res, target_max), target_min) - return (res,) @@ -271,4 +270,10 @@ def concatenate_tensors(self, reverse, **kwargs): return (concatenated,) -__nodes__ = [StringReplace, FitNumber, GetBatchFromHistory, AnyToString, ConcatImages] +__nodes__ = [ + StringReplace, + FitNumber, + GetBatchFromHistory, + AnyToString, + ConcatImages, +] diff --git a/utils.py b/utils.py index 0d15bf2..9520870 100644 --- a/utils.py +++ b/utils.py @@ -585,9 +585,6 @@ def create_uv_map_tensor(width=512, height=512): # region ANIMATION Utilities def apply_easing(value, easing_type): - if value < 0 or value > 1: - raise ValueError(f"The value should be between 0 and 1. (value is {value})") - if easing_type == "Linear": return value From 142624eea616a5622387b1b641c02605455ee6f1 Mon Sep 17 00:00:00 2001 From: melMass Date: Sat, 4 Nov 2023 16:14:06 +0100 Subject: [PATCH 10/12] =?UTF-8?q?feat:=20=E2=9C=A8=20Math=20Expression=20n?= =?UTF-8?q?ode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nodes/graph_utils.py | 47 +++++++++++++++++++++++++++++++++++++++++++ web/comfy_shared.js | 26 +++++++++++++----------- web/mtb_widgets.js | 48 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/nodes/graph_utils.py b/nodes/graph_utils.py index 7bd49aa..5e10ae7 100644 --- a/nodes/graph_utils.py +++ b/nodes/graph_utils.py @@ -170,6 +170,52 @@ def replace_str(self, string: str, old: str, new: str): return (string,) +class MTB_MathExpression: + """Node to evaluate a simple math expression string""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "expression": ("STRING", {"default": "", "multiline": True}), + } + } + + FUNCTION = "eval_expression" + RETURN_TYPES = ("FLOAT", "INT") + RETURN_NAMES = ("result (float)", "result (int)") + CATEGORY = "mtb/math" + DESCRIPTION = "evaluate a simple math expression string (!! Fallsback to eval)" + + def eval_expression(self, expression, **kwargs): + import math + from ast import literal_eval + + for key, value in kwargs.items(): + print(f"Replacing placeholder <{key}> with value {value}") + expression = expression.replace(f"<{key}>", str(value)) + + result = -1 + try: + result = literal_eval(expression) + except SyntaxError as e: + raise ValueError( + f"The expression syntax is wrong '{expression}': {e}" + ) from e + + except ValueError: + try: + expression = expression.replace("^", "**") + result = eval(expression) + except Exception as e: + # Handle any other exceptions and provide a meaningful error message + raise ValueError( + f"Error evaluating expression '{expression}': {e}" + ) from e + + return (result, int(result)) + + class FitNumber: """Fit the input float using a source and target range""" @@ -276,4 +322,5 @@ def concatenate_tensors(self, reverse, **kwargs): GetBatchFromHistory, AnyToString, ConcatImages, + MTB_MathExpression, ] diff --git a/web/comfy_shared.js b/web/comfy_shared.js index c034c87..f7a83d9 100644 --- a/web/comfy_shared.js +++ b/web/comfy_shared.js @@ -118,7 +118,8 @@ export const dynamic_connection = ( index, connected, connectionPrefix = 'input_', - connectionType = 'PSDLAYER' + connectionType = 'PSDLAYER', + nameArray = [] ) => { // remove all non connected inputs if (!connected && node.inputs.length > 1) { @@ -134,23 +135,24 @@ export const dynamic_connection = ( // make inputs sequential again for (let i = 0; i < node.inputs.length; i++) { - node.inputs[i].label = `${connectionPrefix}${i + 1}` - node.inputs[i].name = `${connectionPrefix}${i + 1}` + const name = + i < nameArray.length ? nameArray[i] : `${connectionPrefix}${i + 1}` + node.inputs[i].label = name + node.inputs[i].name = name } } // add an extra input if (node.inputs[node.inputs.length - 1].link != undefined) { - log( - `Adding input ${node.inputs.length + 1} (${connectionPrefix}${ - node.inputs.length + 1 - })` - ) + const nextIndex = node.inputs.length + const name = + nextIndex < nameArray.length + ? nameArray[nextIndex] + : `${connectionPrefix}${nextIndex + 1}` - node.addInput( - `${connectionPrefix}${node.inputs.length + 1}`, - connectionType - ) + log(`Adding input ${nextIndex + 1} (${name})`) + + node.addInput(name, connectionType) } } diff --git a/web/mtb_widgets.js b/web/mtb_widgets.js index 03918f0..fca9162 100644 --- a/web/mtb_widgets.js +++ b/web/mtb_widgets.js @@ -494,6 +494,10 @@ const mtb_widgets = { async beforeRegisterNodeDef(nodeType, nodeData, app) { // const rinputs = nodeData.input?.required + if (!nodeData.name.endsWith('(mtb)')) { + return + } + let has_custom = false if (nodeData.input && nodeData.input.required) { for (const i of Object.keys(nodeData.input.required)) { @@ -892,6 +896,50 @@ const mtb_widgets = { break } + case 'Math Expression (mtb)': { + const onNodeCreated = nodeType.prototype.onNodeCreated + nodeType.prototype.onNodeCreated = function () { + const r = onNodeCreated + ? onNodeCreated.apply(this, arguments) + : undefined + this.addInput(`x`, '*') + return r + } + + const onConnectionsChange = nodeType.prototype.onConnectionsChange + nodeType.prototype.onConnectionsChange = function ( + type, + index, + connected, + link_info + ) { + const r = onConnectionsChange + ? onConnectionsChange.apply(this, arguments) + : undefined + shared.dynamic_connection(this, index, connected, 'var_', '*', [ + 'x', + 'y', + 'z', + ]) + + //- infer type + if (link_info) { + const fromNode = this.graph._nodes.find( + (otherNode) => otherNode.id == link_info.origin_id + ) + const type = fromNode.outputs[link_info.origin_slot].type + this.inputs[index].type = type + // this.inputs[index].label = type.toLowerCase() + } + //- restore dynamic input + if (!connected) { + this.inputs[index].type = '*' + this.inputs[index].label = `number_${index + 1}` + } + } + + break + } case 'Save Tensors (mtb)': { const onDrawBackground = nodeType.prototype.onDrawBackground nodeType.prototype.onDrawBackground = function (ctx, canvas) { From 9afad1a1680073006d946be10f8c97b75ddfe253 Mon Sep 17 00:00:00 2001 From: melMass Date: Sat, 4 Nov 2023 16:19:10 +0100 Subject: [PATCH 11/12] =?UTF-8?q?chore:=20=E2=9C=A8=20local=20stuff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- INSTALL.md | 6 ------ nodes/batch.py | 33 +++++++++++++++------------------ 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index e3f7d65..83c7bdc 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -4,7 +4,6 @@ - [ComfyUI Manager](#comfyui-manager) - [Virtual Env](#virtual-env) - [Models Download](#models-download) - - [Web Extensions](#web-extensions) - [Old installation method (MANUAL)](#old-installation-method-manual) - [Dependencies](#dependencies) @@ -35,11 +34,6 @@ then follow the prompt or just press enter to download every models. python scripts/download_models.py -y ``` -### Web Extensions - -On first run the script [tries to symlink](https://github.com/melMass/comfy_mtb/blob/d982b69a58c05ccead9c49370764beaa4549992a/__init__.py#L45-L61) the [web extensions](https://github.com/melMass/comfy_mtb/tree/main/web) to your comfy `web/extensions` folder. In case it fails you can manually copy the mtb folder to `ComfyUI/web/extensions` it only provides a color widget for now shared by a few nodes: - -color widget preview ## Old installation method (MANUAL) ### Dependencies diff --git a/nodes/batch.py b/nodes/batch.py index 108f94d..238c402 100644 --- a/nodes/batch.py +++ b/nodes/batch.py @@ -200,7 +200,7 @@ def INPUT_TYPES(cls): "required": { "mode": ( ["Single", "Steps"], - {"default": "Single"}, + {"default": "Steps"}, ), "count": ("INT", {"default": 1}), "min": ("FLOAT", {"default": 0.0}), @@ -411,11 +411,11 @@ def INPUT_TYPES(cls): "position_amount_x": ("FLOAT", {"default": 1.0}), "position_amount_y": ("FLOAT", {"default": 1.0}), "rotation_amount": ("FLOAT", {"default": 10.0}), - "frequency": ("FLOAT", {"default": 1.0}), - "frequency_divider": ("FLOAT", {"default": 1.0}), - "octaves": ("INT", {"default": 2}), + "frequency": ("FLOAT", {"default": 1.0, "min": 0.005}), + "frequency_divider": ("FLOAT", {"default": 1.0, "min": 0.005}), + "octaves": ("INT", {"default": 1, "min": 1}), "seed": ("INT", {"default": 0}), - } + }, } RETURN_TYPES = ("IMAGE", "FLOATS", "FLOATS", "FLOATS") @@ -423,14 +423,6 @@ def INPUT_TYPES(cls): FUNCTION = "apply_shake" CATEGORY = "mtb/batch" - def rehash(self, seed, octaves): - np.random.seed(seed) - self.position_offset = np.random.uniform(-1e3, 1e3, 3) - self.rotation_offset = np.random.uniform(-1e3, 1e3, 3) - self.noise_pattern = self.generate_fractal_noise_2d( - (512, 512), (32, 32), octaves - ) - # def interpolant(self, t): # return t * t * t * (t * (t * 6 - 15) + 10) @@ -456,8 +448,7 @@ def generate_perlin_noise_2d( Raises: ValueError: If shape is not a multiple of res. """ - if interpolant is None: - interpolant = DEFAULT_INTERPOLANT + interpolant = interpolant or DEFAULT_INTERPOLANT delta = (res[0] / shape[0], res[1] / shape[1]) d = (shape[0] // res[0], shape[1] // res[1]) grid = ( @@ -521,8 +512,8 @@ def generate_fractal_noise_2d( ValueError: If shape is not a multiple of (lacunarity**(octaves-1)*res). """ - if interpolant is None: - interpolant = DEFAULT_INTERPOLANT + interpolant = interpolant or DEFAULT_INTERPOLANT + noise = np.zeros(shape) frequency = 1 amplitude = 1 @@ -552,7 +543,13 @@ def apply_shake( octaves, seed, ): - self.rehash(seed, octaves) + # Rehash + np.random.seed(seed) + self.position_offset = np.random.uniform(-1e3, 1e3, 3) + self.rotation_offset = np.random.uniform(-1e3, 1e3, 3) + self.noise_pattern = self.generate_perlin_noise_2d( + (512, 512), (32, 32), (True, True) + ) # Assuming frame count is derived from the first dimension of images tensor frame_count = images.shape[0] From 537a0d8108d0caa3ab2daeafd1d25d680214ef26 Mon Sep 17 00:00:00 2001 From: melMass Date: Sat, 4 Nov 2023 16:21:36 +0100 Subject: [PATCH 12/12] =?UTF-8?q?chore:=20=E2=9C=A8=20update=20node=5Flist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- node_list.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/node_list.json b/node_list.json index 014dcd0..3b3e7ad 100644 --- a/node_list.json +++ b/node_list.json @@ -2,9 +2,11 @@ "Animation Builder (mtb)": "Convenient way to manage basic animation maths at the core of many of my workflows", "Any To String (mtb)": "Tries to take any input and convert it to a string", "Batch Float (mtb)": "Generates a batch of float values with interpolation", - "Batch Float Assemble (mtb)": "Assembles mutiple batches of floats into a single stream (batch)", - "Batch Float Fill (mtb)": "Fills a batch float with a single value until it reaches the target length", - "Batch Make (mtb)": "Simply duplicates the input frame as a batch", + "Batch Float Assemble (mtb)": "Assembles mutiple batches of floats into a single stream (batch)", + "Batch Float Fill (mtb)": "Fills a batch float with a single value until it reaches the target length", + "Batch Make (mtb)": "Simply duplicates the input frame as a batch", + "Batch Merge (mtb)": "Merges multiple image batches with different frame counts", + "Batch Shake (mtb)": "Applies a shaking effect to batches of images.", "Batch Shape (mtb)": "Generates a batch of 2D shapes with optional shading (experimental)", "Batch Transform (mtb)": "Transform a batch of images using a batch of keyframes", "Bbox (mtb)": "The bounding box (BBOX) custom type used by other nodes", @@ -38,6 +40,7 @@ "Load Image From Url (mtb)": "Load an image from the given URL", "Load Image Sequence (mtb)": "Load an image sequence from a folder. The current frame is used to determine which image to load.\n\n Usually used in conjunction with the `Primitive` node set to increment to load a sequence of images from a folder.\n Use -1 to load all matching frames as a batch.\n ", "Mask To Image (mtb)": "Converts a mask (alpha) to an RGB image with a color and background", + "Math Expression (mtb)": "Node to evaluate a simple math expression string", "Model Patch Seamless (mtb)": "Uses the stable diffusion 'hack' to infer seamless images by setting the model layers padding mode to circular (experimental)", "Qr Code (mtb)": "Basic QR Code generator", "Restore Face (mtb)": "Uses GFPGan to restore faces", @@ -45,6 +48,7 @@ "Save Image Grid (mtb)": "Save all the images in the input batch as a grid of images.", "Save Image Sequence (mtb)": "Save an image sequence to a folder. The current frame is used to determine which image to save.\n\n This is merely a wrapper around the `save_images` function with formatting for the output folder and filename.\n ", "Save Tensors (mtb)": "Save torch tensors (image, mask or latent) to disk, useful to debug things outside comfy", + "Sharpen (mtb)": "Sharpens an image using a Gaussian kernel.", "Smart Step (mtb)": "Utils to control the steps start/stop of the KAdvancedSampler in percentage", "Stack Images (mtb)": "Stack the input images horizontally or vertically", "String Replace (mtb)": "Basic string replacement",