diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..318ccb91b715b9c5d35bec945bacce10c3d4beaa 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +custom_nodes/ComfyUI_WanVideoWrapper/configs/T5_tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text +custom_nodes/ComfyUI-KJNodes/docs/images/2024-04-03_20_49_29-ComfyUI.png filter=lfs diff=lfs merge=lfs -text +custom_nodes/ComfyUI-KJNodes/fonts/FreeMono.ttf filter=lfs diff=lfs merge=lfs -text +custom_nodes/ComfyUI-KJNodes/fonts/FreeMonoBoldOblique.otf filter=lfs diff=lfs merge=lfs -text +custom_nodes/ComfyUI-KJNodes/fonts/TTNorms-Black.otf filter=lfs diff=lfs merge=lfs -text +custom_nodes/ComfyUI-to-Python-Extension/images/comfyui_to_python_banner.png filter=lfs diff=lfs merge=lfs -text +custom_nodes/ComfyUI-to-Python-Extension/images/SDXL-UI-Example.PNG filter=lfs diff=lfs merge=lfs -text +input/009c.mp4 filter=lfs diff=lfs merge=lfs -text +input/dasha.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4e8cea71e55139979744378e9918065955bcd7ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +__pycache__/ +*.py[cod] +/output/ +/input/ +!/input/example.png +/models/ +/temp/ +/custom_nodes/ +!custom_nodes/example_node.py.example +extra_model_paths.yaml +/.vs +.vscode/ +.idea/ +venv/ +.venv/ +/web/extensions/* +!/web/extensions/logging.js.example +!/web/extensions/core/ +/tests-ui/data/object_info.json +/user/ +*.log +web_custom_versions/ +.DS_Store +openapi.yaml +filtered-openapi.yaml +uv.lock diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000000000000000000000000000000000000..12f18712f4306f1b76e6b291956797b01469cd80 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,84 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +# Use forward slashes (/) also on windows to provide an os agnostic path +script_location = alembic_db + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library. +# Any required deps can installed by adding `alembic[tz]` to the pip requirements +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic_db/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic_db/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +# version_path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +version_path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = sqlite:///user/comfyui.db + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME diff --git a/alembic_db/README.md b/alembic_db/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3b808c7cab30eaab91c9ff5c1b2d5cc960904e14 --- /dev/null +++ b/alembic_db/README.md @@ -0,0 +1,4 @@ +## Generate new revision + +1. Update models in `/app/database/models.py` +2. Run `alembic revision --autogenerate -m "{your message}"` diff --git a/alembic_db/env.py b/alembic_db/env.py new file mode 100644 index 0000000000000000000000000000000000000000..4d7770679875e2fad065fba0c9222758a40d266b --- /dev/null +++ b/alembic_db/env.py @@ -0,0 +1,64 @@ +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + + +from app.database.models import Base +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + Calls to context.execute() here emit the given string to the + script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + In this scenario we need to create an Engine + and associate a connection with the context. + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic_db/script.py.mako b/alembic_db/script.py.mako new file mode 100644 index 0000000000000000000000000000000000000000..480b130d632ca677c11f23d9fe82cf4014d15e0c --- /dev/null +++ b/alembic_db/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/api_server/__init__.py b/api_server/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/api_server/routes/__init__.py b/api_server/routes/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/api_server/routes/internal/README.md b/api_server/routes/internal/README.md new file mode 100644 index 0000000000000000000000000000000000000000..35330c36f83962385b4afe4653c5967f2bdb73c1 --- /dev/null +++ b/api_server/routes/internal/README.md @@ -0,0 +1,3 @@ +# ComfyUI Internal Routes + +All routes under the `/internal` path are designated for **internal use by ComfyUI only**. These routes are not intended for use by external applications may change at any time without notice. diff --git a/api_server/routes/internal/__init__.py b/api_server/routes/internal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/api_server/routes/internal/internal_routes.py b/api_server/routes/internal/internal_routes.py new file mode 100644 index 0000000000000000000000000000000000000000..613b0f7c7cf1cdbb24aae3e880e749839ac1110e --- /dev/null +++ b/api_server/routes/internal/internal_routes.py @@ -0,0 +1,73 @@ +from aiohttp import web +from typing import Optional +from folder_paths import folder_names_and_paths, get_directory_by_type +from api_server.services.terminal_service import TerminalService +import app.logger +import os + +class InternalRoutes: + ''' + The top level web router for internal routes: /internal/* + The endpoints here should NOT be depended upon. It is for ComfyUI frontend use only. + Check README.md for more information. + ''' + + def __init__(self, prompt_server): + self.routes: web.RouteTableDef = web.RouteTableDef() + self._app: Optional[web.Application] = None + self.prompt_server = prompt_server + self.terminal_service = TerminalService(prompt_server) + + def setup_routes(self): + @self.routes.get('/logs') + async def get_logs(request): + return web.json_response("".join([(l["t"] + " - " + l["m"]) for l in app.logger.get_logs()])) + + @self.routes.get('/logs/raw') + async def get_raw_logs(request): + self.terminal_service.update_size() + return web.json_response({ + "entries": list(app.logger.get_logs()), + "size": {"cols": self.terminal_service.cols, "rows": self.terminal_service.rows} + }) + + @self.routes.patch('/logs/subscribe') + async def subscribe_logs(request): + json_data = await request.json() + client_id = json_data["clientId"] + enabled = json_data["enabled"] + if enabled: + self.terminal_service.subscribe(client_id) + else: + self.terminal_service.unsubscribe(client_id) + + return web.Response(status=200) + + + @self.routes.get('/folder_paths') + async def get_folder_paths(request): + response = {} + for key in folder_names_and_paths: + response[key] = folder_names_and_paths[key][0] + return web.json_response(response) + + @self.routes.get('/files/{directory_type}') + async def get_files(request: web.Request) -> web.Response: + directory_type = request.match_info['directory_type'] + if directory_type not in ("output", "input", "temp"): + return web.json_response({"error": "Invalid directory type"}, status=400) + + directory = get_directory_by_type(directory_type) + sorted_files = sorted( + (entry for entry in os.scandir(directory) if entry.is_file()), + key=lambda entry: -entry.stat().st_mtime + ) + return web.json_response([entry.name for entry in sorted_files], status=200) + + + def get_app(self): + if self._app is None: + self._app = web.Application() + self.setup_routes() + self._app.add_routes(self.routes) + return self._app diff --git a/api_server/services/__init__.py b/api_server/services/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/api_server/services/terminal_service.py b/api_server/services/terminal_service.py new file mode 100644 index 0000000000000000000000000000000000000000..ab4371f4f855c6d347744f751fd1bc5cc0f5a173 --- /dev/null +++ b/api_server/services/terminal_service.py @@ -0,0 +1,60 @@ +from app.logger import on_flush +import os +import shutil + + +class TerminalService: + def __init__(self, server): + self.server = server + self.cols = None + self.rows = None + self.subscriptions = set() + on_flush(self.send_messages) + + def get_terminal_size(self): + try: + size = os.get_terminal_size() + return (size.columns, size.lines) + except OSError: + try: + size = shutil.get_terminal_size() + return (size.columns, size.lines) + except OSError: + return (80, 24) # fallback to 80x24 + + def update_size(self): + columns, lines = self.get_terminal_size() + changed = False + + if columns != self.cols: + self.cols = columns + changed = True + + if lines != self.rows: + self.rows = lines + changed = True + + if changed: + return {"cols": self.cols, "rows": self.rows} + + return None + + def subscribe(self, client_id): + self.subscriptions.add(client_id) + + def unsubscribe(self, client_id): + self.subscriptions.discard(client_id) + + def send_messages(self, entries): + if not len(entries) or not len(self.subscriptions): + return + + new_size = self.update_size() + + for client_id in self.subscriptions.copy(): # prevent: Set changed size during iteration + if client_id not in self.server.sockets: + # Automatically unsub if the socket has disconnected + self.unsubscribe(client_id) + continue + + self.server.send_sync("logs", {"entries": entries, "size": new_size}, client_id) diff --git a/api_server/utils/file_operations.py b/api_server/utils/file_operations.py new file mode 100644 index 0000000000000000000000000000000000000000..32d6e047a5da09d0f382bb3fa93ce8d52b699aa7 --- /dev/null +++ b/api_server/utils/file_operations.py @@ -0,0 +1,42 @@ +import os +from typing import List, Union, TypedDict, Literal +from typing_extensions import TypeGuard +class FileInfo(TypedDict): + name: str + path: str + type: Literal["file"] + size: int + +class DirectoryInfo(TypedDict): + name: str + path: str + type: Literal["directory"] + +FileSystemItem = Union[FileInfo, DirectoryInfo] + +def is_file_info(item: FileSystemItem) -> TypeGuard[FileInfo]: + return item["type"] == "file" + +class FileSystemOperations: + @staticmethod + def walk_directory(directory: str) -> List[FileSystemItem]: + file_list: List[FileSystemItem] = [] + for root, dirs, files in os.walk(directory): + for name in files: + file_path = os.path.join(root, name) + relative_path = os.path.relpath(file_path, directory) + file_list.append({ + "name": name, + "path": relative_path, + "type": "file", + "size": os.path.getsize(file_path) + }) + for name in dirs: + dir_path = os.path.join(root, name) + relative_path = os.path.relpath(dir_path, directory) + file_list.append({ + "name": name, + "path": relative_path, + "type": "directory" + }) + return file_list diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..105c366ac8b567d5474f3c76892039016f68f210 --- /dev/null +++ b/app.py @@ -0,0 +1,560 @@ +import os +import cv2 +import numpy as np +import random +import sys +import subprocess +from typing import Sequence, Mapping, Any, Union +import torch +from tqdm import tqdm +import argparse +import json +import logging + +import shutil +import gradio as gr +import spaces +from huggingface_hub import snapshot_download +import time +import traceback + +from utils import get_path_after_pexel + +LOCAL_GRADIO_TMP = os.path.abspath("./gradio_tmp") +os.makedirs(LOCAL_GRADIO_TMP, exist_ok=True) +os.environ["GRADIO_TEMP_DIR"] = LOCAL_GRADIO_TMP + + +HF_REPOS = { + "QingyanBai/Ditto_models": ["models_comfy/ditto_global_comfy.safetensors"], + "Kijai/WanVideo_comfy": [ + "Wan2_1-T2V-14B_fp8_e4m3fn.safetensors", + "Wan21_CausVid_14B_T2V_lora_rank32_v2.safetensors", + "Wan2_1_VAE_bf16.safetensors", + "umt5-xxl-enc-bf16.safetensors", + ], +} + +MODELS_ROOT = os.path.abspath(os.path.join(os.getcwd(), "models")) +PATHS = { + "diffusion_model": os.path.join(MODELS_ROOT, "diffusion_models"), + "vae_wan": os.path.join(MODELS_ROOT, "vae", "wan"), + "loras": os.path.join(MODELS_ROOT, "loras"), + "text_encoders": os.path.join(MODELS_ROOT, "text_encoders"), +} + +REQUIRED_FILES = [ + ("Wan2_1-T2V-14B_fp8_e4m3fn.safetensors", "diffusion_model"), + ("ditto_global_comfy.safetensors", "diffusion_model"), + ("Wan21_CausVid_14B_T2V_lora_rank32_v2.safetensors", "loras"), + ("Wan2_1_VAE_bf16.safetensors", "vae_wan"), + ("umt5-xxl-enc-bf16.safetensors", "text_encoders"), +] + +def ensure_dir(path: str) -> None: + os.makedirs(path, exist_ok=True) + +def ensure_models() -> None: + for filename, key in REQUIRED_FILES: + target_dir = PATHS[key] + ensure_dir(target_dir) + target_path = os.path.join(target_dir, filename) + ready_flag = os.path.join(target_dir, f"{filename}.READY") + + if os.path.exists(target_path) and os.path.getsize(target_path) > 0: + open(ready_flag, "a").close() + continue + + repo_id = None + repo_file_path = None + for repo, files in HF_REPOS.items(): + for file_path in files: + if filename in file_path: + repo_id = repo + repo_file_path = file_path + break + if repo_id: + break + + if repo_id is None: + raise RuntimeError(f"Could not find repository for file: {filename}") + + print(f"Downloading {filename} from {repo_id} to {target_dir} ...") + + snapshot_download( + repo_id=repo_id, + local_dir=target_dir, + local_dir_use_symlinks=False, + allow_patterns=[repo_file_path], + token=os.getenv("HF_TOKEN", None), + ) + + if not os.path.exists(target_path): + found = [] + for root, _, files in os.walk(target_dir): + for f in files: + if f == filename: + found.append(os.path.join(root, f)) + if found: + src = found[0] + if src != target_path: + shutil.copy2(src, target_path) + + if not os.path.exists(target_path): + raise RuntimeError(f"Failed to download required file: {filename}") + + open(ready_flag, "w").close() + print(f"Downloaded and ready: {target_path}") +ensure_models() + + +def ensure_t5_tokenizer() -> None: + """ + Ensure the local T5 tokenizer folder exists and contains valid files. + If missing or corrupted, download from 'google/umt5-xxl' and save locally + to the exact path expected by the WanVideo wrapper nodes. + """ + try: + script_directory = os.path.dirname(os.path.abspath(__file__)) + tokenizer_dir = os.path.join( + script_directory, + "custom_nodes", + "ComfyUI_WanVideoWrapper", + "configs", + "T5_tokenizer", + ) + os.makedirs(tokenizer_dir, exist_ok=True) + + required_files = [ + "tokenizer.json", + "tokenizer_config.json", + "spiece.model", + "special_tokens_map.json", + ] + + def is_valid(path: str) -> bool: + return os.path.exists(path) and os.path.getsize(path) > 0 + + all_ok = all(is_valid(os.path.join(tokenizer_dir, f)) for f in required_files) + if all_ok: + print(f"T5 tokenizer ready at: {tokenizer_dir}") + return + + print(f"Preparing T5 tokenizer at: {tokenizer_dir} ...") + from transformers import AutoTokenizer + + tok = AutoTokenizer.from_pretrained( + "google/umt5-xxl", + use_fast=True, + trust_remote_code=False, + ) + tok.save_pretrained(tokenizer_dir) + + # Re-check + all_ok = all(is_valid(os.path.join(tokenizer_dir, f)) for f in required_files) + if not all_ok: + raise RuntimeError("Tokenizer files not fully prepared after save_pretrained") + print("T5 tokenizer prepared successfully.") + except Exception as e: + print(f"Failed to prepare T5 tokenizer: {e}\n{traceback.format_exc()}") + raise + + +ensure_t5_tokenizer() + + +def setup_global_logging_filter(): + class MemoryLogFilter(logging.Filter): + def filter(self, record): + msg = record.getMessage() + keywords = [ + "Allocated memory:", + "Max allocated memory:", + "Max reserved memory:", + "memory=", + "max_memory=", + "max_reserved=", + "Block swap memory summary", + "Transformer blocks on", + "Total memory used by", + "Non-blocking memory transfer" + ] + return not any(kw in msg for kw in keywords) + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + force=True + ) + logging.getLogger().handlers[0].addFilter(MemoryLogFilter()) + + +setup_global_logging_filter() + + +def tensor_to_video(video_tensor, output_path, fps=20, crf=20): + frames = video_tensor.detach().cpu().numpy() + if frames.dtype != np.uint8: + if frames.max() <= 1.0: + frames = (frames * 255).astype(np.uint8) + else: + frames = frames.astype(np.uint8) + num_frames, height, width, _ = frames.shape + command = [ + 'ffmpeg', + '-y', + '-f', 'rawvideo', + '-vcodec', 'rawvideo', + '-pix_fmt', 'rgb24', + '-s', f'{width}x{height}', + '-r', str(fps), + '-i', '-', + '-c:v', 'libx264', + '-pix_fmt', 'yuv420p', + '-crf', str(crf), + '-preset', 'medium', + '-r', str(fps), + '-an', + output_path + ] + + with subprocess.Popen(command, stdin=subprocess.PIPE, stderr=subprocess.PIPE) as proc: + for frame in frames: + proc.stdin.write(frame.tobytes()) + proc.stdin.close() + if proc.stderr is not None: + proc.stderr.read() + + +def get_value_at_index(obj: Union[Sequence, Mapping], index: int) -> Any: + try: + return obj[index] + except KeyError: + return obj["result"][index] + + +def find_path(name: str, path: str = None) -> str: + if path is None: + path = os.getcwd() + if name in os.listdir(path): + path_name = os.path.join(path, name) + print(f"{name} found: {path_name}") + return path_name + parent_directory = os.path.dirname(path) + if parent_directory == path: + return None + return find_path(name, parent_directory) + + +def add_comfyui_directory_to_sys_path() -> None: + comfyui_path = find_path("ComfyUI") + if comfyui_path is not None and os.path.isdir(comfyui_path): + if comfyui_path not in sys.path: + sys.path.append(comfyui_path) + print(f"'{comfyui_path}' added to sys.path") + + +def add_extra_model_paths() -> None: + try: + from main import load_extra_path_config + except ImportError: + print( + "Could not import load_extra_path_config from main.py. Looking in utils.extra_config instead." + ) + from utils.extra_config import load_extra_path_config + + extra_model_paths = find_path("extra_model_paths.yaml") + + if extra_model_paths is not None: + load_extra_path_config(extra_model_paths) + else: + print("Could not find the extra_model_paths config file.") + + +add_comfyui_directory_to_sys_path() +add_extra_model_paths() + + +def import_custom_nodes() -> None: + import asyncio + import execution + from nodes import init_extra_nodes + import server + + if getattr(import_custom_nodes, "_initialized", False): + return + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + server_instance = server.PromptServer(loop) + execution.PromptQueue(server_instance) + init_extra_nodes() + import_custom_nodes._initialized = True + + +from nodes import NODE_CLASS_MAPPINGS + +print(f"Loading custom nodes and models...") +import_custom_nodes() + + +@spaces.GPU() +def run_pipeline(vpath, prompt, width, height, fps, frame_count, outdir): + try: + import gc + # Clean memory before starting + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + os.makedirs(outdir, exist_ok=True) + + with torch.inference_mode(): + from custom_nodes.ComfyUI_WanVideoWrapper import nodes as wan_nodes + vhs_loadvideo = NODE_CLASS_MAPPINGS["VHS_LoadVideo"]() + + # Set model and settings. + wanvideovacemodelselect = wan_nodes.WanVideoVACEModelSelect() + wanvideovacemodelselect_89 = wanvideovacemodelselect.getvacepath( + vace_model="ditto_global_comfy.safetensors" + ) + + wanvideoslg = wan_nodes.WanVideoSLG() + wanvideoslg_113 = wanvideoslg.process( + blocks="2", + start_percent=0.20000000000000004, + end_percent=0.7000000000000002, + ) + wanvideovaeloader = wan_nodes.WanVideoVAELoader() + wanvideovaeloader_133 = wanvideovaeloader.loadmodel( + model_name="wan/Wan2_1_VAE_bf16.safetensors", precision="bf16" + ) + + loadwanvideot5textencoder = wan_nodes.LoadWanVideoT5TextEncoder() + loadwanvideot5textencoder_134 = loadwanvideot5textencoder.loadmodel( + model_name="umt5-xxl-enc-bf16.safetensors", + precision="bf16", + load_device="offload_device", + quantization="disabled", + ) + + wanvideoblockswap = wan_nodes.WanVideoBlockSwap() + wanvideoblockswap_137 = wanvideoblockswap.setargs( + blocks_to_swap=30, + offload_img_emb=False, + offload_txt_emb=False, + use_non_blocking=True, + vace_blocks_to_swap=0, + ) + + wanvideoloraselect = wan_nodes.WanVideoLoraSelect() + wanvideoloraselect_380 = wanvideoloraselect.getlorapath( + lora="Wan21_CausVid_14B_T2V_lora_rank32_v2.safetensors", + strength=1.0, + low_mem_load=False, + ) + + wanvideomodelloader = wan_nodes.WanVideoModelLoader() + imageresizekjv2 = NODE_CLASS_MAPPINGS["ImageResizeKJv2"]() + wanvideovaceencode = wan_nodes.WanVideoVACEEncode() + wanvideotextencode = wan_nodes.WanVideoTextEncode() + wanvideosampler = wan_nodes.WanVideoSampler() + wanvideodecode = wan_nodes.WanVideoDecode() + wanvideomodelloader_142 = wanvideomodelloader.loadmodel( + model="Wan2_1-T2V-14B_fp8_e4m3fn.safetensors", + base_precision="fp16", + quantization="disabled", + load_device="offload_device", + attention_mode="sdpa", + block_swap_args=get_value_at_index(wanvideoblockswap_137, 0), + lora=get_value_at_index(wanvideoloraselect_380, 0), + vace_model=get_value_at_index(wanvideovacemodelselect_89, 0), + ) + + fname = os.path.basename(vpath) + fname_clean = os.path.splitext(fname)[0] + + vhs_loadvideo_70 = vhs_loadvideo.load_video( + video=vpath, + force_rate=24, + custom_width=width, + custom_height=height, + frame_load_cap=frame_count, + skip_first_frames=1, + select_every_nth=1, + format="AnimateDiff", + unique_id=16696422174153060213, + ) + + imageresizekjv2_205 = imageresizekjv2.resize( + width=width, + height=height, + upscale_method="area", + keep_proportion="resize", + pad_color="0, 0, 0", + crop_position="center", + divisible_by=8, + device="cpu", + image=get_value_at_index(vhs_loadvideo_70, 0), + ) + wanvideovaceencode_29 = wanvideovaceencode.process( + width=width, + height=height, + num_frames=frame_count, + strength=0.9750000000000002, + vace_start_percent=0, + vace_end_percent=1, + tiled_vae=False, + vae=get_value_at_index(wanvideovaeloader_133, 0), + input_frames=get_value_at_index(imageresizekjv2_205, 0), + ) + + wanvideotextencode_148 = wanvideotextencode.process( + positive_prompt=prompt, + negative_prompt="flickering artifact, jpg artifacts, compression, distortion, morphing, low-res, fake, oversaturated, overexposed, over bright, strange behavior, distorted limbs, unnatural motion, unrealistic anatomy, glitch, extra limbs,", + force_offload=True, + t5=get_value_at_index(loadwanvideot5textencoder_134, 0), + model_to_offload=get_value_at_index(wanvideomodelloader_142, 0), + ) + + # Clean memory before sampling (most memory-intensive step) + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + wanvideosampler_2 = wanvideosampler.process( + steps=4, + cfg=1.2000000000000002, + shift=2.0000000000000004, + seed=random.randint(1, 2 ** 64), + force_offload=True, + scheduler="unipc", + riflex_freq_index=0, + denoise_strength=1, + batched_cfg=False, + rope_function="comfy", + model=get_value_at_index(wanvideomodelloader_142, 0), + image_embeds=get_value_at_index(wanvideovaceencode_29, 0), + text_embeds=get_value_at_index(wanvideotextencode_148, 0), + slg_args=get_value_at_index(wanvideoslg_113, 0), + ) + res = wanvideodecode.decode( + enable_vae_tiling=False, + tile_x=272, + tile_y=272, + tile_stride_x=144, + tile_stride_y=128, + vae=get_value_at_index(wanvideovaeloader_133, 0), + samples=get_value_at_index(wanvideosampler_2, 0), + ) + save_path = os.path.join(outdir, f'{fname_clean}_edit.mp4') + tensor_to_video(res[0], save_path, fps=fps) + + # Clean up memory after generation + del res + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + print(f"Done. Saved to: {save_path}") + return save_path + except Exception as e: + err = f"Error: {e}\n{traceback.format_exc()}" + print(err) + # Clean memory on error too + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + raise + + +@spaces.GPU() +def gradio_infer(vfile, prompt, width, height, fps, frame_count, progress=gr.Progress(track_tqdm=True)): + if vfile is None: + return None, "Please upload the video!", "\n".join(logs) + + vpath = vfile if isinstance(vfile, str) else vfile.name + if not os.path.exists(vpath) and hasattr(vfile, "save"): + os.makedirs("uploads", exist_ok=True) + vpath = os.path.join("uploads", os.path.basename(vfile.name)) + vfile.save(vpath) + + outdir = "results" + os.makedirs(outdir, exist_ok=True) + + save_path = run_pipeline( + vpath=vpath, + prompt=prompt, + width=int(width), + height=int(height), + fps=int(fps), + frame_count=int(frame_count), + outdir=outdir, + ) + return save_path + + +def build_interface(): + with gr.Blocks(title="Ditto") as demo: + gr.Markdown( + """# Ditto: Scaling Instruction-Based Video Editing with a High-Quality Synthetic Dataset + +
+ +Note1: The backend of this demo is comfy, please note that due to the use of quantized and distilled models, there may be some quality degradation. +Note2: Considering the limited memory, please try test cases with lower resolution and frame count, otherwise it may cause out of memory error. +If you like this project, please consider starring the repo to motivate us. Thank you! + """ + ) + + with gr.Column(): + with gr.Row(): + vfile = gr.Video(label="Input Video", value=os.path.join("input", "dasha.mp4"), + sources="upload", interactive=True) + out_video = gr.Video(label="Result") + prompt = gr.Textbox(label="Editing Instruction", value="Make it in the style of Japanese anime") + with gr.Row(): + width = gr.Number(label="Width", value=576, precision=0) + height = gr.Number(label="Height", value=324, precision=0) + fps = gr.Number(label="FPS", value=20, precision=0) + frame_count = gr.Number(label="Frame Count", value=49, precision=0) + run_btn = gr.Button("Run", variant="primary") + + run_btn.click( + fn=gradio_infer, + inputs=[vfile, prompt, width, height, fps, frame_count], + outputs=[out_video] + ) + examples = [ + [ + os.path.join("input", "dasha.mp4"), + "Add some fire and flame to the background", + 576, 324, 20, 49 + ], + [ + os.path.join("input", "dasha.mp4"), + "Make it in the style of pencil sketch", + 576, 324, 20, 49 + ], + ] + gr.Examples( + examples=examples, + inputs=[vfile, prompt, width, height, fps, frame_count], + label="Examples" + ) + return demo + + +if __name__ == "__main__": + demo = build_interface() + demo.launch() diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/app/app_settings.py b/app/app_settings.py new file mode 100644 index 0000000000000000000000000000000000000000..c7ac73bf6a59d3e89647aed13b70ca9428e16f2a --- /dev/null +++ b/app/app_settings.py @@ -0,0 +1,65 @@ +import os +import json +from aiohttp import web +import logging + + +class AppSettings(): + def __init__(self, user_manager): + self.user_manager = user_manager + + def get_settings(self, request): + try: + file = self.user_manager.get_request_user_filepath( + request, + "comfy.settings.json" + ) + except KeyError as e: + logging.error("User settings not found.") + raise web.HTTPUnauthorized() from e + if os.path.isfile(file): + try: + with open(file) as f: + return json.load(f) + except: + logging.error(f"The user settings file is corrupted: {file}") + return {} + else: + return {} + + def save_settings(self, request, settings): + file = self.user_manager.get_request_user_filepath( + request, "comfy.settings.json") + with open(file, "w") as f: + f.write(json.dumps(settings, indent=4)) + + def add_routes(self, routes): + @routes.get("/settings") + async def get_settings(request): + return web.json_response(self.get_settings(request)) + + @routes.get("/settings/{id}") + async def get_setting(request): + value = None + settings = self.get_settings(request) + setting_id = request.match_info.get("id", None) + if setting_id and setting_id in settings: + value = settings[setting_id] + return web.json_response(value) + + @routes.post("/settings") + async def post_settings(request): + settings = self.get_settings(request) + new_settings = await request.json() + self.save_settings(request, {**settings, **new_settings}) + return web.Response(status=200) + + @routes.post("/settings/{id}") + async def post_setting(request): + setting_id = request.match_info.get("id", None) + if not setting_id: + return web.Response(status=400) + settings = self.get_settings(request) + settings[setting_id] = await request.json() + self.save_settings(request, settings) + return web.Response(status=200) diff --git a/app/custom_node_manager.py b/app/custom_node_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..281febca952363659eb3925280fab00f07c986d0 --- /dev/null +++ b/app/custom_node_manager.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import os +import folder_paths +import glob +from aiohttp import web +import json +import logging +from functools import lru_cache + +from utils.json_util import merge_json_recursive + + +# Extra locale files to load into main.json +EXTRA_LOCALE_FILES = [ + "nodeDefs.json", + "commands.json", + "settings.json", +] + + +def safe_load_json_file(file_path: str) -> dict: + if not os.path.exists(file_path): + return {} + + try: + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) + except json.JSONDecodeError: + logging.error(f"Error loading {file_path}") + return {} + + +class CustomNodeManager: + @lru_cache(maxsize=1) + def build_translations(self): + """Load all custom nodes translations during initialization. Translations are + expected to be loaded from `locales/` folder. + + The folder structure is expected to be the following: + - custom_nodes/ + - custom_node_1/ + - locales/ + - en/ + - main.json + - commands.json + - settings.json + + returned translations are expected to be in the following format: + { + "en": { + "nodeDefs": {...}, + "commands": {...}, + "settings": {...}, + ...{other main.json keys} + } + } + """ + + translations = {} + + for folder in folder_paths.get_folder_paths("custom_nodes"): + # Sort glob results for deterministic ordering + for custom_node_dir in sorted(glob.glob(os.path.join(folder, "*/"))): + locales_dir = os.path.join(custom_node_dir, "locales") + if not os.path.exists(locales_dir): + continue + + for lang_dir in glob.glob(os.path.join(locales_dir, "*/")): + lang_code = os.path.basename(os.path.dirname(lang_dir)) + + if lang_code not in translations: + translations[lang_code] = {} + + # Load main.json + main_file = os.path.join(lang_dir, "main.json") + node_translations = safe_load_json_file(main_file) + + # Load extra locale files + for extra_file in EXTRA_LOCALE_FILES: + extra_file_path = os.path.join(lang_dir, extra_file) + key = extra_file.split(".")[0] + json_data = safe_load_json_file(extra_file_path) + if json_data: + node_translations[key] = json_data + + if node_translations: + translations[lang_code] = merge_json_recursive( + translations[lang_code], node_translations + ) + + return translations + + def add_routes(self, routes, webapp, loadedModules): + + example_workflow_folder_names = ["example_workflows", "example", "examples", "workflow", "workflows"] + + @routes.get("/workflow_templates") + async def get_workflow_templates(request): + """Returns a web response that contains the map of custom_nodes names and their associated workflow templates. The ones without templates are omitted.""" + + files = [] + + for folder in folder_paths.get_folder_paths("custom_nodes"): + for folder_name in example_workflow_folder_names: + pattern = os.path.join(folder, f"*/{folder_name}/*.json") + matched_files = glob.glob(pattern) + files.extend(matched_files) + + workflow_templates_dict = ( + {} + ) # custom_nodes folder name -> example workflow names + for file in files: + custom_nodes_name = os.path.basename( + os.path.dirname(os.path.dirname(file)) + ) + workflow_name = os.path.splitext(os.path.basename(file))[0] + workflow_templates_dict.setdefault(custom_nodes_name, []).append( + workflow_name + ) + return web.json_response(workflow_templates_dict) + + # Serve workflow templates from custom nodes. + for module_name, module_dir in loadedModules: + for folder_name in example_workflow_folder_names: + workflows_dir = os.path.join(module_dir, folder_name) + + if os.path.exists(workflows_dir): + if folder_name != "example_workflows": + logging.debug( + "Found example workflow folder '%s' for custom node '%s', consider renaming it to 'example_workflows'", + folder_name, module_name) + + webapp.add_routes( + [ + web.static( + "/api/workflow_templates/" + module_name, workflows_dir + ) + ] + ) + + @routes.get("/i18n") + async def get_i18n(request): + """Returns translations from all custom nodes' locales folders.""" + return web.json_response(self.build_translations()) diff --git a/app/database/db.py b/app/database/db.py new file mode 100644 index 0000000000000000000000000000000000000000..1de8b80edd8a5761039c7314c6387d2184a15e14 --- /dev/null +++ b/app/database/db.py @@ -0,0 +1,112 @@ +import logging +import os +import shutil +from app.logger import log_startup_warning +from utils.install_util import get_missing_requirements_message +from comfy.cli_args import args + +_DB_AVAILABLE = False +Session = None + + +try: + from alembic import command + from alembic.config import Config + from alembic.runtime.migration import MigrationContext + from alembic.script import ScriptDirectory + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + _DB_AVAILABLE = True +except ImportError as e: + log_startup_warning( + f""" +------------------------------------------------------------------------ +Error importing dependencies: {e} +{get_missing_requirements_message()} +This error is happening because ComfyUI now uses a local sqlite database. +------------------------------------------------------------------------ +""".strip() + ) + + +def dependencies_available(): + """ + Temporary function to check if the dependencies are available + """ + return _DB_AVAILABLE + + +def can_create_session(): + """ + Temporary function to check if the database is available to create a session + During initial release there may be environmental issues (or missing dependencies) that prevent the database from being created + """ + return dependencies_available() and Session is not None + + +def get_alembic_config(): + root_path = os.path.join(os.path.dirname(__file__), "../..") + config_path = os.path.abspath(os.path.join(root_path, "alembic.ini")) + scripts_path = os.path.abspath(os.path.join(root_path, "alembic_db")) + + config = Config(config_path) + config.set_main_option("script_location", scripts_path) + config.set_main_option("sqlalchemy.url", args.database_url) + + return config + + +def get_db_path(): + url = args.database_url + if url.startswith("sqlite:///"): + return url.split("///")[1] + else: + raise ValueError(f"Unsupported database URL '{url}'.") + + +def init_db(): + db_url = args.database_url + logging.debug(f"Database URL: {db_url}") + db_path = get_db_path() + db_exists = os.path.exists(db_path) + + config = get_alembic_config() + + # Check if we need to upgrade + engine = create_engine(db_url) + conn = engine.connect() + + context = MigrationContext.configure(conn) + current_rev = context.get_current_revision() + + script = ScriptDirectory.from_config(config) + target_rev = script.get_current_head() + + if target_rev is None: + logging.warning("No target revision found.") + elif current_rev != target_rev: + # Backup the database pre upgrade + backup_path = db_path + ".bkp" + if db_exists: + shutil.copy(db_path, backup_path) + else: + backup_path = None + + try: + command.upgrade(config, target_rev) + logging.info(f"Database upgraded from {current_rev} to {target_rev}") + except Exception as e: + if backup_path: + # Restore the database from backup if upgrade fails + shutil.copy(backup_path, db_path) + os.remove(backup_path) + logging.exception("Error upgrading database: ") + raise e + + global Session + Session = sessionmaker(bind=engine) + + +def create_session(): + return Session() diff --git a/app/database/models.py b/app/database/models.py new file mode 100644 index 0000000000000000000000000000000000000000..6facfb8f2b5e7382274021579a453ad3b7853bf7 --- /dev/null +++ b/app/database/models.py @@ -0,0 +1,14 @@ +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + + +def to_dict(obj): + fields = obj.__table__.columns.keys() + return { + field: (val.to_dict() if hasattr(val, "to_dict") else val) + for field in fields + if (val := getattr(obj, field)) + } + +# TODO: Define models here diff --git a/app/frontend_management.py b/app/frontend_management.py new file mode 100644 index 0000000000000000000000000000000000000000..001ebbecb4f291b938b3d9aaa1d2a3c67dbc98e5 --- /dev/null +++ b/app/frontend_management.py @@ -0,0 +1,326 @@ +from __future__ import annotations +import argparse +import logging +import os +import re +import sys +import tempfile +import zipfile +import importlib +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path +from typing import TypedDict, Optional +from importlib.metadata import version + +import requests +from typing_extensions import NotRequired + +from utils.install_util import get_missing_requirements_message, requirements_path + +from comfy.cli_args import DEFAULT_VERSION_STRING +import app.logger + + +def frontend_install_warning_message(): + return f""" +{get_missing_requirements_message()} + +This error is happening because the ComfyUI frontend is no longer shipped as part of the main repo but as a pip package instead. +""".strip() + + +def check_frontend_version(): + """Check if the frontend version is up to date.""" + + def parse_version(version: str) -> tuple[int, int, int]: + return tuple(map(int, version.split("."))) + + try: + frontend_version_str = version("comfyui-frontend-package") + frontend_version = parse_version(frontend_version_str) + with open(requirements_path, "r", encoding="utf-8") as f: + required_frontend = parse_version(f.readline().split("=")[-1]) + if frontend_version < required_frontend: + app.logger.log_startup_warning( + f""" +________________________________________________________________________ +WARNING WARNING WARNING WARNING WARNING + +Installed frontend version {".".join(map(str, frontend_version))} is lower than the recommended version {".".join(map(str, required_frontend))}. + +{frontend_install_warning_message()} +________________________________________________________________________ +""".strip() + ) + else: + logging.info("ComfyUI frontend version: {}".format(frontend_version_str)) + except Exception as e: + logging.error(f"Failed to check frontend version: {e}") + + +REQUEST_TIMEOUT = 10 # seconds + + +class Asset(TypedDict): + url: str + + +class Release(TypedDict): + id: int + tag_name: str + name: str + prerelease: bool + created_at: str + published_at: str + body: str + assets: NotRequired[list[Asset]] + + +@dataclass +class FrontEndProvider: + owner: str + repo: str + + @property + def folder_name(self) -> str: + return f"{self.owner}_{self.repo}" + + @property + def release_url(self) -> str: + return f"https://api.github.com/repos/{self.owner}/{self.repo}/releases" + + @cached_property + def all_releases(self) -> list[Release]: + releases = [] + api_url = self.release_url + while api_url: + response = requests.get(api_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() # Raises an HTTPError if the response was an error + releases.extend(response.json()) + # GitHub uses the Link header to provide pagination links. Check if it exists and update api_url accordingly. + if "next" in response.links: + api_url = response.links["next"]["url"] + else: + api_url = None + return releases + + @cached_property + def latest_release(self) -> Release: + latest_release_url = f"{self.release_url}/latest" + response = requests.get(latest_release_url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() # Raises an HTTPError if the response was an error + return response.json() + + @cached_property + def latest_prerelease(self) -> Release: + """Get the latest pre-release version - even if it's older than the latest release""" + release = [release for release in self.all_releases if release["prerelease"]] + + if not release: + raise ValueError("No pre-releases found") + + # GitHub returns releases in reverse chronological order, so first is latest + return release[0] + + def get_release(self, version: str) -> Release: + if version == "latest": + return self.latest_release + elif version == "prerelease": + return self.latest_prerelease + else: + for release in self.all_releases: + if release["tag_name"] in [version, f"v{version}"]: + return release + raise ValueError(f"Version {version} not found in releases") + + +def download_release_asset_zip(release: Release, destination_path: str) -> None: + """Download dist.zip from github release.""" + asset_url = None + for asset in release.get("assets", []): + if asset["name"] == "dist.zip": + asset_url = asset["url"] + break + + if not asset_url: + raise ValueError("dist.zip not found in the release assets") + + # Use a temporary file to download the zip content + with tempfile.TemporaryFile() as tmp_file: + headers = {"Accept": "application/octet-stream"} + response = requests.get( + asset_url, headers=headers, allow_redirects=True, timeout=REQUEST_TIMEOUT + ) + response.raise_for_status() # Ensure we got a successful response + + # Write the content to the temporary file + tmp_file.write(response.content) + + # Go back to the beginning of the temporary file + tmp_file.seek(0) + + # Extract the zip file content to the destination path + with zipfile.ZipFile(tmp_file, "r") as zip_ref: + zip_ref.extractall(destination_path) + + +class FrontendManager: + CUSTOM_FRONTENDS_ROOT = str(Path(__file__).parents[1] / "web_custom_versions") + + @classmethod + def default_frontend_path(cls) -> str: + try: + import comfyui_frontend_package + + return str(importlib.resources.files(comfyui_frontend_package) / "static") + except ImportError: + logging.error( + f""" +********** ERROR *********** + +comfyui-frontend-package is not installed. + +{frontend_install_warning_message()} + +********** ERROR *********** +""".strip() + ) + sys.exit(-1) + + @classmethod + def templates_path(cls) -> str: + try: + import comfyui_workflow_templates + + return str( + importlib.resources.files(comfyui_workflow_templates) / "templates" + ) + except ImportError: + logging.error( + f""" +********** ERROR *********** + +comfyui-workflow-templates is not installed. + +{frontend_install_warning_message()} + +********** ERROR *********** +""".strip() + ) + + @classmethod + def embedded_docs_path(cls) -> str: + """Get the path to embedded documentation""" + try: + import comfyui_embedded_docs + + return str( + importlib.resources.files(comfyui_embedded_docs) / "docs" + ) + except ImportError: + logging.info("comfyui-embedded-docs package not found") + return None + + @classmethod + def parse_version_string(cls, value: str) -> tuple[str, str, str]: + """ + Args: + value (str): The version string to parse. + + Returns: + tuple[str, str]: A tuple containing provider name and version. + + Raises: + argparse.ArgumentTypeError: If the version string is invalid. + """ + VERSION_PATTERN = r"^([a-zA-Z0-9][a-zA-Z0-9-]{0,38})/([a-zA-Z0-9_.-]+)@(v?\d+\.\d+\.\d+[-._a-zA-Z0-9]*|latest|prerelease)$" + match_result = re.match(VERSION_PATTERN, value) + if match_result is None: + raise argparse.ArgumentTypeError(f"Invalid version string: {value}") + + return match_result.group(1), match_result.group(2), match_result.group(3) + + @classmethod + def init_frontend_unsafe( + cls, version_string: str, provider: Optional[FrontEndProvider] = None + ) -> str: + """ + Initializes the frontend for the specified version. + + Args: + version_string (str): The version string. + provider (FrontEndProvider, optional): The provider to use. Defaults to None. + + Returns: + str: The path to the initialized frontend. + + Raises: + Exception: If there is an error during the initialization process. + main error source might be request timeout or invalid URL. + """ + if version_string == DEFAULT_VERSION_STRING: + check_frontend_version() + return cls.default_frontend_path() + + repo_owner, repo_name, version = cls.parse_version_string(version_string) + + if version.startswith("v"): + expected_path = str( + Path(cls.CUSTOM_FRONTENDS_ROOT) + / f"{repo_owner}_{repo_name}" + / version.lstrip("v") + ) + if os.path.exists(expected_path): + logging.info( + f"Using existing copy of specific frontend version tag: {repo_owner}/{repo_name}@{version}" + ) + return expected_path + + logging.info( + f"Initializing frontend: {repo_owner}/{repo_name}@{version}, requesting version details from GitHub..." + ) + + provider = provider or FrontEndProvider(repo_owner, repo_name) + release = provider.get_release(version) + + semantic_version = release["tag_name"].lstrip("v") + web_root = str( + Path(cls.CUSTOM_FRONTENDS_ROOT) / provider.folder_name / semantic_version + ) + if not os.path.exists(web_root): + try: + os.makedirs(web_root, exist_ok=True) + logging.info( + "Downloading frontend(%s) version(%s) to (%s)", + provider.folder_name, + semantic_version, + web_root, + ) + logging.debug(release) + download_release_asset_zip(release, destination_path=web_root) + finally: + # Clean up the directory if it is empty, i.e. the download failed + if not os.listdir(web_root): + os.rmdir(web_root) + + return web_root + + @classmethod + def init_frontend(cls, version_string: str) -> str: + """ + Initializes the frontend with the specified version string. + + Args: + version_string (str): The version string to initialize the frontend with. + + Returns: + str: The path of the initialized frontend. + """ + try: + return cls.init_frontend_unsafe(version_string) + except Exception as e: + logging.error("Failed to initialize frontend: %s", e) + logging.info("Falling back to the default frontend.") + check_frontend_version() + return cls.default_frontend_path() diff --git a/app/logger.py b/app/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..3d26d98fe28005a3ed8b6deb7da34823b98b5696 --- /dev/null +++ b/app/logger.py @@ -0,0 +1,98 @@ +from collections import deque +from datetime import datetime +import io +import logging +import sys +import threading + +logs = None +stdout_interceptor = None +stderr_interceptor = None + + +class LogInterceptor(io.TextIOWrapper): + def __init__(self, stream, *args, **kwargs): + buffer = stream.buffer + encoding = stream.encoding + super().__init__(buffer, *args, **kwargs, encoding=encoding, line_buffering=stream.line_buffering) + self._lock = threading.Lock() + self._flush_callbacks = [] + self._logs_since_flush = [] + + def write(self, data): + entry = {"t": datetime.now().isoformat(), "m": data} + with self._lock: + self._logs_since_flush.append(entry) + + # Simple handling for cr to overwrite the last output if it isnt a full line + # else logs just get full of progress messages + if isinstance(data, str) and data.startswith("\r") and not logs[-1]["m"].endswith("\n"): + logs.pop() + logs.append(entry) + super().write(data) + + def flush(self): + super().flush() + for cb in self._flush_callbacks: + cb(self._logs_since_flush) + self._logs_since_flush = [] + + def on_flush(self, callback): + self._flush_callbacks.append(callback) + + +def get_logs(): + return logs + + +def on_flush(callback): + if stdout_interceptor is not None: + stdout_interceptor.on_flush(callback) + if stderr_interceptor is not None: + stderr_interceptor.on_flush(callback) + +def setup_logger(log_level: str = 'INFO', capacity: int = 300, use_stdout: bool = False): + global logs + if logs: + return + + # Override output streams and log to buffer + logs = deque(maxlen=capacity) + + global stdout_interceptor + global stderr_interceptor + stdout_interceptor = sys.stdout = LogInterceptor(sys.stdout) + stderr_interceptor = sys.stderr = LogInterceptor(sys.stderr) + + # Setup default global logger + logger = logging.getLogger() + logger.setLevel(log_level) + + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(logging.Formatter("%(message)s")) + + if use_stdout: + # Only errors and critical to stderr + stream_handler.addFilter(lambda record: not record.levelno < logging.ERROR) + + # Lesser to stdout + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setFormatter(logging.Formatter("%(message)s")) + stdout_handler.addFilter(lambda record: record.levelno < logging.ERROR) + logger.addHandler(stdout_handler) + + logger.addHandler(stream_handler) + + +STARTUP_WARNINGS = [] + + +def log_startup_warning(msg): + logging.warning(msg) + STARTUP_WARNINGS.append(msg) + + +def print_startup_warnings(): + for s in STARTUP_WARNINGS: + logging.warning(s) + STARTUP_WARNINGS.clear() diff --git a/app/model_manager.py b/app/model_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..74d942fb85c560dd85903e1cc82ef64968b74316 --- /dev/null +++ b/app/model_manager.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import os +import base64 +import json +import time +import logging +import folder_paths +import glob +import comfy.utils +from aiohttp import web +from PIL import Image +from io import BytesIO +from folder_paths import map_legacy, filter_files_extensions, filter_files_content_types + + +class ModelFileManager: + def __init__(self) -> None: + self.cache: dict[str, tuple[list[dict], dict[str, float], float]] = {} + + def get_cache(self, key: str, default=None) -> tuple[list[dict], dict[str, float], float] | None: + return self.cache.get(key, default) + + def set_cache(self, key: str, value: tuple[list[dict], dict[str, float], float]): + self.cache[key] = value + + def clear_cache(self): + self.cache.clear() + + def add_routes(self, routes): + # NOTE: This is an experiment to replace `/models` + @routes.get("/experiment/models") + async def get_model_folders(request): + model_types = list(folder_paths.folder_names_and_paths.keys()) + folder_black_list = ["configs", "custom_nodes"] + output_folders: list[dict] = [] + for folder in model_types: + if folder in folder_black_list: + continue + output_folders.append({"name": folder, "folders": folder_paths.get_folder_paths(folder)}) + return web.json_response(output_folders) + + # NOTE: This is an experiment to replace `/models/{folder}` + @routes.get("/experiment/models/{folder}") + async def get_all_models(request): + folder = request.match_info.get("folder", None) + if not folder in folder_paths.folder_names_and_paths: + return web.Response(status=404) + files = self.get_model_file_list(folder) + return web.json_response(files) + + @routes.get("/experiment/models/preview/{folder}/{path_index}/{filename:.*}") + async def get_model_preview(request): + folder_name = request.match_info.get("folder", None) + path_index = int(request.match_info.get("path_index", None)) + filename = request.match_info.get("filename", None) + + if not folder_name in folder_paths.folder_names_and_paths: + return web.Response(status=404) + + folders = folder_paths.folder_names_and_paths[folder_name] + folder = folders[0][path_index] + full_filename = os.path.join(folder, filename) + + previews = self.get_model_previews(full_filename) + default_preview = previews[0] if len(previews) > 0 else None + if default_preview is None or (isinstance(default_preview, str) and not os.path.isfile(default_preview)): + return web.Response(status=404) + + try: + with Image.open(default_preview) as img: + img_bytes = BytesIO() + img.save(img_bytes, format="WEBP") + img_bytes.seek(0) + return web.Response(body=img_bytes.getvalue(), content_type="image/webp") + except: + return web.Response(status=404) + + def get_model_file_list(self, folder_name: str): + folder_name = map_legacy(folder_name) + folders = folder_paths.folder_names_and_paths[folder_name] + output_list: list[dict] = [] + + for index, folder in enumerate(folders[0]): + if not os.path.isdir(folder): + continue + out = self.cache_model_file_list_(folder) + if out is None: + out = self.recursive_search_models_(folder, index) + self.set_cache(folder, out) + output_list.extend(out[0]) + + return output_list + + def cache_model_file_list_(self, folder: str): + model_file_list_cache = self.get_cache(folder) + + if model_file_list_cache is None: + return None + if not os.path.isdir(folder): + return None + if os.path.getmtime(folder) != model_file_list_cache[1]: + return None + for x in model_file_list_cache[1]: + time_modified = model_file_list_cache[1][x] + folder = x + if os.path.getmtime(folder) != time_modified: + return None + + return model_file_list_cache + + def recursive_search_models_(self, directory: str, pathIndex: int) -> tuple[list[str], dict[str, float], float]: + if not os.path.isdir(directory): + return [], {}, time.perf_counter() + + excluded_dir_names = [".git"] + # TODO use settings + include_hidden_files = False + + result: list[str] = [] + dirs: dict[str, float] = {} + + for dirpath, subdirs, filenames in os.walk(directory, followlinks=True, topdown=True): + subdirs[:] = [d for d in subdirs if d not in excluded_dir_names] + if not include_hidden_files: + subdirs[:] = [d for d in subdirs if not d.startswith(".")] + filenames = [f for f in filenames if not f.startswith(".")] + + filenames = filter_files_extensions(filenames, folder_paths.supported_pt_extensions) + + for file_name in filenames: + try: + relative_path = os.path.relpath(os.path.join(dirpath, file_name), directory) + result.append(relative_path) + except: + logging.warning(f"Warning: Unable to access {file_name}. Skipping this file.") + continue + + for d in subdirs: + path: str = os.path.join(dirpath, d) + try: + dirs[path] = os.path.getmtime(path) + except FileNotFoundError: + logging.warning(f"Warning: Unable to access {path}. Skipping this path.") + continue + + return [{"name": f, "pathIndex": pathIndex} for f in result], dirs, time.perf_counter() + + def get_model_previews(self, filepath: str) -> list[str | BytesIO]: + dirname = os.path.dirname(filepath) + + if not os.path.exists(dirname): + return [] + + basename = os.path.splitext(filepath)[0] + match_files = glob.glob(f"{basename}.*", recursive=False) + image_files = filter_files_content_types(match_files, "image") + safetensors_file = next(filter(lambda x: x.endswith(".safetensors"), match_files), None) + safetensors_metadata = {} + + result: list[str | BytesIO] = [] + + for filename in image_files: + _basename = os.path.splitext(filename)[0] + if _basename == basename: + result.append(filename) + if _basename == f"{basename}.preview": + result.append(filename) + + if safetensors_file: + safetensors_filepath = os.path.join(dirname, safetensors_file) + header = comfy.utils.safetensors_header(safetensors_filepath, max_size=8*1024*1024) + if header: + safetensors_metadata = json.loads(header) + safetensors_images = safetensors_metadata.get("__metadata__", {}).get("ssmd_cover_images", None) + if safetensors_images: + safetensors_images = json.loads(safetensors_images) + for image in safetensors_images: + result.append(BytesIO(base64.b64decode(image))) + + return result + + def __exit__(self, exc_type, exc_value, traceback): + self.clear_cache() diff --git a/app/user_manager.py b/app/user_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..d31da5b9b89dda908e900041ef41b082dbac38eb --- /dev/null +++ b/app/user_manager.py @@ -0,0 +1,436 @@ +from __future__ import annotations +import json +import os +import re +import uuid +import glob +import shutil +import logging +from aiohttp import web +from urllib import parse +from comfy.cli_args import args +import folder_paths +from .app_settings import AppSettings +from typing import TypedDict + +default_user = "default" + + +class FileInfo(TypedDict): + path: str + size: int + modified: int + + +def get_file_info(path: str, relative_to: str) -> FileInfo: + return { + "path": os.path.relpath(path, relative_to).replace(os.sep, '/'), + "size": os.path.getsize(path), + "modified": os.path.getmtime(path) + } + + +class UserManager(): + def __init__(self): + user_directory = folder_paths.get_user_directory() + + self.settings = AppSettings(self) + if not os.path.exists(user_directory): + os.makedirs(user_directory, exist_ok=True) + if not args.multi_user: + logging.warning("****** User settings have been changed to be stored on the server instead of browser storage. ******") + logging.warning("****** For multi-user setups add the --multi-user CLI argument to enable multiple user profiles. ******") + + if args.multi_user: + if os.path.isfile(self.get_users_file()): + with open(self.get_users_file()) as f: + self.users = json.load(f) + else: + self.users = {} + else: + self.users = {"default": "default"} + + def get_users_file(self): + return os.path.join(folder_paths.get_user_directory(), "users.json") + + def get_request_user_id(self, request): + user = "default" + if args.multi_user and "comfy-user" in request.headers: + user = request.headers["comfy-user"] + + if user not in self.users: + raise KeyError("Unknown user: " + user) + + return user + + def get_request_user_filepath(self, request, file, type="userdata", create_dir=True): + user_directory = folder_paths.get_user_directory() + + if type == "userdata": + root_dir = user_directory + else: + raise KeyError("Unknown filepath type:" + type) + + user = self.get_request_user_id(request) + path = user_root = os.path.abspath(os.path.join(root_dir, user)) + + # prevent leaving /{type} + if os.path.commonpath((root_dir, user_root)) != root_dir: + return None + + if file is not None: + # Check if filename is url encoded + if "%" in file: + file = parse.unquote(file) + + # prevent leaving /{type}/{user} + path = os.path.abspath(os.path.join(user_root, file)) + if os.path.commonpath((user_root, path)) != user_root: + return None + + parent = os.path.split(path)[0] + + if create_dir and not os.path.exists(parent): + os.makedirs(parent, exist_ok=True) + + return path + + def add_user(self, name): + name = name.strip() + if not name: + raise ValueError("username not provided") + user_id = re.sub("[^a-zA-Z0-9-_]+", '-', name) + user_id = user_id + "_" + str(uuid.uuid4()) + + self.users[user_id] = name + + with open(self.get_users_file(), "w") as f: + json.dump(self.users, f) + + return user_id + + def add_routes(self, routes): + self.settings.add_routes(routes) + + @routes.get("/users") + async def get_users(request): + if args.multi_user: + return web.json_response({"storage": "server", "users": self.users}) + else: + user_dir = self.get_request_user_filepath(request, None, create_dir=False) + return web.json_response({ + "storage": "server", + "migrated": os.path.exists(user_dir) + }) + + @routes.post("/users") + async def post_users(request): + body = await request.json() + username = body["username"] + if username in self.users.values(): + return web.json_response({"error": "Duplicate username."}, status=400) + + user_id = self.add_user(username) + return web.json_response(user_id) + + @routes.get("/userdata") + async def listuserdata(request): + """ + List user data files in a specified directory. + + This endpoint allows listing files in a user's data directory, with options for recursion, + full file information, and path splitting. + + Query Parameters: + - dir (required): The directory to list files from. + - recurse (optional): If "true", recursively list files in subdirectories. + - full_info (optional): If "true", return detailed file information (path, size, modified time). + - split (optional): If "true", split file paths into components (only applies when full_info is false). + + Returns: + - 400: If 'dir' parameter is missing. + - 403: If the requested path is not allowed. + - 404: If the requested directory does not exist. + - 200: JSON response with the list of files or file information. + + The response format depends on the query parameters: + - Default: List of relative file paths. + - full_info=true: List of dictionaries with file details. + - split=true (and full_info=false): List of lists, each containing path components. + """ + directory = request.rel_url.query.get('dir', '') + if not directory: + return web.Response(status=400, text="Directory not provided") + + path = self.get_request_user_filepath(request, directory) + if not path: + return web.Response(status=403, text="Invalid directory") + + if not os.path.exists(path): + return web.Response(status=404, text="Directory not found") + + recurse = request.rel_url.query.get('recurse', '').lower() == "true" + full_info = request.rel_url.query.get('full_info', '').lower() == "true" + split_path = request.rel_url.query.get('split', '').lower() == "true" + + # Use different patterns based on whether we're recursing or not + if recurse: + pattern = os.path.join(glob.escape(path), '**', '*') + else: + pattern = os.path.join(glob.escape(path), '*') + + def process_full_path(full_path: str) -> FileInfo | str | list[str]: + if full_info: + return get_file_info(full_path, path) + + rel_path = os.path.relpath(full_path, path).replace(os.sep, '/') + if split_path: + return [rel_path] + rel_path.split('/') + + return rel_path + + results = [ + process_full_path(full_path) + for full_path in glob.glob(pattern, recursive=recurse) + if os.path.isfile(full_path) + ] + + return web.json_response(results) + + @routes.get("/v2/userdata") + async def list_userdata_v2(request): + """ + List files and directories in a user's data directory. + + This endpoint provides a structured listing of contents within a specified + subdirectory of the user's data storage. + + Query Parameters: + - path (optional): The relative path within the user's data directory + to list. Defaults to the root (''). + + Returns: + - 400: If the requested path is invalid, outside the user's data directory, or is not a directory. + - 404: If the requested path does not exist. + - 403: If the user is invalid. + - 500: If there is an error reading the directory contents. + - 200: JSON response containing a list of file and directory objects. + Each object includes: + - name: The name of the file or directory. + - type: 'file' or 'directory'. + - path: The relative path from the user's data root. + - size (for files): The size in bytes. + - modified (for files): The last modified timestamp (Unix epoch). + """ + requested_rel_path = request.rel_url.query.get('path', '') + + # URL-decode the path parameter + try: + requested_rel_path = parse.unquote(requested_rel_path) + except Exception as e: + logging.warning(f"Failed to decode path parameter: {requested_rel_path}, Error: {e}") + return web.Response(status=400, text="Invalid characters in path parameter") + + + # Check user validity and get the absolute path for the requested directory + try: + base_user_path = self.get_request_user_filepath(request, None, create_dir=False) + + if requested_rel_path: + target_abs_path = self.get_request_user_filepath(request, requested_rel_path, create_dir=False) + else: + target_abs_path = base_user_path + + except KeyError as e: + # Invalid user detected by get_request_user_id inside get_request_user_filepath + logging.warning(f"Access denied for user: {e}") + return web.Response(status=403, text="Invalid user specified in request") + + + if not target_abs_path: + # Path traversal or other issue detected by get_request_user_filepath + return web.Response(status=400, text="Invalid path requested") + + # Handle cases where the user directory or target path doesn't exist + if not os.path.exists(target_abs_path): + # Check if it's the base user directory that's missing (new user case) + if target_abs_path == base_user_path: + # It's okay if the base user directory doesn't exist yet, return empty list + return web.json_response([]) + else: + # A specific subdirectory was requested but doesn't exist + return web.Response(status=404, text="Requested path not found") + + if not os.path.isdir(target_abs_path): + return web.Response(status=400, text="Requested path is not a directory") + + results = [] + try: + for root, dirs, files in os.walk(target_abs_path, topdown=True): + # Process directories + for dir_name in dirs: + dir_path = os.path.join(root, dir_name) + rel_path = os.path.relpath(dir_path, base_user_path).replace(os.sep, '/') + results.append({ + "name": dir_name, + "path": rel_path, + "type": "directory" + }) + + # Process files + for file_name in files: + file_path = os.path.join(root, file_name) + rel_path = os.path.relpath(file_path, base_user_path).replace(os.sep, '/') + entry_info = { + "name": file_name, + "path": rel_path, + "type": "file" + } + try: + stats = os.stat(file_path) # Use os.stat for potentially better performance with os.walk + entry_info["size"] = stats.st_size + entry_info["modified"] = stats.st_mtime + except OSError as stat_error: + logging.warning(f"Could not stat file {file_path}: {stat_error}") + pass # Include file with available info + results.append(entry_info) + except OSError as e: + logging.error(f"Error listing directory {target_abs_path}: {e}") + return web.Response(status=500, text="Error reading directory contents") + + # Sort results alphabetically, directories first then files + results.sort(key=lambda x: (x['type'] != 'directory', x['name'].lower())) + + return web.json_response(results) + + def get_user_data_path(request, check_exists = False, param = "file"): + file = request.match_info.get(param, None) + if not file: + return web.Response(status=400) + + path = self.get_request_user_filepath(request, file) + if not path: + return web.Response(status=403) + + if check_exists and not os.path.exists(path): + return web.Response(status=404) + + return path + + @routes.get("/userdata/{file}") + async def getuserdata(request): + path = get_user_data_path(request, check_exists=True) + if not isinstance(path, str): + return path + + return web.FileResponse(path) + + @routes.post("/userdata/{file}") + async def post_userdata(request): + """ + Upload or update a user data file. + + This endpoint handles file uploads to a user's data directory, with options for + controlling overwrite behavior and response format. + + Query Parameters: + - overwrite (optional): If "false", prevents overwriting existing files. Defaults to "true". + - full_info (optional): If "true", returns detailed file information (path, size, modified time). + If "false", returns only the relative file path. + + Path Parameters: + - file: The target file path (URL encoded if necessary). + + Returns: + - 400: If 'file' parameter is missing. + - 403: If the requested path is not allowed. + - 409: If overwrite=false and the file already exists. + - 200: JSON response with either: + - Full file information (if full_info=true) + - Relative file path (if full_info=false) + + The request body should contain the raw file content to be written. + """ + path = get_user_data_path(request) + if not isinstance(path, str): + return path + + overwrite = request.query.get("overwrite", 'true') != "false" + full_info = request.query.get('full_info', 'false').lower() == "true" + + if not overwrite and os.path.exists(path): + return web.Response(status=409, text="File already exists") + + body = await request.read() + + with open(path, "wb") as f: + f.write(body) + + user_path = self.get_request_user_filepath(request, None) + if full_info: + resp = get_file_info(path, user_path) + else: + resp = os.path.relpath(path, user_path) + + return web.json_response(resp) + + @routes.delete("/userdata/{file}") + async def delete_userdata(request): + path = get_user_data_path(request, check_exists=True) + if not isinstance(path, str): + return path + + os.remove(path) + + return web.Response(status=204) + + @routes.post("/userdata/{file}/move/{dest}") + async def move_userdata(request): + """ + Move or rename a user data file. + + This endpoint handles moving or renaming files within a user's data directory, with options for + controlling overwrite behavior and response format. + + Path Parameters: + - file: The source file path (URL encoded if necessary) + - dest: The destination file path (URL encoded if necessary) + + Query Parameters: + - overwrite (optional): If "false", prevents overwriting existing files. Defaults to "true". + - full_info (optional): If "true", returns detailed file information (path, size, modified time). + If "false", returns only the relative file path. + + Returns: + - 400: If either 'file' or 'dest' parameter is missing + - 403: If either requested path is not allowed + - 404: If the source file does not exist + - 409: If overwrite=false and the destination file already exists + - 200: JSON response with either: + - Full file information (if full_info=true) + - Relative file path (if full_info=false) + """ + source = get_user_data_path(request, check_exists=True) + if not isinstance(source, str): + return source + + dest = get_user_data_path(request, check_exists=False, param="dest") + if not isinstance(source, str): + return dest + + overwrite = request.query.get("overwrite", 'true') != "false" + full_info = request.query.get('full_info', 'false').lower() == "true" + + if not overwrite and os.path.exists(dest): + return web.Response(status=409, text="File already exists") + + logging.info(f"moving '{source}' -> '{dest}'") + shutil.move(source, dest) + + user_path = self.get_request_user_filepath(request, None) + if full_info: + resp = get_file_info(dest, user_path) + else: + resp = os.path.relpath(dest, user_path) + + return web.json_response(resp) diff --git a/comfy/checkpoint_pickle.py b/comfy/checkpoint_pickle.py new file mode 100644 index 0000000000000000000000000000000000000000..206551d3c1cf0d654c907534629a800196ba138b --- /dev/null +++ b/comfy/checkpoint_pickle.py @@ -0,0 +1,13 @@ +import pickle + +load = pickle.load + +class Empty: + pass + +class Unpickler(pickle.Unpickler): + def find_class(self, module, name): + #TODO: safe unpickle + if module.startswith("pytorch_lightning"): + return Empty + return super().find_class(module, name) diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.py new file mode 100644 index 0000000000000000000000000000000000000000..ec01665e2181d4aea1c9cbb5b95bdc7715a3e8a8 --- /dev/null +++ b/comfy/cldm/cldm.py @@ -0,0 +1,433 @@ +#taken from: https://github.com/lllyasviel/ControlNet +#and modified + +import torch +import torch.nn as nn + +from ..ldm.modules.diffusionmodules.util import ( + timestep_embedding, +) + +from ..ldm.modules.attention import SpatialTransformer +from ..ldm.modules.diffusionmodules.openaimodel import UNetModel, TimestepEmbedSequential, ResBlock, Downsample +from ..ldm.util import exists +from .control_types import UNION_CONTROLNET_TYPES +from collections import OrderedDict +import comfy.ops +from comfy.ldm.modules.attention import optimized_attention + +class OptimizedAttention(nn.Module): + def __init__(self, c, nhead, dropout=0.0, dtype=None, device=None, operations=None): + super().__init__() + self.heads = nhead + self.c = c + + self.in_proj = operations.Linear(c, c * 3, bias=True, dtype=dtype, device=device) + self.out_proj = operations.Linear(c, c, bias=True, dtype=dtype, device=device) + + def forward(self, x): + x = self.in_proj(x) + q, k, v = x.split(self.c, dim=2) + out = optimized_attention(q, k, v, self.heads) + return self.out_proj(out) + +class QuickGELU(nn.Module): + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + +class ResBlockUnionControlnet(nn.Module): + def __init__(self, dim, nhead, dtype=None, device=None, operations=None): + super().__init__() + self.attn = OptimizedAttention(dim, nhead, dtype=dtype, device=device, operations=operations) + self.ln_1 = operations.LayerNorm(dim, dtype=dtype, device=device) + self.mlp = nn.Sequential( + OrderedDict([("c_fc", operations.Linear(dim, dim * 4, dtype=dtype, device=device)), ("gelu", QuickGELU()), + ("c_proj", operations.Linear(dim * 4, dim, dtype=dtype, device=device))])) + self.ln_2 = operations.LayerNorm(dim, dtype=dtype, device=device) + + def attention(self, x: torch.Tensor): + return self.attn(x) + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + +class ControlledUnetModel(UNetModel): + #implemented in the ldm unet + pass + +class ControlNet(nn.Module): + def __init__( + self, + image_size, + in_channels, + model_channels, + hint_channels, + num_res_blocks, + dropout=0, + channel_mult=(1, 2, 4, 8), + conv_resample=True, + dims=2, + num_classes=None, + use_checkpoint=False, + dtype=torch.float32, + num_heads=-1, + num_head_channels=-1, + num_heads_upsample=-1, + use_scale_shift_norm=False, + resblock_updown=False, + use_new_attention_order=False, + use_spatial_transformer=False, # custom transformer support + transformer_depth=1, # custom transformer support + context_dim=None, # custom transformer support + n_embed=None, # custom support for prediction of discrete ids into codebook of first stage vq model + legacy=True, + disable_self_attentions=None, + num_attention_blocks=None, + disable_middle_self_attn=False, + use_linear_in_transformer=False, + adm_in_channels=None, + transformer_depth_middle=None, + transformer_depth_output=None, + attn_precision=None, + union_controlnet_num_control_type=None, + device=None, + operations=comfy.ops.disable_weight_init, + **kwargs, + ): + super().__init__() + assert use_spatial_transformer == True, "use_spatial_transformer has to be true" + if use_spatial_transformer: + assert context_dim is not None, 'Fool!! You forgot to include the dimension of your cross-attention conditioning...' + + if context_dim is not None: + assert use_spatial_transformer, 'Fool!! You forgot to use the spatial transformer for your cross-attention conditioning...' + # from omegaconf.listconfig import ListConfig + # if type(context_dim) == ListConfig: + # context_dim = list(context_dim) + + if num_heads_upsample == -1: + num_heads_upsample = num_heads + + if num_heads == -1: + assert num_head_channels != -1, 'Either num_heads or num_head_channels has to be set' + + if num_head_channels == -1: + assert num_heads != -1, 'Either num_heads or num_head_channels has to be set' + + self.dims = dims + self.image_size = image_size + self.in_channels = in_channels + self.model_channels = model_channels + + if isinstance(num_res_blocks, int): + self.num_res_blocks = len(channel_mult) * [num_res_blocks] + else: + if len(num_res_blocks) != len(channel_mult): + raise ValueError("provide num_res_blocks either as an int (globally constant) or " + "as a list/tuple (per-level) with the same length as channel_mult") + self.num_res_blocks = num_res_blocks + + if disable_self_attentions is not None: + # should be a list of booleans, indicating whether to disable self-attention in TransformerBlocks or not + assert len(disable_self_attentions) == len(channel_mult) + if num_attention_blocks is not None: + assert len(num_attention_blocks) == len(self.num_res_blocks) + assert all(map(lambda i: self.num_res_blocks[i] >= num_attention_blocks[i], range(len(num_attention_blocks)))) + + transformer_depth = transformer_depth[:] + + self.dropout = dropout + self.channel_mult = channel_mult + self.conv_resample = conv_resample + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + self.dtype = dtype + self.num_heads = num_heads + self.num_head_channels = num_head_channels + self.num_heads_upsample = num_heads_upsample + self.predict_codebook_ids = n_embed is not None + + time_embed_dim = model_channels * 4 + self.time_embed = nn.Sequential( + operations.Linear(model_channels, time_embed_dim, dtype=self.dtype, device=device), + nn.SiLU(), + operations.Linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device), + ) + + if self.num_classes is not None: + if isinstance(self.num_classes, int): + self.label_emb = nn.Embedding(num_classes, time_embed_dim) + elif self.num_classes == "continuous": + self.label_emb = nn.Linear(1, time_embed_dim) + elif self.num_classes == "sequential": + assert adm_in_channels is not None + self.label_emb = nn.Sequential( + nn.Sequential( + operations.Linear(adm_in_channels, time_embed_dim, dtype=self.dtype, device=device), + nn.SiLU(), + operations.Linear(time_embed_dim, time_embed_dim, dtype=self.dtype, device=device), + ) + ) + else: + raise ValueError() + + self.input_blocks = nn.ModuleList( + [ + TimestepEmbedSequential( + operations.conv_nd(dims, in_channels, model_channels, 3, padding=1, dtype=self.dtype, device=device) + ) + ] + ) + self.zero_convs = nn.ModuleList([self.make_zero_conv(model_channels, operations=operations, dtype=self.dtype, device=device)]) + + self.input_hint_block = TimestepEmbedSequential( + operations.conv_nd(dims, hint_channels, 16, 3, padding=1, dtype=self.dtype, device=device), + nn.SiLU(), + operations.conv_nd(dims, 16, 16, 3, padding=1, dtype=self.dtype, device=device), + nn.SiLU(), + operations.conv_nd(dims, 16, 32, 3, padding=1, stride=2, dtype=self.dtype, device=device), + nn.SiLU(), + operations.conv_nd(dims, 32, 32, 3, padding=1, dtype=self.dtype, device=device), + nn.SiLU(), + operations.conv_nd(dims, 32, 96, 3, padding=1, stride=2, dtype=self.dtype, device=device), + nn.SiLU(), + operations.conv_nd(dims, 96, 96, 3, padding=1, dtype=self.dtype, device=device), + nn.SiLU(), + operations.conv_nd(dims, 96, 256, 3, padding=1, stride=2, dtype=self.dtype, device=device), + nn.SiLU(), + operations.conv_nd(dims, 256, model_channels, 3, padding=1, dtype=self.dtype, device=device) + ) + + self._feature_size = model_channels + input_block_chans = [model_channels] + ch = model_channels + ds = 1 + for level, mult in enumerate(channel_mult): + for nr in range(self.num_res_blocks[level]): + layers = [ + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=mult * model_channels, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype, + device=device, + operations=operations, + ) + ] + ch = mult * model_channels + num_transformers = transformer_depth.pop(0) + if num_transformers > 0: + if num_head_channels == -1: + dim_head = ch // num_heads + else: + num_heads = ch // num_head_channels + dim_head = num_head_channels + if legacy: + #num_heads = 1 + dim_head = ch // num_heads if use_spatial_transformer else num_head_channels + if exists(disable_self_attentions): + disabled_sa = disable_self_attentions[level] + else: + disabled_sa = False + + if not exists(num_attention_blocks) or nr < num_attention_blocks[level]: + layers.append( + SpatialTransformer( + ch, num_heads, dim_head, depth=num_transformers, context_dim=context_dim, + disable_self_attn=disabled_sa, use_linear=use_linear_in_transformer, + use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations + ) + ) + self.input_blocks.append(TimestepEmbedSequential(*layers)) + self.zero_convs.append(self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device)) + self._feature_size += ch + input_block_chans.append(ch) + if level != len(channel_mult) - 1: + out_ch = ch + self.input_blocks.append( + TimestepEmbedSequential( + ResBlock( + ch, + time_embed_dim, + dropout, + out_channels=out_ch, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + down=True, + dtype=self.dtype, + device=device, + operations=operations + ) + if resblock_updown + else Downsample( + ch, conv_resample, dims=dims, out_channels=out_ch, dtype=self.dtype, device=device, operations=operations + ) + ) + ) + ch = out_ch + input_block_chans.append(ch) + self.zero_convs.append(self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device)) + ds *= 2 + self._feature_size += ch + + if num_head_channels == -1: + dim_head = ch // num_heads + else: + num_heads = ch // num_head_channels + dim_head = num_head_channels + if legacy: + #num_heads = 1 + dim_head = ch // num_heads if use_spatial_transformer else num_head_channels + mid_block = [ + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype, + device=device, + operations=operations + )] + if transformer_depth_middle >= 0: + mid_block += [SpatialTransformer( # always uses a self-attn + ch, num_heads, dim_head, depth=transformer_depth_middle, context_dim=context_dim, + disable_self_attn=disable_middle_self_attn, use_linear=use_linear_in_transformer, + use_checkpoint=use_checkpoint, attn_precision=attn_precision, dtype=self.dtype, device=device, operations=operations + ), + ResBlock( + ch, + time_embed_dim, + dropout, + dims=dims, + use_checkpoint=use_checkpoint, + use_scale_shift_norm=use_scale_shift_norm, + dtype=self.dtype, + device=device, + operations=operations + )] + self.middle_block = TimestepEmbedSequential(*mid_block) + self.middle_block_out = self.make_zero_conv(ch, operations=operations, dtype=self.dtype, device=device) + self._feature_size += ch + + if union_controlnet_num_control_type is not None: + self.num_control_type = union_controlnet_num_control_type + num_trans_channel = 320 + num_trans_head = 8 + num_trans_layer = 1 + num_proj_channel = 320 + # task_scale_factor = num_trans_channel ** 0.5 + self.task_embedding = nn.Parameter(torch.empty(self.num_control_type, num_trans_channel, dtype=self.dtype, device=device)) + + self.transformer_layes = nn.Sequential(*[ResBlockUnionControlnet(num_trans_channel, num_trans_head, dtype=self.dtype, device=device, operations=operations) for _ in range(num_trans_layer)]) + self.spatial_ch_projs = operations.Linear(num_trans_channel, num_proj_channel, dtype=self.dtype, device=device) + #----------------------------------------------------------------------------------------------------- + + control_add_embed_dim = 256 + class ControlAddEmbedding(nn.Module): + def __init__(self, in_dim, out_dim, num_control_type, dtype=None, device=None, operations=None): + super().__init__() + self.num_control_type = num_control_type + self.in_dim = in_dim + self.linear_1 = operations.Linear(in_dim * num_control_type, out_dim, dtype=dtype, device=device) + self.linear_2 = operations.Linear(out_dim, out_dim, dtype=dtype, device=device) + def forward(self, control_type, dtype, device): + c_type = torch.zeros((self.num_control_type,), device=device) + c_type[control_type] = 1.0 + c_type = timestep_embedding(c_type.flatten(), self.in_dim, repeat_only=False).to(dtype).reshape((-1, self.num_control_type * self.in_dim)) + return self.linear_2(torch.nn.functional.silu(self.linear_1(c_type))) + + self.control_add_embedding = ControlAddEmbedding(control_add_embed_dim, time_embed_dim, self.num_control_type, dtype=self.dtype, device=device, operations=operations) + else: + self.task_embedding = None + self.control_add_embedding = None + + def union_controlnet_merge(self, hint, control_type, emb, context): + # Equivalent to: https://github.com/xinsir6/ControlNetPlus/tree/main + inputs = [] + condition_list = [] + + for idx in range(min(1, len(control_type))): + controlnet_cond = self.input_hint_block(hint[idx], emb, context) + feat_seq = torch.mean(controlnet_cond, dim=(2, 3)) + if idx < len(control_type): + feat_seq += self.task_embedding[control_type[idx]].to(dtype=feat_seq.dtype, device=feat_seq.device) + + inputs.append(feat_seq.unsqueeze(1)) + condition_list.append(controlnet_cond) + + x = torch.cat(inputs, dim=1) + x = self.transformer_layes(x) + controlnet_cond_fuser = None + for idx in range(len(control_type)): + alpha = self.spatial_ch_projs(x[:, idx]) + alpha = alpha.unsqueeze(-1).unsqueeze(-1) + o = condition_list[idx] + alpha + if controlnet_cond_fuser is None: + controlnet_cond_fuser = o + else: + controlnet_cond_fuser += o + return controlnet_cond_fuser + + def make_zero_conv(self, channels, operations=None, dtype=None, device=None): + return TimestepEmbedSequential(operations.conv_nd(self.dims, channels, channels, 1, padding=0, dtype=dtype, device=device)) + + def forward(self, x, hint, timesteps, context, y=None, **kwargs): + t_emb = timestep_embedding(timesteps, self.model_channels, repeat_only=False).to(x.dtype) + emb = self.time_embed(t_emb) + + guided_hint = None + if self.control_add_embedding is not None: #Union Controlnet + control_type = kwargs.get("control_type", []) + + if any([c >= self.num_control_type for c in control_type]): + max_type = max(control_type) + max_type_name = { + v: k for k, v in UNION_CONTROLNET_TYPES.items() + }[max_type] + raise ValueError( + f"Control type {max_type_name}({max_type}) is out of range for the number of control types" + + f"({self.num_control_type}) supported.\n" + + "Please consider using the ProMax ControlNet Union model.\n" + + "https://huggingface.co/xinsir/controlnet-union-sdxl-1.0/tree/main" + ) + + emb += self.control_add_embedding(control_type, emb.dtype, emb.device) + if len(control_type) > 0: + if len(hint.shape) < 5: + hint = hint.unsqueeze(dim=0) + guided_hint = self.union_controlnet_merge(hint, control_type, emb, context) + + if guided_hint is None: + guided_hint = self.input_hint_block(hint, emb, context) + + out_output = [] + out_middle = [] + + if self.num_classes is not None: + assert y.shape[0] == x.shape[0] + emb = emb + self.label_emb(y) + + h = x + for module, zero_conv in zip(self.input_blocks, self.zero_convs): + if guided_hint is not None: + h = module(h, emb, context) + h += guided_hint + guided_hint = None + else: + h = module(h, emb, context) + out_output.append(zero_conv(h, emb, context)) + + h = self.middle_block(h, emb, context) + out_middle.append(self.middle_block_out(h, emb, context)) + + return {"middle": out_middle, "output": out_output} + diff --git a/comfy/cldm/control_types.py b/comfy/cldm/control_types.py new file mode 100644 index 0000000000000000000000000000000000000000..4128631a305a13d65c3c37ced17179d23fbbdcff --- /dev/null +++ b/comfy/cldm/control_types.py @@ -0,0 +1,10 @@ +UNION_CONTROLNET_TYPES = { + "openpose": 0, + "depth": 1, + "hed/pidi/scribble/ted": 2, + "canny/lineart/anime_lineart/mlsd": 3, + "normal": 4, + "segment": 5, + "tile": 6, + "repaint": 7, +} diff --git a/comfy/cldm/dit_embedder.py b/comfy/cldm/dit_embedder.py new file mode 100644 index 0000000000000000000000000000000000000000..f9bf31012b1319e929cab381c9a3b32be62fc589 --- /dev/null +++ b/comfy/cldm/dit_embedder.py @@ -0,0 +1,120 @@ +import math +from typing import List, Optional, Tuple + +import torch +import torch.nn as nn +from torch import Tensor + +from comfy.ldm.modules.diffusionmodules.mmdit import DismantledBlock, PatchEmbed, VectorEmbedder, TimestepEmbedder, get_2d_sincos_pos_embed_torch + + +class ControlNetEmbedder(nn.Module): + + def __init__( + self, + img_size: int, + patch_size: int, + in_chans: int, + attention_head_dim: int, + num_attention_heads: int, + adm_in_channels: int, + num_layers: int, + main_model_double: int, + double_y_emb: bool, + device: torch.device, + dtype: torch.dtype, + pos_embed_max_size: Optional[int] = None, + operations = None, + ): + super().__init__() + self.main_model_double = main_model_double + self.dtype = dtype + self.hidden_size = num_attention_heads * attention_head_dim + self.patch_size = patch_size + self.x_embedder = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=self.hidden_size, + strict_img_size=pos_embed_max_size is None, + device=device, + dtype=dtype, + operations=operations, + ) + + self.t_embedder = TimestepEmbedder(self.hidden_size, dtype=dtype, device=device, operations=operations) + + self.double_y_emb = double_y_emb + if self.double_y_emb: + self.orig_y_embedder = VectorEmbedder( + adm_in_channels, self.hidden_size, dtype, device, operations=operations + ) + self.y_embedder = VectorEmbedder( + self.hidden_size, self.hidden_size, dtype, device, operations=operations + ) + else: + self.y_embedder = VectorEmbedder( + adm_in_channels, self.hidden_size, dtype, device, operations=operations + ) + + self.transformer_blocks = nn.ModuleList( + DismantledBlock( + hidden_size=self.hidden_size, num_heads=num_attention_heads, qkv_bias=True, + dtype=dtype, device=device, operations=operations + ) + for _ in range(num_layers) + ) + + # self.use_y_embedder = pooled_projection_dim != self.time_text_embed.text_embedder.linear_1.in_features + # TODO double check this logic when 8b + self.use_y_embedder = True + + self.controlnet_blocks = nn.ModuleList([]) + for _ in range(len(self.transformer_blocks)): + controlnet_block = operations.Linear(self.hidden_size, self.hidden_size, dtype=dtype, device=device) + self.controlnet_blocks.append(controlnet_block) + + self.pos_embed_input = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_chans=in_chans, + embed_dim=self.hidden_size, + strict_img_size=False, + device=device, + dtype=dtype, + operations=operations, + ) + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + y: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + hint = None, + ) -> Tuple[Tensor, List[Tensor]]: + x_shape = list(x.shape) + x = self.x_embedder(x) + if not self.double_y_emb: + h = (x_shape[-2] + 1) // self.patch_size + w = (x_shape[-1] + 1) // self.patch_size + x += get_2d_sincos_pos_embed_torch(self.hidden_size, w, h, device=x.device) + c = self.t_embedder(timesteps, dtype=x.dtype) + if y is not None and self.y_embedder is not None: + if self.double_y_emb: + y = self.orig_y_embedder(y) + y = self.y_embedder(y) + c = c + y + + x = x + self.pos_embed_input(hint) + + block_out = () + + repeat = math.ceil(self.main_model_double / len(self.transformer_blocks)) + for i in range(len(self.transformer_blocks)): + out = self.transformer_blocks[i](x, c) + if not self.double_y_emb: + x = out + block_out += (self.controlnet_blocks[i](out),) * repeat + + return {"output": block_out} diff --git a/comfy/cldm/mmdit.py b/comfy/cldm/mmdit.py new file mode 100644 index 0000000000000000000000000000000000000000..b7764085e9431259d43701654af897effe0799da --- /dev/null +++ b/comfy/cldm/mmdit.py @@ -0,0 +1,81 @@ +import torch +from typing import Optional +import comfy.ldm.modules.diffusionmodules.mmdit + +class ControlNet(comfy.ldm.modules.diffusionmodules.mmdit.MMDiT): + def __init__( + self, + num_blocks = None, + control_latent_channels = None, + dtype = None, + device = None, + operations = None, + **kwargs, + ): + super().__init__(dtype=dtype, device=device, operations=operations, final_layer=False, num_blocks=num_blocks, **kwargs) + # controlnet_blocks + self.controlnet_blocks = torch.nn.ModuleList([]) + for _ in range(len(self.joint_blocks)): + self.controlnet_blocks.append(operations.Linear(self.hidden_size, self.hidden_size, device=device, dtype=dtype)) + + if control_latent_channels is None: + control_latent_channels = self.in_channels + + self.pos_embed_input = comfy.ldm.modules.diffusionmodules.mmdit.PatchEmbed( + None, + self.patch_size, + control_latent_channels, + self.hidden_size, + bias=True, + strict_img_size=False, + dtype=dtype, + device=device, + operations=operations + ) + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + y: Optional[torch.Tensor] = None, + context: Optional[torch.Tensor] = None, + hint = None, + ) -> torch.Tensor: + + #weird sd3 controlnet specific stuff + y = torch.zeros_like(y) + + if self.context_processor is not None: + context = self.context_processor(context) + + hw = x.shape[-2:] + x = self.x_embedder(x) + self.cropped_pos_embed(hw, device=x.device).to(dtype=x.dtype, device=x.device) + x += self.pos_embed_input(hint) + + c = self.t_embedder(timesteps, dtype=x.dtype) + if y is not None and self.y_embedder is not None: + y = self.y_embedder(y) + c = c + y + + if context is not None: + context = self.context_embedder(context) + + output = [] + + blocks = len(self.joint_blocks) + for i in range(blocks): + context, x = self.joint_blocks[i]( + context, + x, + c=c, + use_checkpoint=self.use_checkpoint, + ) + + out = self.controlnet_blocks[i](x) + count = self.depth // blocks + if i == blocks - 1: + count -= 1 + for j in range(count): + output.append(out) + + return {"output": output} diff --git a/comfy/cli_args.py b/comfy/cli_args.py new file mode 100644 index 0000000000000000000000000000000000000000..7234a7ba097530cb49d3c4180d5f6a71e4b4d55f --- /dev/null +++ b/comfy/cli_args.py @@ -0,0 +1,235 @@ +import argparse +import enum +import os +import comfy.options + + +class EnumAction(argparse.Action): + """ + Argparse action for handling Enums + """ + def __init__(self, **kwargs): + # Pop off the type value + enum_type = kwargs.pop("type", None) + + # Ensure an Enum subclass is provided + if enum_type is None: + raise ValueError("type must be assigned an Enum when using EnumAction") + if not issubclass(enum_type, enum.Enum): + raise TypeError("type must be an Enum when using EnumAction") + + # Generate choices from the Enum + choices = tuple(e.value for e in enum_type) + kwargs.setdefault("choices", choices) + kwargs.setdefault("metavar", f"[{','.join(list(choices))}]") + + super(EnumAction, self).__init__(**kwargs) + + self._enum = enum_type + + def __call__(self, parser, namespace, values, option_string=None): + # Convert value back into an Enum + value = self._enum(values) + setattr(namespace, self.dest, value) + + +parser = argparse.ArgumentParser() + +parser.add_argument("--listen", type=str, default="127.0.0.1", metavar="IP", nargs="?", const="0.0.0.0,::", help="Specify the IP address to listen on (default: 127.0.0.1). You can give a list of ip addresses by separating them with a comma like: 127.2.2.2,127.3.3.3 If --listen is provided without an argument, it defaults to 0.0.0.0,:: (listens on all ipv4 and ipv6)") +parser.add_argument("--port", type=int, default=8188, help="Set the listen port.") +parser.add_argument("--tls-keyfile", type=str, help="Path to TLS (SSL) key file. Enables TLS, makes app accessible at https://... requires --tls-certfile to function") +parser.add_argument("--tls-certfile", type=str, help="Path to TLS (SSL) certificate file. Enables TLS, makes app accessible at https://... requires --tls-keyfile to function") +parser.add_argument("--enable-cors-header", type=str, default=None, metavar="ORIGIN", nargs="?", const="*", help="Enable CORS (Cross-Origin Resource Sharing) with optional origin or allow all with default '*'.") +parser.add_argument("--max-upload-size", type=float, default=100, help="Set the maximum upload size in MB.") + +parser.add_argument("--base-directory", type=str, default=None, help="Set the ComfyUI base directory for models, custom_nodes, input, output, temp, and user directories.") +parser.add_argument("--extra-model-paths-config", type=str, default=None, metavar="PATH", nargs='+', action='append', help="Load one or more extra_model_paths.yaml files.") +parser.add_argument("--output-directory", type=str, default=None, help="Set the ComfyUI output directory. Overrides --base-directory.") +parser.add_argument("--temp-directory", type=str, default=None, help="Set the ComfyUI temp directory (default is in the ComfyUI directory). Overrides --base-directory.") +parser.add_argument("--input-directory", type=str, default=None, help="Set the ComfyUI input directory. Overrides --base-directory.") +parser.add_argument("--auto-launch", action="store_true", help="Automatically launch ComfyUI in the default browser.") +parser.add_argument("--disable-auto-launch", action="store_true", help="Disable auto launching the browser.") +parser.add_argument("--cuda-device", type=int, default=None, metavar="DEVICE_ID", help="Set the id of the cuda device this instance will use.") +cm_group = parser.add_mutually_exclusive_group() +cm_group.add_argument("--cuda-malloc", action="store_true", help="Enable cudaMallocAsync (enabled by default for torch 2.0 and up).") +cm_group.add_argument("--disable-cuda-malloc", action="store_true", help="Disable cudaMallocAsync.") + + +fp_group = parser.add_mutually_exclusive_group() +fp_group.add_argument("--force-fp32", action="store_true", help="Force fp32 (If this makes your GPU work better please report it).") +fp_group.add_argument("--force-fp16", action="store_true", help="Force fp16.") + +fpunet_group = parser.add_mutually_exclusive_group() +fpunet_group.add_argument("--fp32-unet", action="store_true", help="Run the diffusion model in fp32.") +fpunet_group.add_argument("--fp64-unet", action="store_true", help="Run the diffusion model in fp64.") +fpunet_group.add_argument("--bf16-unet", action="store_true", help="Run the diffusion model in bf16.") +fpunet_group.add_argument("--fp16-unet", action="store_true", help="Run the diffusion model in fp16") +fpunet_group.add_argument("--fp8_e4m3fn-unet", action="store_true", help="Store unet weights in fp8_e4m3fn.") +fpunet_group.add_argument("--fp8_e5m2-unet", action="store_true", help="Store unet weights in fp8_e5m2.") +fpunet_group.add_argument("--fp8_e8m0fnu-unet", action="store_true", help="Store unet weights in fp8_e8m0fnu.") + +fpvae_group = parser.add_mutually_exclusive_group() +fpvae_group.add_argument("--fp16-vae", action="store_true", help="Run the VAE in fp16, might cause black images.") +fpvae_group.add_argument("--fp32-vae", action="store_true", help="Run the VAE in full precision fp32.") +fpvae_group.add_argument("--bf16-vae", action="store_true", help="Run the VAE in bf16.") + +parser.add_argument("--cpu-vae", action="store_true", help="Run the VAE on the CPU.") + +fpte_group = parser.add_mutually_exclusive_group() +fpte_group.add_argument("--fp8_e4m3fn-text-enc", action="store_true", help="Store text encoder weights in fp8 (e4m3fn variant).") +fpte_group.add_argument("--fp8_e5m2-text-enc", action="store_true", help="Store text encoder weights in fp8 (e5m2 variant).") +fpte_group.add_argument("--fp16-text-enc", action="store_true", help="Store text encoder weights in fp16.") +fpte_group.add_argument("--fp32-text-enc", action="store_true", help="Store text encoder weights in fp32.") +fpte_group.add_argument("--bf16-text-enc", action="store_true", help="Store text encoder weights in bf16.") + +parser.add_argument("--force-channels-last", action="store_true", help="Force channels last format when inferencing the models.") + +parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.") + +parser.add_argument("--oneapi-device-selector", type=str, default=None, metavar="SELECTOR_STRING", help="Sets the oneAPI device(s) this instance will use.") +parser.add_argument("--disable-ipex-optimize", action="store_true", help="Disables ipex.optimize default when loading models with Intel's Extension for Pytorch.") +parser.add_argument("--supports-fp8-compute", action="store_true", help="ComfyUI will act like if the device supports fp8 compute.") + +class LatentPreviewMethod(enum.Enum): + NoPreviews = "none" + Auto = "auto" + Latent2RGB = "latent2rgb" + TAESD = "taesd" + +parser.add_argument("--preview-method", type=LatentPreviewMethod, default=LatentPreviewMethod.NoPreviews, help="Default preview method for sampler nodes.", action=EnumAction) + +parser.add_argument("--preview-size", type=int, default=512, help="Sets the maximum preview size for sampler nodes.") + +cache_group = parser.add_mutually_exclusive_group() +cache_group.add_argument("--cache-classic", action="store_true", help="Use the old style (aggressive) caching.") +cache_group.add_argument("--cache-lru", type=int, default=0, help="Use LRU caching with a maximum of N node results cached. May use more RAM/VRAM.") +cache_group.add_argument("--cache-none", action="store_true", help="Reduced RAM/VRAM usage at the expense of executing every node for each run.") + +attn_group = parser.add_mutually_exclusive_group() +attn_group.add_argument("--use-split-cross-attention", action="store_true", help="Use the split cross attention optimization. Ignored when xformers is used.") +attn_group.add_argument("--use-quad-cross-attention", action="store_true", help="Use the sub-quadratic cross attention optimization . Ignored when xformers is used.") +attn_group.add_argument("--use-pytorch-cross-attention", action="store_true", help="Use the new pytorch 2.0 cross attention function.") +attn_group.add_argument("--use-sage-attention", action="store_true", help="Use sage attention.") +attn_group.add_argument("--use-flash-attention", action="store_true", help="Use FlashAttention.") + +parser.add_argument("--disable-xformers", action="store_true", help="Disable xformers.") + +upcast = parser.add_mutually_exclusive_group() +upcast.add_argument("--force-upcast-attention", action="store_true", help="Force enable attention upcasting, please report if it fixes black images.") +upcast.add_argument("--dont-upcast-attention", action="store_true", help="Disable all upcasting of attention. Should be unnecessary except for debugging.") + + +vram_group = parser.add_mutually_exclusive_group() +vram_group.add_argument("--gpu-only", action="store_true", help="Store and run everything (text encoders/CLIP models, etc... on the GPU).") +vram_group.add_argument("--highvram", action="store_true", help="By default models will be unloaded to CPU memory after being used. This option keeps them in GPU memory.") +vram_group.add_argument("--normalvram", action="store_true", help="Used to force normal vram use if lowvram gets automatically enabled.") +vram_group.add_argument("--lowvram", action="store_true", help="Split the unet in parts to use less vram.") +vram_group.add_argument("--novram", action="store_true", help="When lowvram isn't enough.") +vram_group.add_argument("--cpu", action="store_true", help="To use the CPU for everything (slow).") + +parser.add_argument("--reserve-vram", type=float, default=None, help="Set the amount of vram in GB you want to reserve for use by your OS/other software. By default some amount is reserved depending on your OS.") + +parser.add_argument("--async-offload", action="store_true", help="Use async weight offloading.") + +parser.add_argument("--default-hashing-function", type=str, choices=['md5', 'sha1', 'sha256', 'sha512'], default='sha256', help="Allows you to choose the hash function to use for duplicate filename / contents comparison. Default is sha256.") + +parser.add_argument("--disable-smart-memory", action="store_true", help="Force ComfyUI to agressively offload to regular ram instead of keeping models in vram when it can.") +parser.add_argument("--deterministic", action="store_true", help="Make pytorch use slower deterministic algorithms when it can. Note that this might not make images deterministic in all cases.") + +class PerformanceFeature(enum.Enum): + Fp16Accumulation = "fp16_accumulation" + Fp8MatrixMultiplication = "fp8_matrix_mult" + CublasOps = "cublas_ops" + +parser.add_argument("--fast", nargs="*", type=PerformanceFeature, help="Enable some untested and potentially quality deteriorating optimizations. --fast with no arguments enables everything. You can pass a list specific optimizations if you only want to enable specific ones. Current valid optimizations: fp16_accumulation fp8_matrix_mult cublas_ops") + +parser.add_argument("--mmap-torch-files", action="store_true", help="Use mmap when loading ckpt/pt files.") + +parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.") +parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.") +parser.add_argument("--windows-standalone-build", action="store_true", help="Windows standalone build: Enable convenient things that most people using the standalone windows build will probably enjoy (like auto opening the page on startup).") + +parser.add_argument("--disable-metadata", action="store_true", help="Disable saving prompt metadata in files.") +parser.add_argument("--disable-all-custom-nodes", action="store_true", help="Disable loading all custom nodes.") +parser.add_argument("--whitelist-custom-nodes", type=str, nargs='+', default=[], help="Specify custom node folders to load even when --disable-all-custom-nodes is enabled.") +parser.add_argument("--disable-api-nodes", action="store_true", help="Disable loading all api nodes.") + +parser.add_argument("--multi-user", action="store_true", help="Enables per-user storage.") + +parser.add_argument("--verbose", default='INFO', const='DEBUG', nargs="?", choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='Set the logging level') +parser.add_argument("--log-stdout", action="store_true", help="Send normal process output to stdout instead of stderr (default).") + +# The default built-in provider hosted under web/ +DEFAULT_VERSION_STRING = "comfyanonymous/ComfyUI@latest" + +parser.add_argument( + "--front-end-version", + type=str, + default=DEFAULT_VERSION_STRING, + help=""" + Specifies the version of the frontend to be used. This command needs internet connectivity to query and + download available frontend implementations from GitHub releases. + + The version string should be in the format of: + [repoOwner]/[repoName]@[version] + where version is one of: "latest" or a valid version number (e.g. "1.0.0") + """, +) + +def is_valid_directory(path: str) -> str: + """Validate if the given path is a directory, and check permissions.""" + if not os.path.exists(path): + raise argparse.ArgumentTypeError(f"The path '{path}' does not exist.") + if not os.path.isdir(path): + raise argparse.ArgumentTypeError(f"'{path}' is not a directory.") + if not os.access(path, os.R_OK): + raise argparse.ArgumentTypeError(f"You do not have read permissions for '{path}'.") + return path + +parser.add_argument( + "--front-end-root", + type=is_valid_directory, + default=None, + help="The local filesystem path to the directory where the frontend is located. Overrides --front-end-version.", +) + +parser.add_argument("--user-directory", type=is_valid_directory, default=None, help="Set the ComfyUI user directory with an absolute path. Overrides --base-directory.") + +parser.add_argument("--enable-compress-response-body", action="store_true", help="Enable compressing response body.") + +parser.add_argument( + "--comfy-api-base", + type=str, + default="https://api.comfy.org", + help="Set the base URL for the ComfyUI API. (default: https://api.comfy.org)", +) + +database_default_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "user", "comfyui.db") +) +parser.add_argument("--database-url", type=str, default=f"sqlite:///{database_default_path}", help="Specify the database URL, e.g. for an in-memory database you can use 'sqlite:///:memory:'.") + +if comfy.options.args_parsing: + args = parser.parse_args() +else: + args = parser.parse_args([]) + +if args.windows_standalone_build: + args.auto_launch = True + +if args.disable_auto_launch: + args.auto_launch = False + +if args.force_fp16: + args.fp16_unet = True + + +# '--fast' is not provided, use an empty set +if args.fast is None: + args.fast = set() +# '--fast' is provided with an empty list, enable all optimizations +elif args.fast == []: + args.fast = set(PerformanceFeature) +# '--fast' is provided with a list of performance features, use that list +else: + args.fast = set(args.fast) diff --git a/comfy/clip_config_bigg.json b/comfy/clip_config_bigg.json new file mode 100644 index 0000000000000000000000000000000000000000..35261deef14a68fcc6c5b1fc32914b5c102781a9 --- /dev/null +++ b/comfy/clip_config_bigg.json @@ -0,0 +1,23 @@ +{ + "architectures": [ + "CLIPTextModel" + ], + "attention_dropout": 0.0, + "bos_token_id": 0, + "dropout": 0.0, + "eos_token_id": 49407, + "hidden_act": "gelu", + "hidden_size": 1280, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 5120, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 77, + "model_type": "clip_text_model", + "num_attention_heads": 20, + "num_hidden_layers": 32, + "pad_token_id": 1, + "projection_dim": 1280, + "torch_dtype": "float32", + "vocab_size": 49408 +} diff --git a/comfy/clip_model.py b/comfy/clip_model.py new file mode 100644 index 0000000000000000000000000000000000000000..c8294d4832e06adb3611a3b9311c1bca06297df7 --- /dev/null +++ b/comfy/clip_model.py @@ -0,0 +1,244 @@ +import torch +from comfy.ldm.modules.attention import optimized_attention_for_device +import comfy.ops + +class CLIPAttention(torch.nn.Module): + def __init__(self, embed_dim, heads, dtype, device, operations): + super().__init__() + + self.heads = heads + self.q_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + self.k_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + self.v_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + + self.out_proj = operations.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + + def forward(self, x, mask=None, optimized_attention=None): + q = self.q_proj(x) + k = self.k_proj(x) + v = self.v_proj(x) + + out = optimized_attention(q, k, v, self.heads, mask) + return self.out_proj(out) + +ACTIVATIONS = {"quick_gelu": lambda a: a * torch.sigmoid(1.702 * a), + "gelu": torch.nn.functional.gelu, + "gelu_pytorch_tanh": lambda a: torch.nn.functional.gelu(a, approximate="tanh"), +} + +class CLIPMLP(torch.nn.Module): + def __init__(self, embed_dim, intermediate_size, activation, dtype, device, operations): + super().__init__() + self.fc1 = operations.Linear(embed_dim, intermediate_size, bias=True, dtype=dtype, device=device) + self.activation = ACTIVATIONS[activation] + self.fc2 = operations.Linear(intermediate_size, embed_dim, bias=True, dtype=dtype, device=device) + + def forward(self, x): + x = self.fc1(x) + x = self.activation(x) + x = self.fc2(x) + return x + +class CLIPLayer(torch.nn.Module): + def __init__(self, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations): + super().__init__() + self.layer_norm1 = operations.LayerNorm(embed_dim, dtype=dtype, device=device) + self.self_attn = CLIPAttention(embed_dim, heads, dtype, device, operations) + self.layer_norm2 = operations.LayerNorm(embed_dim, dtype=dtype, device=device) + self.mlp = CLIPMLP(embed_dim, intermediate_size, intermediate_activation, dtype, device, operations) + + def forward(self, x, mask=None, optimized_attention=None): + x += self.self_attn(self.layer_norm1(x), mask, optimized_attention) + x += self.mlp(self.layer_norm2(x)) + return x + + +class CLIPEncoder(torch.nn.Module): + def __init__(self, num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations): + super().__init__() + self.layers = torch.nn.ModuleList([CLIPLayer(embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations) for i in range(num_layers)]) + + def forward(self, x, mask=None, intermediate_output=None): + optimized_attention = optimized_attention_for_device(x.device, mask=mask is not None, small_input=True) + + if intermediate_output is not None: + if intermediate_output < 0: + intermediate_output = len(self.layers) + intermediate_output + + intermediate = None + for i, l in enumerate(self.layers): + x = l(x, mask, optimized_attention) + if i == intermediate_output: + intermediate = x.clone() + return x, intermediate + +class CLIPEmbeddings(torch.nn.Module): + def __init__(self, embed_dim, vocab_size=49408, num_positions=77, dtype=None, device=None, operations=None): + super().__init__() + self.token_embedding = operations.Embedding(vocab_size, embed_dim, dtype=dtype, device=device) + self.position_embedding = operations.Embedding(num_positions, embed_dim, dtype=dtype, device=device) + + def forward(self, input_tokens, dtype=torch.float32): + return self.token_embedding(input_tokens, out_dtype=dtype) + comfy.ops.cast_to(self.position_embedding.weight, dtype=dtype, device=input_tokens.device) + + +class CLIPTextModel_(torch.nn.Module): + def __init__(self, config_dict, dtype, device, operations): + num_layers = config_dict["num_hidden_layers"] + embed_dim = config_dict["hidden_size"] + heads = config_dict["num_attention_heads"] + intermediate_size = config_dict["intermediate_size"] + intermediate_activation = config_dict["hidden_act"] + num_positions = config_dict["max_position_embeddings"] + self.eos_token_id = config_dict["eos_token_id"] + + super().__init__() + self.embeddings = CLIPEmbeddings(embed_dim, num_positions=num_positions, dtype=dtype, device=device, operations=operations) + self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations) + self.final_layer_norm = operations.LayerNorm(embed_dim, dtype=dtype, device=device) + + def forward(self, input_tokens=None, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=torch.float32): + if embeds is not None: + x = embeds + comfy.ops.cast_to(self.embeddings.position_embedding.weight, dtype=dtype, device=embeds.device) + else: + x = self.embeddings(input_tokens, dtype=dtype) + + mask = None + if attention_mask is not None: + mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])).expand(attention_mask.shape[0], 1, attention_mask.shape[-1], attention_mask.shape[-1]) + mask = mask.masked_fill(mask.to(torch.bool), -torch.finfo(x.dtype).max) + + causal_mask = torch.full((x.shape[1], x.shape[1]), -torch.finfo(x.dtype).max, dtype=x.dtype, device=x.device).triu_(1) + + if mask is not None: + mask += causal_mask + else: + mask = causal_mask + + x, i = self.encoder(x, mask=mask, intermediate_output=intermediate_output) + x = self.final_layer_norm(x) + if i is not None and final_layer_norm_intermediate: + i = self.final_layer_norm(i) + + if num_tokens is not None: + pooled_output = x[list(range(x.shape[0])), list(map(lambda a: a - 1, num_tokens))] + else: + pooled_output = x[torch.arange(x.shape[0], device=x.device), (torch.round(input_tokens).to(dtype=torch.int, device=x.device) == self.eos_token_id).int().argmax(dim=-1),] + return x, i, pooled_output + +class CLIPTextModel(torch.nn.Module): + def __init__(self, config_dict, dtype, device, operations): + super().__init__() + self.num_layers = config_dict["num_hidden_layers"] + self.text_model = CLIPTextModel_(config_dict, dtype, device, operations) + embed_dim = config_dict["hidden_size"] + self.text_projection = operations.Linear(embed_dim, embed_dim, bias=False, dtype=dtype, device=device) + self.dtype = dtype + + def get_input_embeddings(self): + return self.text_model.embeddings.token_embedding + + def set_input_embeddings(self, embeddings): + self.text_model.embeddings.token_embedding = embeddings + + def forward(self, *args, **kwargs): + x = self.text_model(*args, **kwargs) + out = self.text_projection(x[2]) + return (x[0], x[1], out, x[2]) + + +class CLIPVisionEmbeddings(torch.nn.Module): + def __init__(self, embed_dim, num_channels=3, patch_size=14, image_size=224, model_type="", dtype=None, device=None, operations=None): + super().__init__() + + num_patches = (image_size // patch_size) ** 2 + if model_type == "siglip_vision_model": + self.class_embedding = None + patch_bias = True + else: + num_patches = num_patches + 1 + self.class_embedding = torch.nn.Parameter(torch.empty(embed_dim, dtype=dtype, device=device)) + patch_bias = False + + self.patch_embedding = operations.Conv2d( + in_channels=num_channels, + out_channels=embed_dim, + kernel_size=patch_size, + stride=patch_size, + bias=patch_bias, + dtype=dtype, + device=device + ) + + self.position_embedding = operations.Embedding(num_patches, embed_dim, dtype=dtype, device=device) + + def forward(self, pixel_values): + embeds = self.patch_embedding(pixel_values).flatten(2).transpose(1, 2) + if self.class_embedding is not None: + embeds = torch.cat([comfy.ops.cast_to_input(self.class_embedding, embeds).expand(pixel_values.shape[0], 1, -1), embeds], dim=1) + return embeds + comfy.ops.cast_to_input(self.position_embedding.weight, embeds) + + +class CLIPVision(torch.nn.Module): + def __init__(self, config_dict, dtype, device, operations): + super().__init__() + num_layers = config_dict["num_hidden_layers"] + embed_dim = config_dict["hidden_size"] + heads = config_dict["num_attention_heads"] + intermediate_size = config_dict["intermediate_size"] + intermediate_activation = config_dict["hidden_act"] + model_type = config_dict["model_type"] + + self.embeddings = CLIPVisionEmbeddings(embed_dim, config_dict["num_channels"], config_dict["patch_size"], config_dict["image_size"], model_type=model_type, dtype=dtype, device=device, operations=operations) + if model_type == "siglip_vision_model": + self.pre_layrnorm = lambda a: a + self.output_layernorm = True + else: + self.pre_layrnorm = operations.LayerNorm(embed_dim) + self.output_layernorm = False + self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device, operations) + self.post_layernorm = operations.LayerNorm(embed_dim) + + def forward(self, pixel_values, attention_mask=None, intermediate_output=None): + x = self.embeddings(pixel_values) + x = self.pre_layrnorm(x) + #TODO: attention_mask? + x, i = self.encoder(x, mask=None, intermediate_output=intermediate_output) + if self.output_layernorm: + x = self.post_layernorm(x) + pooled_output = x + else: + pooled_output = self.post_layernorm(x[:, 0, :]) + return x, i, pooled_output + +class LlavaProjector(torch.nn.Module): + def __init__(self, in_dim, out_dim, dtype, device, operations): + super().__init__() + self.linear_1 = operations.Linear(in_dim, out_dim, bias=True, device=device, dtype=dtype) + self.linear_2 = operations.Linear(out_dim, out_dim, bias=True, device=device, dtype=dtype) + + def forward(self, x): + return self.linear_2(torch.nn.functional.gelu(self.linear_1(x[:, 1:]))) + +class CLIPVisionModelProjection(torch.nn.Module): + def __init__(self, config_dict, dtype, device, operations): + super().__init__() + self.vision_model = CLIPVision(config_dict, dtype, device, operations) + if "projection_dim" in config_dict: + self.visual_projection = operations.Linear(config_dict["hidden_size"], config_dict["projection_dim"], bias=False) + else: + self.visual_projection = lambda a: a + + if "llava3" == config_dict.get("projector_type", None): + self.multi_modal_projector = LlavaProjector(config_dict["hidden_size"], 4096, dtype, device, operations) + else: + self.multi_modal_projector = None + + def forward(self, *args, **kwargs): + x = self.vision_model(*args, **kwargs) + out = self.visual_projection(x[2]) + projected = None + if self.multi_modal_projector is not None: + projected = self.multi_modal_projector(x[1]) + + return (x[0], x[1], out, projected) diff --git a/comfy/clip_vision.py b/comfy/clip_vision.py new file mode 100644 index 0000000000000000000000000000000000000000..00aab9164e5ec2060e34f22df9d1098cfe7b7e47 --- /dev/null +++ b/comfy/clip_vision.py @@ -0,0 +1,148 @@ +from .utils import load_torch_file, transformers_convert, state_dict_prefix_replace +import os +import torch +import json +import logging + +import comfy.ops +import comfy.model_patcher +import comfy.model_management +import comfy.utils +import comfy.clip_model +import comfy.image_encoders.dino2 + +class Output: + def __getitem__(self, key): + return getattr(self, key) + def __setitem__(self, key, item): + setattr(self, key, item) + +def clip_preprocess(image, size=224, mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711], crop=True): + image = image[:, :, :, :3] if image.shape[3] > 3 else image + mean = torch.tensor(mean, device=image.device, dtype=image.dtype) + std = torch.tensor(std, device=image.device, dtype=image.dtype) + image = image.movedim(-1, 1) + if not (image.shape[2] == size and image.shape[3] == size): + if crop: + scale = (size / min(image.shape[2], image.shape[3])) + scale_size = (round(scale * image.shape[2]), round(scale * image.shape[3])) + else: + scale_size = (size, size) + + image = torch.nn.functional.interpolate(image, size=scale_size, mode="bicubic", antialias=True) + h = (image.shape[2] - size)//2 + w = (image.shape[3] - size)//2 + image = image[:,:,h:h+size,w:w+size] + image = torch.clip((255. * image), 0, 255).round() / 255.0 + return (image - mean.view([3,1,1])) / std.view([3,1,1]) + +IMAGE_ENCODERS = { + "clip_vision_model": comfy.clip_model.CLIPVisionModelProjection, + "siglip_vision_model": comfy.clip_model.CLIPVisionModelProjection, + "dinov2": comfy.image_encoders.dino2.Dinov2Model, +} + +class ClipVisionModel(): + def __init__(self, json_config): + with open(json_config) as f: + config = json.load(f) + + self.image_size = config.get("image_size", 224) + self.image_mean = config.get("image_mean", [0.48145466, 0.4578275, 0.40821073]) + self.image_std = config.get("image_std", [0.26862954, 0.26130258, 0.27577711]) + model_class = IMAGE_ENCODERS.get(config.get("model_type", "clip_vision_model")) + self.load_device = comfy.model_management.text_encoder_device() + offload_device = comfy.model_management.text_encoder_offload_device() + self.dtype = comfy.model_management.text_encoder_dtype(self.load_device) + self.model = model_class(config, self.dtype, offload_device, comfy.ops.manual_cast) + self.model.eval() + + self.patcher = comfy.model_patcher.ModelPatcher(self.model, load_device=self.load_device, offload_device=offload_device) + + def load_sd(self, sd): + return self.model.load_state_dict(sd, strict=False) + + def get_sd(self): + return self.model.state_dict() + + def encode_image(self, image, crop=True): + comfy.model_management.load_model_gpu(self.patcher) + pixel_values = clip_preprocess(image.to(self.load_device), size=self.image_size, mean=self.image_mean, std=self.image_std, crop=crop).float() + out = self.model(pixel_values=pixel_values, intermediate_output=-2) + + outputs = Output() + outputs["last_hidden_state"] = out[0].to(comfy.model_management.intermediate_device()) + outputs["image_embeds"] = out[2].to(comfy.model_management.intermediate_device()) + outputs["penultimate_hidden_states"] = out[1].to(comfy.model_management.intermediate_device()) + outputs["mm_projected"] = out[3] + return outputs + +def convert_to_transformers(sd, prefix): + sd_k = sd.keys() + if "{}transformer.resblocks.0.attn.in_proj_weight".format(prefix) in sd_k: + keys_to_replace = { + "{}class_embedding".format(prefix): "vision_model.embeddings.class_embedding", + "{}conv1.weight".format(prefix): "vision_model.embeddings.patch_embedding.weight", + "{}positional_embedding".format(prefix): "vision_model.embeddings.position_embedding.weight", + "{}ln_post.bias".format(prefix): "vision_model.post_layernorm.bias", + "{}ln_post.weight".format(prefix): "vision_model.post_layernorm.weight", + "{}ln_pre.bias".format(prefix): "vision_model.pre_layrnorm.bias", + "{}ln_pre.weight".format(prefix): "vision_model.pre_layrnorm.weight", + } + + for x in keys_to_replace: + if x in sd_k: + sd[keys_to_replace[x]] = sd.pop(x) + + if "{}proj".format(prefix) in sd_k: + sd['visual_projection.weight'] = sd.pop("{}proj".format(prefix)).transpose(0, 1) + + sd = transformers_convert(sd, prefix, "vision_model.", 48) + else: + replace_prefix = {prefix: ""} + sd = state_dict_prefix_replace(sd, replace_prefix) + return sd + +def load_clipvision_from_sd(sd, prefix="", convert_keys=False): + if convert_keys: + sd = convert_to_transformers(sd, prefix) + if "vision_model.encoder.layers.47.layer_norm1.weight" in sd: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_g.json") + elif "vision_model.encoder.layers.30.layer_norm1.weight" in sd: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_h.json") + elif "vision_model.encoder.layers.22.layer_norm1.weight" in sd: + embed_shape = sd["vision_model.embeddings.position_embedding.weight"].shape[0] + if sd["vision_model.encoder.layers.0.layer_norm1.weight"].shape[0] == 1152: + if embed_shape == 729: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_siglip_384.json") + elif embed_shape == 1024: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_siglip_512.json") + elif embed_shape == 577: + if "multi_modal_projector.linear_1.bias" in sd: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl_336_llava.json") + else: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl_336.json") + else: + json_config = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clip_vision_config_vitl.json") + elif "embeddings.patch_embeddings.projection.weight" in sd: + json_config = os.path.join(os.path.join(os.path.dirname(os.path.realpath(__file__)), "image_encoders"), "dino2_giant.json") + else: + return None + + clip = ClipVisionModel(json_config) + m, u = clip.load_sd(sd) + if len(m) > 0: + logging.warning("missing clip vision: {}".format(m)) + u = set(u) + keys = list(sd.keys()) + for k in keys: + if k not in u: + sd.pop(k) + return clip + +def load(ckpt_path): + sd = load_torch_file(ckpt_path) + if "visual.transformer.resblocks.0.attn.in_proj_weight" in sd: + return load_clipvision_from_sd(sd, prefix="visual.", convert_keys=True) + else: + return load_clipvision_from_sd(sd) diff --git a/comfy/clip_vision_config_g.json b/comfy/clip_vision_config_g.json new file mode 100644 index 0000000000000000000000000000000000000000..708e7e21ac3513a719d6a49e88e756f5ef7e2c8d --- /dev/null +++ b/comfy/clip_vision_config_g.json @@ -0,0 +1,18 @@ +{ + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "gelu", + "hidden_size": 1664, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 8192, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 48, + "patch_size": 14, + "projection_dim": 1280, + "torch_dtype": "float32" +} diff --git a/comfy/clip_vision_config_h.json b/comfy/clip_vision_config_h.json new file mode 100644 index 0000000000000000000000000000000000000000..bb71be419a4be0ad5c8c157850de032a65593cb9 --- /dev/null +++ b/comfy/clip_vision_config_h.json @@ -0,0 +1,18 @@ +{ + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "gelu", + "hidden_size": 1280, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 5120, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 32, + "patch_size": 14, + "projection_dim": 1024, + "torch_dtype": "float32" +} diff --git a/comfy/clip_vision_config_vitl.json b/comfy/clip_vision_config_vitl.json new file mode 100644 index 0000000000000000000000000000000000000000..c59b8ed5a4c1f41fbcc9e6811d2c7dfe44273de7 --- /dev/null +++ b/comfy/clip_vision_config_vitl.json @@ -0,0 +1,18 @@ +{ + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "quick_gelu", + "hidden_size": 1024, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 24, + "patch_size": 14, + "projection_dim": 768, + "torch_dtype": "float32" +} diff --git a/comfy/clip_vision_config_vitl_336.json b/comfy/clip_vision_config_vitl_336.json new file mode 100644 index 0000000000000000000000000000000000000000..f26945273d99e88f207d64dcec78feee63b4b625 --- /dev/null +++ b/comfy/clip_vision_config_vitl_336.json @@ -0,0 +1,18 @@ +{ + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "quick_gelu", + "hidden_size": 1024, + "image_size": 336, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-5, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 24, + "patch_size": 14, + "projection_dim": 768, + "torch_dtype": "float32" +} diff --git a/comfy/clip_vision_config_vitl_336_llava.json b/comfy/clip_vision_config_vitl_336_llava.json new file mode 100644 index 0000000000000000000000000000000000000000..f23a50d8b77fa29de2af621fb50c5825cf9b1a86 --- /dev/null +++ b/comfy/clip_vision_config_vitl_336_llava.json @@ -0,0 +1,19 @@ +{ + "attention_dropout": 0.0, + "dropout": 0.0, + "hidden_act": "quick_gelu", + "hidden_size": 1024, + "image_size": 336, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-5, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 24, + "patch_size": 14, + "projection_dim": 768, + "projector_type": "llava3", + "torch_dtype": "float32" +} diff --git a/comfy/clip_vision_siglip_384.json b/comfy/clip_vision_siglip_384.json new file mode 100644 index 0000000000000000000000000000000000000000..532e03ac181d8849a7202445d42565f01441177b --- /dev/null +++ b/comfy/clip_vision_siglip_384.json @@ -0,0 +1,13 @@ +{ + "num_channels": 3, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 384, + "intermediate_size": 4304, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_hidden_layers": 27, + "patch_size": 14, + "image_mean": [0.5, 0.5, 0.5], + "image_std": [0.5, 0.5, 0.5] +} diff --git a/comfy/clip_vision_siglip_512.json b/comfy/clip_vision_siglip_512.json new file mode 100644 index 0000000000000000000000000000000000000000..7fb93ce15e6da3ce7653a7a91ab278d707a096b4 --- /dev/null +++ b/comfy/clip_vision_siglip_512.json @@ -0,0 +1,13 @@ +{ + "num_channels": 3, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "image_size": 512, + "intermediate_size": 4304, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_hidden_layers": 27, + "patch_size": 16, + "image_mean": [0.5, 0.5, 0.5], + "image_std": [0.5, 0.5, 0.5] +} diff --git a/comfy/comfy_types/README.md b/comfy/comfy_types/README.md new file mode 100644 index 0000000000000000000000000000000000000000..20a786a5eac805a0b3f41c4fa42be558819b89c8 --- /dev/null +++ b/comfy/comfy_types/README.md @@ -0,0 +1,43 @@ +# Comfy Typing +## Type hinting for ComfyUI Node development + +This module provides type hinting and concrete convenience types for node developers. +If cloned to the custom_nodes directory of ComfyUI, types can be imported using: + +```python +from comfy.comfy_types import IO, ComfyNodeABC, CheckLazyMixin + +class ExampleNode(ComfyNodeABC): + @classmethod + def INPUT_TYPES(s) -> InputTypeDict: + return {"required": {}} +``` + +Full example is in [examples/example_nodes.py](examples/example_nodes.py). + +# Types +A few primary types are documented below. More complete information is available via the docstrings on each type. + +## `IO` + +A string enum of built-in and a few custom data types. Includes the following special types and their requisite plumbing: + +- `ANY`: `"*"` +- `NUMBER`: `"FLOAT,INT"` +- `PRIMITIVE`: `"STRING,FLOAT,INT,BOOLEAN"` + +## `ComfyNodeABC` + +An abstract base class for nodes, offering type-hinting / autocomplete, and somewhat-alright docstrings. + +### Type hinting for `INPUT_TYPES` + + + +### `INPUT_TYPES` return dict + + + +### Options for individual inputs + + diff --git a/comfy/comfy_types/__init__.py b/comfy/comfy_types/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7640fbe3f827bdaac2384e0dcc96d243ec257f2f --- /dev/null +++ b/comfy/comfy_types/__init__.py @@ -0,0 +1,46 @@ +import torch +from typing import Callable, Protocol, TypedDict, Optional, List +from .node_typing import IO, InputTypeDict, ComfyNodeABC, CheckLazyMixin, FileLocator + + +class UnetApplyFunction(Protocol): + """Function signature protocol on comfy.model_base.BaseModel.apply_model""" + + def __call__(self, x: torch.Tensor, t: torch.Tensor, **kwargs) -> torch.Tensor: + pass + + +class UnetApplyConds(TypedDict): + """Optional conditions for unet apply function.""" + + c_concat: Optional[torch.Tensor] + c_crossattn: Optional[torch.Tensor] + control: Optional[torch.Tensor] + transformer_options: Optional[dict] + + +class UnetParams(TypedDict): + # Tensor of shape [B, C, H, W] + input: torch.Tensor + # Tensor of shape [B] + timestep: torch.Tensor + c: UnetApplyConds + # List of [0, 1], [0], [1], ... + # 0 means conditional, 1 means conditional unconditional + cond_or_uncond: List[int] + + +UnetWrapperFunction = Callable[[UnetApplyFunction, UnetParams], torch.Tensor] + + +__all__ = [ + "UnetWrapperFunction", + UnetApplyConds.__name__, + UnetParams.__name__, + UnetApplyFunction.__name__, + IO.__name__, + InputTypeDict.__name__, + ComfyNodeABC.__name__, + CheckLazyMixin.__name__, + FileLocator.__name__, +] diff --git a/comfy/comfy_types/examples/example_nodes.py b/comfy/comfy_types/examples/example_nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..6e19c545153db36b2c2e7481e5c8bc96ccbbe1f1 --- /dev/null +++ b/comfy/comfy_types/examples/example_nodes.py @@ -0,0 +1,28 @@ +from comfy.comfy_types import IO, ComfyNodeABC, InputTypeDict +from inspect import cleandoc + + +class ExampleNode(ComfyNodeABC): + """An example node that just adds 1 to an input integer. + + * Requires a modern IDE to provide any benefit (detail: an IDE configured with analysis paths etc). + * This node is intended as an example for developers only. + """ + + DESCRIPTION = cleandoc(__doc__) + CATEGORY = "examples" + + @classmethod + def INPUT_TYPES(s) -> InputTypeDict: + return { + "required": { + "input_int": (IO.INT, {"defaultInput": True}), + } + } + + RETURN_TYPES = (IO.INT,) + RETURN_NAMES = ("input_plus_one",) + FUNCTION = "execute" + + def execute(self, input_int: int): + return (input_int + 1,) diff --git a/comfy/comfy_types/examples/input_options.png b/comfy/comfy_types/examples/input_options.png new file mode 100644 index 0000000000000000000000000000000000000000..ac859bbc0c15728e6f5464f9388f44d5e38dd650 Binary files /dev/null and b/comfy/comfy_types/examples/input_options.png differ diff --git a/comfy/comfy_types/examples/input_types.png b/comfy/comfy_types/examples/input_types.png new file mode 100644 index 0000000000000000000000000000000000000000..27e031ccf9c32958da0d41164b567e252c1b0c5c Binary files /dev/null and b/comfy/comfy_types/examples/input_types.png differ diff --git a/comfy/comfy_types/examples/required_hint.png b/comfy/comfy_types/examples/required_hint.png new file mode 100644 index 0000000000000000000000000000000000000000..22c0182a0aed3245201f2ffac86a6d3ad5b7e2ea Binary files /dev/null and b/comfy/comfy_types/examples/required_hint.png differ diff --git a/comfy/comfy_types/node_typing.py b/comfy/comfy_types/node_typing.py new file mode 100644 index 0000000000000000000000000000000000000000..071b98332ee2a588448fde472f9c0f104e72d13d --- /dev/null +++ b/comfy/comfy_types/node_typing.py @@ -0,0 +1,350 @@ +"""Comfy-specific type hinting""" + +from __future__ import annotations +from typing import Literal, TypedDict, Optional +from typing_extensions import NotRequired +from abc import ABC, abstractmethod +from enum import Enum + + +class StrEnum(str, Enum): + """Base class for string enums. Python's StrEnum is not available until 3.11.""" + + def __str__(self) -> str: + return self.value + + +class IO(StrEnum): + """Node input/output data types. + + Includes functionality for ``"*"`` (`ANY`) and ``"MULTI,TYPES"``. + """ + + STRING = "STRING" + IMAGE = "IMAGE" + MASK = "MASK" + LATENT = "LATENT" + BOOLEAN = "BOOLEAN" + INT = "INT" + FLOAT = "FLOAT" + COMBO = "COMBO" + CONDITIONING = "CONDITIONING" + SAMPLER = "SAMPLER" + SIGMAS = "SIGMAS" + GUIDER = "GUIDER" + NOISE = "NOISE" + CLIP = "CLIP" + CONTROL_NET = "CONTROL_NET" + VAE = "VAE" + MODEL = "MODEL" + LORA_MODEL = "LORA_MODEL" + LOSS_MAP = "LOSS_MAP" + CLIP_VISION = "CLIP_VISION" + CLIP_VISION_OUTPUT = "CLIP_VISION_OUTPUT" + STYLE_MODEL = "STYLE_MODEL" + GLIGEN = "GLIGEN" + UPSCALE_MODEL = "UPSCALE_MODEL" + AUDIO = "AUDIO" + WEBCAM = "WEBCAM" + POINT = "POINT" + FACE_ANALYSIS = "FACE_ANALYSIS" + BBOX = "BBOX" + SEGS = "SEGS" + VIDEO = "VIDEO" + + ANY = "*" + """Always matches any type, but at a price. + + Causes some functionality issues (e.g. reroutes, link types), and should be avoided whenever possible. + """ + NUMBER = "FLOAT,INT" + """A float or an int - could be either""" + PRIMITIVE = "STRING,FLOAT,INT,BOOLEAN" + """Could be any of: string, float, int, or bool""" + + def __ne__(self, value: object) -> bool: + if self == "*" or value == "*": + return False + if not isinstance(value, str): + return True + a = frozenset(self.split(",")) + b = frozenset(value.split(",")) + return not (b.issubset(a) or a.issubset(b)) + + +class RemoteInputOptions(TypedDict): + route: str + """The route to the remote source.""" + refresh_button: bool + """Specifies whether to show a refresh button in the UI below the widget.""" + control_after_refresh: Literal["first", "last"] + """Specifies the control after the refresh button is clicked. If "first", the first item will be automatically selected, and so on.""" + timeout: int + """The maximum amount of time to wait for a response from the remote source in milliseconds.""" + max_retries: int + """The maximum number of retries before aborting the request.""" + refresh: int + """The TTL of the remote input's value in milliseconds. Specifies the interval at which the remote input's value is refreshed.""" + + +class MultiSelectOptions(TypedDict): + placeholder: NotRequired[str] + """The placeholder text to display in the multi-select widget when no items are selected.""" + chip: NotRequired[bool] + """Specifies whether to use chips instead of comma separated values for the multi-select widget.""" + + +class InputTypeOptions(TypedDict): + """Provides type hinting for the return type of the INPUT_TYPES node function. + + Due to IDE limitations with unions, for now all options are available for all types (e.g. `label_on` is hinted even when the type is not `IO.BOOLEAN`). + + Comfy Docs: https://docs.comfy.org/custom-nodes/backend/datatypes + """ + + default: NotRequired[bool | str | float | int | list | tuple] + """The default value of the widget""" + defaultInput: NotRequired[bool] + """@deprecated in v1.16 frontend. v1.16 frontend allows input socket and widget to co-exist. + - defaultInput on required inputs should be dropped. + - defaultInput on optional inputs should be replaced with forceInput. + Ref: https://github.com/Comfy-Org/ComfyUI_frontend/pull/3364 + """ + forceInput: NotRequired[bool] + """Forces the input to be an input slot rather than a widget even a widget is available for the input type.""" + lazy: NotRequired[bool] + """Declares that this input uses lazy evaluation""" + rawLink: NotRequired[bool] + """When a link exists, rather than receiving the evaluated value, you will receive the link (i.e. `["nodeId",&": 5909,
+ "CON": 5910,
+ "Ġrepl": 5911,
+ "Ġregular": 5912,
+ "Storage": 5913,
+ "ramework": 5914,
+ "Ġgoal": 5915,
+ "Ġtouch": 5916,
+ ".widget": 5917,
+ "Ġbuilt": 5918,
+ "des": 5919,
+ "Part": 5920,
+ "(re": 5921,
+ "Ġworth": 5922,
+ "hib": 5923,
+ "game": 5924,
+ "91": 5925,
+ "192": 5926,
+ "Ġв": 5927,
+ "acion": 5928,
+ "ĠWhite": 5929,
+ "(type": 5930,
+ "(`": 5931,
+ "81": 5932,
+ "Ġnatural": 5933,
+ "Ġinj": 5934,
+ "Ġcalcul": 5935,
+ "ĠApril": 5936,
+ ".List": 5937,
+ "Ġassociated": 5938,
+ "ĉSystem": 5939,
+ "~~": 5940,
+ "=[": 5941,
+ "Ġstorage": 5942,
+ "Ġbytes": 5943,
+ "Ġtravel": 5944,
+ "Ġsou": 5945,
+ "Ġpassed": 5946,
+ "!=": 5947,
+ "ascript": 5948,
+ ".open": 5949,
+ "Ġgrid": 5950,
+ "Ġbus": 5951,
+ "Ġrecogn": 5952,
+ "Ab": 5953,
+ "Ġhon": 5954,
+ "ĠCenter": 5955,
+ "Ġprec": 5956,
+ "build": 5957,
+ "73": 5958,
+ "HTML": 5959,
+ "ĠSan": 5960,
+ "Ġcountries": 5961,
+ "aled": 5962,
+ "token": 5963,
+ "kt": 5964,
+ "Ġqual": 5965,
+ "Last": 5966,
+ "adow": 5967,
+ "Ġmanufact": 5968,
+ "idad": 5969,
+ "jango": 5970,
+ "Next": 5971,
+ "xf": 5972,
+ ".a": 5973,
+ "Ġporno": 5974,
+ "ĠPM": 5975,
+ "erve": 5976,
+ "iting": 5977,
+ "_th": 5978,
+ "ci": 5979,
+ "=None": 5980,
+ "gs": 5981,
+ "Ġlogin": 5982,
+ "atives": 5983,
+ "']);Ċ": 5984,
+ "Äħ": 5985,
+ "Ġill": 5986,
+ "IA": 5987,
+ "children": 5988,
+ "DO": 5989,
+ "Ġlevels": 5990,
+ "Ġ{{": 5991,
+ "Ġlooks": 5992,
+ "Ġ\"#": 5993,
+ "ToString": 5994,
+ "Ġnecessary": 5995,
+ "ĠĠĠĊ": 5996,
+ "cell": 5997,
+ "Entry": 5998,
+ "Ġ'#": 5999,
+ "Ġextrem": 6000,
+ "Selector": 6001,
+ "Ġplaceholder": 6002,
+ "Load": 6003,
+ "Ġreleased": 6004,
+ "ORE": 6005,
+ "Enumer": 6006,
+ "ĠTV": 6007,
+ "SET": 6008,
+ "inq": 6009,
+ "Press": 6010,
+ "ĠDepartment": 6011,
+ "Ġproperties": 6012,
+ "Ġrespond": 6013,
+ "Search": 6014,
+ "ael": 6015,
+ "Ġrequ": 6016,
+ "ĠBook": 6017,
+ "/Ċ": 6018,
+ "(st": 6019,
+ "Ġfinancial": 6020,
+ "icket": 6021,
+ "_input": 6022,
+ "Ġthreat": 6023,
+ "(in": 6024,
+ "Strip": 6025,
+ "ìĿ": 6026,
+ "ção": 6027,
+ "71": 6028,
+ "Ġevidence": 6029,
+ "));": 6030,
+ "ĠBro": 6031,
+ "Ġ[];Ċ": 6032,
+ "Ġou": 6033,
+ "buf": 6034,
+ "Script": 6035,
+ "dat": 6036,
+ "Ġrule": 6037,
+ "#import": 6038,
+ "=\"/": 6039,
+ "Serial": 6040,
+ "Ġstarting": 6041,
+ "[index": 6042,
+ "ae": 6043,
+ "Ġcontrib": 6044,
+ "session": 6045,
+ "_new": 6046,
+ "utable": 6047,
+ "ober": 6048,
+ "Ġ\"./": 6049,
+ "Ġlogger": 6050,
+ "Ġrecently": 6051,
+ "Ġreturned": 6052,
+ "ččĊ": 6053,
+ ")))Ċ": 6054,
+ "itions": 6055,
+ "Ġseek": 6056,
+ "Ġcommunic": 6057,
+ "Ġ\".": 6058,
+ "Ġusername": 6059,
+ "ECT": 6060,
+ "DS": 6061,
+ "Ġotherwise": 6062,
+ "ĠGerman": 6063,
+ ".aw": 6064,
+ "Adapter": 6065,
+ "ixel": 6066,
+ "Ġsystems": 6067,
+ "Ġdrop": 6068,
+ "83": 6069,
+ "Ġstructure": 6070,
+ "Ġ$(\"#": 6071,
+ "encies": 6072,
+ "anning": 6073,
+ "ĠLink": 6074,
+ "ĠResponse": 6075,
+ "Ġstri": 6076,
+ "ż": 6077,
+ "ĠDB": 6078,
+ "æĹ": 6079,
+ "android": 6080,
+ "submit": 6081,
+ "otion": 6082,
+ "92": 6083,
+ "(@": 6084,
+ ".test": 6085,
+ "82": 6086,
+ "ĊĊĊĊĊĊĊĊ": 6087,
+ "];čĊ": 6088,
+ "Ġdirectly": 6089,
+ "Ġ\"%": 6090,
+ "ris": 6091,
+ "elta": 6092,
+ "AIL": 6093,
+ "){čĊ": 6094,
+ "mine": 6095,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 6096,
+ "(k": 6097,
+ "bon": 6098,
+ "asic": 6099,
+ "pite": 6100,
+ "___": 6101,
+ "Max": 6102,
+ "Ġerrors": 6103,
+ "ĠWhile": 6104,
+ "Ġarguments": 6105,
+ "Ġensure": 6106,
+ "Right": 6107,
+ "-based": 6108,
+ "Web": 6109,
+ "Ġ-=": 6110,
+ "Ġintrodu": 6111,
+ "ĠInst": 6112,
+ "ĠWash": 6113,
+ "ordin": 6114,
+ "join": 6115,
+ "Database": 6116,
+ "Ġgrad": 6117,
+ "Ġusually": 6118,
+ "ITE": 6119,
+ "Props": 6120,
+ "?>Ċ": 6121,
+ "ĠGo": 6122,
+ "@Override": 6123,
+ "REF": 6124,
+ "Ġip": 6125,
+ "ĠAustral": 6126,
+ "Ġist": 6127,
+ "ViewById": 6128,
+ "Ġserious": 6129,
+ "Ġcustomer": 6130,
+ ".prototype": 6131,
+ "odo": 6132,
+ "cor": 6133,
+ "Ġdoor": 6134,
+ "ĠWITHOUT": 6135,
+ "Ġplant": 6136,
+ "Ġbegan": 6137,
+ "Ġdistance": 6138,
+ "()).": 6139,
+ "Ġchance": 6140,
+ "Ġord": 6141,
+ "came": 6142,
+ "pragma": 6143,
+ "Ġprotect": 6144,
+ "ragment": 6145,
+ "ĠNode": 6146,
+ "ening": 6147,
+ "Ñĩ": 6148,
+ "Ġroute": 6149,
+ "ĠSchool": 6150,
+ "hi": 6151,
+ "Ġneighb": 6152,
+ "After": 6153,
+ "licit": 6154,
+ "Ġcontr": 6155,
+ "Ġprimary": 6156,
+ "AA": 6157,
+ ".WriteLine": 6158,
+ "utils": 6159,
+ "Ġbi": 6160,
+ "Red": 6161,
+ ".Linq": 6162,
+ ".object": 6163,
+ "Ġleaders": 6164,
+ "unities": 6165,
+ "Ġgun": 6166,
+ "onth": 6167,
+ "ĠDev": 6168,
+ "FILE": 6169,
+ "Ġcomments": 6170,
+ "_len": 6171,
+ "arrow": 6172,
+ "amount": 6173,
+ "Range": 6174,
+ "sert": 6175,
+ "GridView": 6176,
+ "Ġupdated": 6177,
+ "ĠMo": 6178,
+ "Ġinform": 6179,
+ "ociety": 6180,
+ "ala": 6181,
+ "Access": 6182,
+ "Ġhab": 6183,
+ "Ġcreat": 6184,
+ "_arg": 6185,
+ "ĠJanuary": 6186,
+ "ĠDay": 6187,
+ "\")čĊ": 6188,
+ "uple": 6189,
+ "document": 6190,
+ "gorith": 6191,
+ "menu": 6192,
+ "ĠOver": 6193,
+ "bb": 6194,
+ ".title": 6195,
+ "_out": 6196,
+ "Ġled": 6197,
+ "uri": 6198,
+ "Ġ?>": 6199,
+ "gl": 6200,
+ "Ġbank": 6201,
+ "ayment": 6202,
+ "ĉprintf": 6203,
+ "MD": 6204,
+ "Ġsample": 6205,
+ "Ġhands": 6206,
+ "ĠVersion": 6207,
+ "uario": 6208,
+ "Ġoffers": 6209,
+ "ityEngine": 6210,
+ "Ġshape": 6211,
+ "Ġsleep": 6212,
+ "_point": 6213,
+ "Settings": 6214,
+ "Ġachie": 6215,
+ "Ġsold": 6216,
+ "ota": 6217,
+ ".bind": 6218,
+ "Am": 6219,
+ "Ġsafe": 6220,
+ "Store": 6221,
+ "Ġshared": 6222,
+ "Ġpriv": 6223,
+ "_VAL": 6224,
+ "Ġsens": 6225,
+ "){": 6226,
+ "Ġremember": 6227,
+ "shared": 6228,
+ "element": 6229,
+ "Ġshoot": 6230,
+ "Vert": 6231,
+ "cout": 6232,
+ "Ġenv": 6233,
+ "_label": 6234,
+ "Ġ>Ċ": 6235,
+ "run": 6236,
+ "Ġscene": 6237,
+ "(array": 6238,
+ "device": 6239,
+ "_title": 6240,
+ "agon": 6241,
+ "]čĊ": 6242,
+ "aby": 6243,
+ "Ġbecame": 6244,
+ "boolean": 6245,
+ "Ġpark": 6246,
+ "ĠCode": 6247,
+ "upload": 6248,
+ "riday": 6249,
+ "ĠSeptember": 6250,
+ "Fe": 6251,
+ "Ġsen": 6252,
+ "cing": 6253,
+ "FL": 6254,
+ "Col": 6255,
+ "uts": 6256,
+ "_page": 6257,
+ "inn": 6258,
+ "Ġimplied": 6259,
+ "aling": 6260,
+ "Ġyourself": 6261,
+ ".Count": 6262,
+ "conf": 6263,
+ "Ġaud": 6264,
+ "_init": 6265,
+ ".)": 6266,
+ "Ġwrote": 6267,
+ "003": 6268,
+ "NG": 6269,
+ ".Error": 6270,
+ "ä»": 6271,
+ ".for": 6272,
+ "Ġequal": 6273,
+ "ĠRequest": 6274,
+ "Ġserial": 6275,
+ "Ġallows": 6276,
+ "XX": 6277,
+ "Ġmiddle": 6278,
+ "chor": 6279,
+ "195": 6280,
+ "94": 6281,
+ "ø": 6282,
+ "erval": 6283,
+ ".Column": 6284,
+ "reading": 6285,
+ "Ġescort": 6286,
+ "ĠAugust": 6287,
+ "Ġquickly": 6288,
+ "Ġweap": 6289,
+ "ĠCG": 6290,
+ "ropri": 6291,
+ "ho": 6292,
+ "Ġcop": 6293,
+ "(struct": 6294,
+ "ĠBig": 6295,
+ "Ġvs": 6296,
+ "Ġfrequ": 6297,
+ ".Value": 6298,
+ "Ġactions": 6299,
+ "Ġproper": 6300,
+ "Ġinn": 6301,
+ "Ġobjects": 6302,
+ "Ġmatrix": 6303,
+ "avascript": 6304,
+ "Ġones": 6305,
+ ".group": 6306,
+ "Ġgreen": 6307,
+ "Ġpaint": 6308,
+ "ools": 6309,
+ "ycl": 6310,
+ "encode": 6311,
+ "olt": 6312,
+ "comment": 6313,
+ ".api": 6314,
+ "Dir": 6315,
+ "Ġune": 6316,
+ "izont": 6317,
+ ".position": 6318,
+ "Ġdesigned": 6319,
+ "_val": 6320,
+ "avi": 6321,
+ "iring": 6322,
+ "tab": 6323,
+ "Ġlayer": 6324,
+ "Ġviews": 6325,
+ "Ġreve": 6326,
+ "rael": 6327,
+ "ĠON": 6328,
+ "rics": 6329,
+ "160": 6330,
+ "np": 6331,
+ "Ġcore": 6332,
+ "());čĊ": 6333,
+ "Main": 6334,
+ "Ġexpert": 6335,
+ "ĉĉčĊ": 6336,
+ "_en": 6337,
+ "Ġ/>": 6338,
+ "utter": 6339,
+ "IAL": 6340,
+ "ails": 6341,
+ "ĠKing": 6342,
+ "*/ĊĊ": 6343,
+ "ĠMet": 6344,
+ "_end": 6345,
+ "addr": 6346,
+ "ora": 6347,
+ "Ġir": 6348,
+ "Min": 6349,
+ "Ġsurpr": 6350,
+ "Ġrepe": 6351,
+ "Ġdirectory": 6352,
+ "PUT": 6353,
+ "-S": 6354,
+ "Ġelection": 6355,
+ "haps": 6356,
+ ".pre": 6357,
+ "cm": 6358,
+ "Values": 6359,
+ "Ġ\"Ċ": 6360,
+ "column": 6361,
+ "ivil": 6362,
+ "Login": 6363,
+ "inue": 6364,
+ "93": 6365,
+ "Ġbeautiful": 6366,
+ "Ġsecret": 6367,
+ "(event": 6368,
+ "Ġchat": 6369,
+ "ums": 6370,
+ "Ġorigin": 6371,
+ "Ġeffects": 6372,
+ "Ġmanagement": 6373,
+ "illa": 6374,
+ "tk": 6375,
+ "Ġsetting": 6376,
+ "ĠCour": 6377,
+ "Ġmassage": 6378,
+ "ĉend": 6379,
+ "Ġhappy": 6380,
+ "Ġfinish": 6381,
+ "Ġcamera": 6382,
+ "ĠVer": 6383,
+ "ĠDemocr": 6384,
+ "ĠHer": 6385,
+ "(Q": 6386,
+ "cons": 6387,
+ "ita": 6388,
+ "Ġ'.": 6389,
+ "{}": 6390,
+ "ĉC": 6391,
+ "Ġstuff": 6392,
+ "194": 6393,
+ "Ġ:Ċ": 6394,
+ "ĠAR": 6395,
+ "Task": 6396,
+ "hidden": 6397,
+ "eros": 6398,
+ "IGN": 6399,
+ "atio": 6400,
+ "ĠHealth": 6401,
+ "olute": 6402,
+ "Enter": 6403,
+ "'>": 6404,
+ "ĠTwitter": 6405,
+ "ĠCounty": 6406,
+ "scribe": 6407,
+ "Ġ=>Ċ": 6408,
+ "Ġhy": 6409,
+ "fit": 6410,
+ "Ġmilitary": 6411,
+ "Ġsale": 6412,
+ "required": 6413,
+ "non": 6414,
+ "bootstrap": 6415,
+ "hold": 6416,
+ "rim": 6417,
+ "-old": 6418,
+ "ĠDown": 6419,
+ "Ġmention": 6420,
+ "contact": 6421,
+ "_group": 6422,
+ "oday": 6423,
+ "Ġtown": 6424,
+ "Ġsolution": 6425,
+ "uate": 6426,
+ "elling": 6427,
+ "]->": 6428,
+ "otes": 6429,
+ "ental": 6430,
+ "omen": 6431,
+ "ospital": 6432,
+ "ĠSup": 6433,
+ "_EN": 6434,
+ "Ġslow": 6435,
+ "SESSION": 6436,
+ "Ġblue": 6437,
+ "ago": 6438,
+ "Ġlives": 6439,
+ "Ġ^": 6440,
+ ".un": 6441,
+ "inst": 6442,
+ "enge": 6443,
+ "Ġcustomers": 6444,
+ "Ġcast": 6445,
+ "udget": 6446,
+ "ï¼ģ": 6447,
+ "icens": 6448,
+ "Ġdetermin": 6449,
+ "Selected": 6450,
+ "_pl": 6451,
+ "ueue": 6452,
+ "Ġdark": 6453,
+ "//ĊĊ": 6454,
+ "si": 6455,
+ "thern": 6456,
+ "ĠJapan": 6457,
+ "/w": 6458,
+ "PU": 6459,
+ "ĠEast": 6460,
+ "ovie": 6461,
+ "Ġpackage": 6462,
+ "Ġnor": 6463,
+ "Ġapi": 6464,
+ "bot": 6465,
+ "\"];Ċ": 6466,
+ "_post": 6467,
+ "ulate": 6468,
+ "Ġclub": 6469,
+ "'));Ċ": 6470,
+ "Ġloop": 6471,
+ "PIO": 6472,
+ "ione": 6473,
+ "shot": 6474,
+ "Initial": 6475,
+ "Ġplayed": 6476,
+ "register": 6477,
+ "rought": 6478,
+ "_max": 6479,
+ "acement": 6480,
+ "match": 6481,
+ "raphics": 6482,
+ "AST": 6483,
+ "Ġexisting": 6484,
+ "Ġcomplex": 6485,
+ "DA": 6486,
+ ".Ch": 6487,
+ ".common": 6488,
+ "mo": 6489,
+ "Ġ'../../": 6490,
+ "ito": 6491,
+ "Ġanalysis": 6492,
+ "Ġdeliver": 6493,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 6494,
+ "idx": 6495,
+ "Ãł": 6496,
+ "ongo": 6497,
+ "ĠEnglish": 6498,
+ "Ċ": 10197,
+ "_default": 10198,
+ "ĠDatabase": 10199,
+ "rep": 10200,
+ "ESS": 10201,
+ "nergy": 10202,
+ ".Find": 10203,
+ "_mask": 10204,
+ "Ġrise": 10205,
+ "Ġkernel": 10206,
+ "::$": 10207,
+ ".Q": 10208,
+ "Ġoffering": 10209,
+ "decl": 10210,
+ "ĠCS": 10211,
+ "Ġlisted": 10212,
+ "Ġmostly": 10213,
+ "enger": 10214,
+ "Ġblocks": 10215,
+ "olo": 10216,
+ "Ġgoverning": 10217,
+ "\\F": 10218,
+ "Ġconcent": 10219,
+ ".getText": 10220,
+ "Ġmb": 10221,
+ "Ġoccurred": 10222,
+ "Ġchanging": 10223,
+ "Scene": 10224,
+ "_CODE": 10225,
+ "Beh": 10226,
+ "\"The": 10227,
+ "Ġtile": 10228,
+ "ĠAssociation": 10229,
+ "ĉP": 10230,
+ "alty": 10231,
+ "_ad": 10232,
+ "odies": 10233,
+ "iated": 10234,
+ "Ġprepared": 10235,
+ "possible": 10236,
+ "Ġmort": 10237,
+ "TEST": 10238,
+ "142": 10239,
+ "Ġignore": 10240,
+ "Ġcalc": 10241,
+ "Ġrs": 10242,
+ "ĠassertEquals": 10243,
+ "Ġsz": 10244,
+ "ĠTHIS": 10245,
+ ".\"Ċ": 10246,
+ "Ġcanvas": 10247,
+ "java": 10248,
+ "Ġdut": 10249,
+ "VALID": 10250,
+ ".sql": 10251,
+ ".input": 10252,
+ "Ġaux": 10253,
+ "Sup": 10254,
+ "Ġartist": 10255,
+ "Vec": 10256,
+ "_TIME": 10257,
+ ".stringify": 10258,
+ "etween": 10259,
+ "ĠCategory": 10260,
+ "Ġ[-": 10261,
+ "ĠDevExpress": 10262,
+ "ĠJul": 10263,
+ "Ġring": 10264,
+ ".ed": 10265,
+ "YY": 10266,
+ "Let": 10267,
+ "TextField": 10268,
+ "Ġflat": 10269,
+ "_print": 10270,
+ "ĠOTHER": 10271,
+ "adian": 10272,
+ "Ġchecked": 10273,
+ "ele": 10274,
+ "Align": 10275,
+ "standing": 10276,
+ "Ġ[],": 10277,
+ "Ġlab": 10278,
+ "ucky": 10279,
+ "ĠChristmas": 10280,
+ "(image": 10281,
+ ".module": 10282,
+ "Ġlots": 10283,
+ "Ġslightly": 10284,
+ "(final": 10285,
+ "erge": 10286,
+ "è¿": 10287,
+ "147": 10288,
+ "ĠPolice": 10289,
+ "143": 10290,
+ "ĠRight": 10291,
+ "Ġaward": 10292,
+ "ĠOS": 10293,
+ "Ġ{}ĊĊ": 10294,
+ "Ġptr": 10295,
+ "oves": 10296,
+ "icated": 10297,
+ "ем": 10298,
+ "Ġmanage": 10299,
+ "oliday": 10300,
+ "Amount": 10301,
+ "oolStrip": 10302,
+ "tbody": 10303,
+ "Nav": 10304,
+ "wrap": 10305,
+ "BB": 10306,
+ "Ġwatching": 10307,
+ "arios": 10308,
+ "Ġoptional": 10309,
+ "_K": 10310,
+ "ĠLicensed": 10311,
+ ".Map": 10312,
+ "Timer": 10313,
+ "ĠAP": 10314,
+ "ĠRev": 10315,
+ "(o": 10316,
+ ",c": 10317,
+ "umin": 10318,
+ "etailed": 10319,
+ "ĠHy": 10320,
+ "Ġblank": 10321,
+ "agger": 10322,
+ "ĠSelf": 10323,
+ "()[": 10324,
+ ".make": 10325,
+ "earn": 10326,
+ "channel": 10327,
+ ";Ċ": 10342,
+ "World": 10343,
+ "Ġpython": 10344,
+ "Ġlif": 10345,
+ "Ġtrav": 10346,
+ "Ġconven": 10347,
+ "company": 10348,
+ "ĠClub": 10349,
+ "138": 10350,
+ "Ver": 10351,
+ "Btn": 10352,
+ "Ġzone": 10353,
+ "products": 10354,
+ "ĠEduc": 10355,
+ "Ġverify": 10356,
+ "ĠMil": 10357,
+ "ono": 10358,
+ "]);ĊĊ": 10359,
+ "ENCE": 10360,
+ "Ġpacket": 10361,
+ "Ġcer": 10362,
+ "Ġenumer": 10363,
+ "Ġpars": 10364,
+ "formed": 10365,
+ "Ġoccup": 10366,
+ "tre": 10367,
+ "Ġexercise": 10368,
+ "Day": 10369,
+ "_sum": 10370,
+ "Ġasking": 10371,
+ "aption": 10372,
+ "Ġorders": 10373,
+ "Ġspending": 10374,
+ "ĠERR": 10375,
+ ".Dis": 10376,
+ "ĠUtil": 10377,
+ "âĢľI": 10378,
+ "\\'": 10379,
+ "?)": 10380,
+ "/>Ċ": 10381,
+ "Ġemot": 10382,
+ "Ġinfluence": 10383,
+ "ĠAfrica": 10384,
+ "atters": 10385,
+ "Ùħ": 10386,
+ ".session": 10387,
+ "Ġchief": 10388,
+ "ĉĉĉĉĉĉĉĉĉĉĉ": 10389,
+ "Ġtom": 10390,
+ "cluded": 10391,
+ "serial": 10392,
+ "_handler": 10393,
+ ".Type": 10394,
+ "aped": 10395,
+ "Ġpolicies": 10396,
+ "-ex": 10397,
+ "-tr": 10398,
+ "blank": 10399,
+ "merce": 10400,
+ "Ġcoverage": 10401,
+ "Ġrc": 10402,
+ "_matrix": 10403,
+ "_box": 10404,
+ "Ġcharges": 10405,
+ "ĠBoston": 10406,
+ "Pe": 10407,
+ "Ġcircum": 10408,
+ "Ġfilled": 10409,
+ "148": 10410,
+ "Ġnorth": 10411,
+ "ictureBox": 10412,
+ "ĉres": 10413,
+ "è®": 10414,
+ "Ġtermin": 10415,
+ "Ġ[â̦": 10416,
+ "IRECT": 10417,
+ "Ġber": 10418,
+ "Ġ\"../../": 10419,
+ "retch": 10420,
+ ".code": 10421,
+ "_col": 10422,
+ "ĠGovernment": 10423,
+ "Ġargv": 10424,
+ "ĠLord": 10425,
+ "asi": 10426,
+ "Exec": 10427,
+ "ĉlet": 10428,
+ "vertis": 10429,
+ "Ġdiscussion": 10430,
+ "enance": 10431,
+ "outube": 10432,
+ "typeof": 10433,
+ "Ġserved": 10434,
+ "ĠPut": 10435,
+ "ĉx": 10436,
+ "Ġsweet": 10437,
+ "Before": 10438,
+ "ategy": 10439,
+ ".of": 10440,
+ "ĠMaterial": 10441,
+ "Sort": 10442,
+ "ONT": 10443,
+ "igital": 10444,
+ "Why": 10445,
+ "Ġsust": 10446,
+ "Ġç": 10447,
+ "abet": 10448,
+ "Ġsegment": 10449,
+ "Ġ[],Ċ": 10450,
+ "ĠMuslim": 10451,
+ "ĠfindViewById": 10452,
+ "cut": 10453,
+ "_TEXT": 10454,
+ "ĠMary": 10455,
+ "Ġloved": 10456,
+ "Ġlie": 10457,
+ "ĠJO": 10458,
+ "Ġisset": 10459,
+ "month": 10460,
+ "Ġprime": 10461,
+ "ti": 10462,
+ "ĠCarol": 10463,
+ "Use": 10464,
+ "146": 10465,
+ "ĠPop": 10466,
+ "ĠSave": 10467,
+ "Interval": 10468,
+ "execute": 10469,
+ "dy": 10470,
+ "ĠIran": 10471,
+ "_cont": 10472,
+ "ĉT": 10473,
+ "Ġphase": 10474,
+ "checkbox": 10475,
+ "week": 10476,
+ "Ġhide": 10477,
+ "Ġtil": 10478,
+ "Ġju": 10479,
+ "Custom": 10480,
+ "burg": 10481,
+ "/M": 10482,
+ "TON": 10483,
+ "Ġquant": 10484,
+ "Ġrub": 10485,
+ "ixels": 10486,
+ "Ġinstalled": 10487,
+ "Ġdump": 10488,
+ "Ġproperly": 10489,
+ "(List": 10490,
+ "Ġdecide": 10491,
+ "apply": 10492,
+ "Has": 10493,
+ "Ġkeeping": 10494,
+ "Ġcitizens": 10495,
+ "Ġjoint": 10496,
+ "pool": 10497,
+ "Socket": 10498,
+ "_op": 10499,
+ "Ġweapon": 10500,
+ "gnore": 10501,
+ "ĠExec": 10502,
+ "otten": 10503,
+ "ĠMS": 10504,
+ "Ġ(-": 10505,
+ "ĠReview": 10506,
+ "Ġexamples": 10507,
+ "Ġtight": 10508,
+ "!(": 10509,
+ "DP": 10510,
+ "ĠMessageBox": 10511,
+ "Ġphotograph": 10512,
+ "164": 10513,
+ "URI": 10514,
+ "ét": 10515,
+ "low": 10516,
+ "ĠGrand": 10517,
+ ".persistence": 10518,
+ "Ġmaintain": 10519,
+ "Ġnums": 10520,
+ "Ġzip": 10521,
+ "ials": 10522,
+ "ĠGets": 10523,
+ "peg": 10524,
+ "ĠBuffer": 10525,
+ "~~~~": 10526,
+ "rastructure": 10527,
+ "ĠPL": 10528,
+ "uen": 10529,
+ "obby": 10530,
+ "sizeof": 10531,
+ "Ġpic": 10532,
+ "Ġseed": 10533,
+ "Ġexperienced": 10534,
+ "Ġodd": 10535,
+ "Ġkick": 10536,
+ "Ġprocedure": 10537,
+ "avigator": 10538,
+ "-on": 10539,
+ ",j": 10540,
+ "ĠAlthough": 10541,
+ "ĠuserId": 10542,
+ "accept": 10543,
+ "Blue": 10544,
+ "IColor": 10545,
+ "layer": 10546,
+ "available": 10547,
+ "Ġends": 10548,
+ ".table": 10549,
+ "Ġdataset": 10550,
+ "bus": 10551,
+ "Ġexplain": 10552,
+ "(pro": 10553,
+ "ĠCommittee": 10554,
+ "Ġnoted": 10555,
+ "]:Ċ": 10556,
+ "Dim": 10557,
+ "stdio": 10558,
+ "154": 10559,
+ ".\",Ċ": 10560,
+ "_source": 10561,
+ "181": 10562,
+ "ĠWeek": 10563,
+ "ĠEdge": 10564,
+ "Ġoperating": 10565,
+ "Ġeste": 10566,
+ "ipl": 10567,
+ "330": 10568,
+ "agination": 10569,
+ "Ġproceed": 10570,
+ "Ġanimation": 10571,
+ ".Models": 10572,
+ "ĠWatch": 10573,
+ "iat": 10574,
+ "Ġoppon": 10575,
+ "/A": 10576,
+ "Report": 10577,
+ "Ġsounds": 10578,
+ "_buf": 10579,
+ "IELD": 10580,
+ "Ġbund": 10581,
+ "ĉget": 10582,
+ ".pr": 10583,
+ "(tmp": 10584,
+ "Ġkid": 10585,
+ ">ĊĊĊ": 10586,
+ "Ġyang": 10587,
+ "NotFound": 10588,
+ "ÑĨ": 10589,
+ "math": 10590,
+ "@gmail": 10591,
+ "ĠLIMIT": 10592,
+ "redients": 10593,
+ "Ġvent": 10594,
+ "avigate": 10595,
+ "Look": 10596,
+ "Ġreligious": 10597,
+ "Ġrand": 10598,
+ "rio": 10599,
+ "(GL": 10600,
+ "_ip": 10601,
+ "uan": 10602,
+ "iciency": 10603,
+ "ĠChange": 10604,
+ ">čĊčĊ": 10605,
+ "ĠEntity": 10606,
+ "Ġrencontre": 10607,
+ "ĠRet": 10608,
+ "plan": 10609,
+ "én": 10610,
+ "BOOL": 10611,
+ "uries": 10612,
+ "train": 10613,
+ "Definition": 10614,
+ "============": 10615,
+ "zz": 10616,
+ "450": 10617,
+ "Animation": 10618,
+ "ĠOK": 10619,
+ "_menu": 10620,
+ ".bl": 10621,
+ "_score": 10622,
+ "Ġacad": 10623,
+ "(System": 10624,
+ "Ġrefresh": 10625,
+ "'=>$": 10626,
+ ".Graphics": 10627,
+ "amento": 10628,
+ "pid": 10629,
+ "tc": 10630,
+ "Ġtips": 10631,
+ "Ġhomes": 10632,
+ "Ġfuel": 10633,
+ "âĸ": 10634,
+ "_helper": 10635,
+ "ĠĠčĊ": 10636,
+ "ĠRoom": 10637,
+ ".Close": 10638,
+ "_attr": 10639,
+ "ĠMount": 10640,
+ "ĠEv": 10641,
+ "arser": 10642,
+ "_top": 10643,
+ "eah": 10644,
+ "ĠDelete": 10645,
+ "ãĢį": 10646,
+ "uke": 10647,
+ "Ġusage": 10648,
+ "aria": 10649,
+ "_dev": 10650,
+ "Ġtexture": 10651,
+ "Ġconversation": 10652,
+ "eper": 10653,
+ "Bean": 10654,
+ "done": 10655,
+ "nonatomic": 10656,
+ "ĠSecond": 10657,
+ "Ġshooting": 10658,
+ "_pre": 10659,
+ "Components": 10660,
+ "Ġ]ĊĊ": 10661,
+ "__,": 10662,
+ "stitution": 10663,
+ ".Char": 10664,
+ ">();ĊĊ": 10665,
+ "Ġpresented": 10666,
+ "Ġwa": 10667,
+ "oker": 10668,
+ "-ĊĊ": 10669,
+ "iner": 10670,
+ "Ġbecoming": 10671,
+ "Ġincident": 10672,
+ "Att": 10673,
+ "162": 10674,
+ "Ġrevealed": 10675,
+ "forc": 10676,
+ "Ġboot": 10677,
+ ".page": 10678,
+ "Enumerator": 10679,
+ "165": 10680,
+ "_->": 10681,
+ "Photo": 10682,
+ "Ġspring": 10683,
+ ".\",": 10684,
+ "ĠDictionary": 10685,
+ "BJECT": 10686,
+ "Ġlocations": 10687,
+ "Ġsamples": 10688,
+ "InputStream": 10689,
+ "ĠBrown": 10690,
+ "Ġstats": 10691,
+ "quality": 10692,
+ "Ñħ": 10693,
+ "-dis": 10694,
+ "Ġhelping": 10695,
+ "Ġped": 10696,
+ "224": 10697,
+ "(se": 10698,
+ "ĠWho": 10699,
+ "alian": 10700,
+ "internal": 10701,
+ "Ġft": 10702,
+ ">().": 10703,
+ "->{": 10704,
+ "Ġmine": 10705,
+ "Ġsector": 10706,
+ "Ġgro": 10707,
+ "Ġopportunities": 10708,
+ "Ġü": 10709,
+ "Ġmp": 10710,
+ "Ġalleged": 10711,
+ "Ġdoubt": 10712,
+ "Mouse": 10713,
+ "About": 10714,
+ "_part": 10715,
+ "Ġchair": 10716,
+ "Ġstopped": 10717,
+ "161": 10718,
+ "loop": 10719,
+ "entities": 10720,
+ "Ġapps": 10721,
+ "ansion": 10722,
+ "Ġmental": 10723,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10724,
+ "FR": 10725,
+ "Ġdefend": 10726,
+ "care": 10727,
+ "Ġideal": 10728,
+ "/api": 10729,
+ "urface": 10730,
+ "011": 10731,
+ "Ġele": 10732,
+ "ulator": 10733,
+ "ĠRights": 10734,
+ "anguages": 10735,
+ "Ġfunds": 10736,
+ "Ġadapt": 10737,
+ "Attributes": 10738,
+ "Ġdeploy": 10739,
+ "opts": 10740,
+ "Ġvalidation": 10741,
+ "Ġconcerns": 10742,
+ "uce": 10743,
+ ".num": 10744,
+ "ulture": 10745,
+ "ila": 10746,
+ "Ġcup": 10747,
+ "Ġpure": 10748,
+ ".Fore": 10749,
+ "183": 10750,
+ "ĠHashMap": 10751,
+ ".valueOf": 10752,
+ "asm": 10753,
+ "MO": 10754,
+ "Ġcs": 10755,
+ "Ġstores": 10756,
+ "Ġ************************************************************************": 10757,
+ "Ġcommunication": 10758,
+ "mem": 10759,
+ ".EventHandler": 10760,
+ ".Status": 10761,
+ "_right": 10762,
+ ".setOn": 10763,
+ "Sheet": 10764,
+ "Ġidentify": 10765,
+ "enerated": 10766,
+ "ordered": 10767,
+ "Ġ\"[": 10768,
+ "Ġswe": 10769,
+ "Condition": 10770,
+ "ĠAccording": 10771,
+ "Ġprepare": 10772,
+ "Ġrob": 10773,
+ "Pool": 10774,
+ "Ġsport": 10775,
+ "rv": 10776,
+ "ĠRouter": 10777,
+ "Ġalternative": 10778,
+ "([]": 10779,
+ "ĠChicago": 10780,
+ "ipher": 10781,
+ "ische": 10782,
+ "ĠDirector": 10783,
+ "kl": 10784,
+ "ĠWil": 10785,
+ "keys": 10786,
+ "Ġmysql": 10787,
+ "Ġwelcome": 10788,
+ "king": 10789,
+ "ĠManager": 10790,
+ "Ġcaught": 10791,
+ ")}Ċ": 10792,
+ "Score": 10793,
+ "_PR": 10794,
+ "Ġsurvey": 10795,
+ "hab": 10796,
+ "Headers": 10797,
+ "ADER": 10798,
+ "Ġdecor": 10799,
+ "Ġturns": 10800,
+ "Ġradius": 10801,
+ "errupt": 10802,
+ "Cor": 10803,
+ "Ġmel": 10804,
+ "Ġintr": 10805,
+ "(q": 10806,
+ "ĠAC": 10807,
+ "amos": 10808,
+ "MAX": 10809,
+ "ĠGrid": 10810,
+ "ĠJesus": 10811,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10812,
+ ".DE": 10813,
+ "Ġts": 10814,
+ "Ġlinked": 10815,
+ "free": 10816,
+ "ĠQt": 10817,
+ "Ġ/**čĊ": 10818,
+ "Ġfaster": 10819,
+ "ctr": 10820,
+ "_J": 10821,
+ "DT": 10822,
+ ".Check": 10823,
+ "Ġcombination": 10824,
+ "Ġintended": 10825,
+ "-the": 10826,
+ "-type": 10827,
+ "182": 10828,
+ "ectors": 10829,
+ "ami": 10830,
+ "uting": 10831,
+ "Ġuma": 10832,
+ "XML": 10833,
+ "UCT": 10834,
+ "Ap": 10835,
+ "ĠRandom": 10836,
+ "Ġran": 10837,
+ ".sort": 10838,
+ "Ġsorted": 10839,
+ ".Un": 10840,
+ "401": 10841,
+ "_PER": 10842,
+ "itory": 10843,
+ "Ġpriority": 10844,
+ "ĠGal": 10845,
+ "ĠOld": 10846,
+ "hot": 10847,
+ "ĠDisplay": 10848,
+ "(sub": 10849,
+ "_TH": 10850,
+ "_Y": 10851,
+ "ĠCare": 10852,
+ "loading": 10853,
+ "Kind": 10854,
+ "_handle": 10855,
+ ",,": 10856,
+ "rase": 10857,
+ "_replace": 10858,
+ ".addEventListener": 10859,
+ "ĠRT": 10860,
+ "172": 10861,
+ "Ġentered": 10862,
+ "gers": 10863,
+ "Ġich": 10864,
+ "(start": 10865,
+ "205": 10866,
+ "/app": 10867,
+ "Ġbrother": 10868,
+ "Memory": 10869,
+ "Outlet": 10870,
+ "Ġutf": 10871,
+ "prec": 10872,
+ "Ġnavigation": 10873,
+ "ORK": 10874,
+ "Ġdst": 10875,
+ "Detail": 10876,
+ "Ġaudience": 10877,
+ "Ġdur": 10878,
+ "Ġcluster": 10879,
+ "unched": 10880,
+ "Ġ],": 10881,
+ "Ġcomfortable": 10882,
+ ".values": 10883,
+ "ĠTotal": 10884,
+ "Ġsnap": 10885,
+ "Ġstandards": 10886,
+ "Ġperformed": 10887,
+ "hand": 10888,
+ "(\"@": 10889,
+ "åŃ": 10890,
+ "Ġphil": 10891,
+ "ibr": 10892,
+ "trim": 10893,
+ "Ġforget": 10894,
+ "157": 10895,
+ "Ġdoctor": 10896,
+ ".TextBox": 10897,
+ "377": 10898,
+ "icons": 10899,
+ ",s": 10900,
+ "ĠOp": 10901,
+ "Sm": 10902,
+ "Stop": 10903,
+ "ĉList": 10904,
+ "ĉu": 10905,
+ "Comment": 10906,
+ "_VERSION": 10907,
+ ".Xtra": 10908,
+ "Person": 10909,
+ "rb": 10910,
+ "LOB": 10911,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 10912,
+ "ĠCentral": 10913,
+ "270": 10914,
+ "ICK": 10915,
+ "raq": 10916,
+ "Ġputting": 10917,
+ "Ġmd": 10918,
+ "ĠLove": 10919,
+ "Program": 10920,
+ "Border": 10921,
+ "oor": 10922,
+ "Ġallowing": 10923,
+ "after": 10924,
+ "Ġentries": 10925,
+ "ĠMaybe": 10926,
+ "]).": 10927,
+ "ĠShort": 10928,
+ ")\\": 10929,
+ ".now": 10930,
+ "friend": 10931,
+ "Ġprefer": 10932,
+ "ĠGPIO": 10933,
+ "osis": 10934,
+ "ĠGameObject": 10935,
+ "Ġskip": 10936,
+ "Ġcompetition": 10937,
+ "_match": 10938,
+ "lications": 10939,
+ "_CONT": 10940,
+ ".groupBox": 10941,
+ "Ġals": 10942,
+ "666": 10943,
+ "\"We": 10944,
+ "_eq": 10945,
+ "lan": 10946,
+ "_search": 10947,
+ "ĠMusic": 10948,
+ "asis": 10949,
+ "Ġbind": 10950,
+ "ĠIsland": 10951,
+ "rum": 10952,
+ "(E": 10953,
+ "Ġseat": 10954,
+ "Video": 10955,
+ "Ġack": 10956,
+ "reek": 10957,
+ "={()": 10958,
+ "Ġrating": 10959,
+ "Ġrestaurant": 10960,
+ "456": 10961,
+ "DEX": 10962,
+ "(buf": 10963,
+ "pping": 10964,
+ "uality": 10965,
+ "Ġleague": 10966,
+ "176": 10967,
+ "Ġfocused": 10968,
+ "apon": 10969,
+ "$data": 10970,
+ "CLUD": 10971,
+ "CLUDING": 10972,
+ "Ġabsolute": 10973,
+ "(query": 10974,
+ "Ġtells": 10975,
+ "Ang": 10976,
+ "Ġcommunities": 10977,
+ "Ġhonest": 10978,
+ "oking": 10979,
+ "Ġapart": 10980,
+ "arity": 10981,
+ "/$": 10982,
+ "_module": 10983,
+ "ĠEnc": 10984,
+ ".an": 10985,
+ ".Config": 10986,
+ "Cre": 10987,
+ "Ġshock": 10988,
+ "ĠArab": 10989,
+ "IENT": 10990,
+ "/re": 10991,
+ "Ġretrie": 10992,
+ "ycler": 10993,
+ "isa": 10994,
+ "ĠOrgan": 10995,
+ ".graph": 10996,
+ "Ġí": 10997,
+ "ĠBAS": 10998,
+ "Enum": 10999,
+ "Ġpossibly": 11000,
+ "ÑĢаÐ": 11001,
+ "ĠJapanese": 11002,
+ "Ġcraft": 11003,
+ "ĠPlace": 11004,
+ "Ġtalent": 11005,
+ "Ġfunding": 11006,
+ "Ġconfirmed": 11007,
+ "Ġcycle": 11008,
+ "/x": 11009,
+ "GE": 11010,
+ "Ġhearing": 11011,
+ "Ġplants": 11012,
+ "Ġmouth": 11013,
+ "pages": 11014,
+ "oria": 11015,
+ "ĠRemove": 11016,
+ "_total": 11017,
+ "Ġod": 11018,
+ "ollapse": 11019,
+ "door": 11020,
+ "Ġbought": 11021,
+ "Ġaddr": 11022,
+ "ARCH": 11023,
+ "_dim": 11024,
+ "dden": 11025,
+ "Ġdecades": 11026,
+ "REQUEST": 11027,
+ "Ġversions": 11028,
+ "fire": 11029,
+ "006": 11030,
+ "Ġmoves": 11031,
+ "fb": 11032,
+ "Ġcoffee": 11033,
+ ".connect": 11034,
+ "ĠRow": 11035,
+ "Ġschema": 11036,
+ "Scope": 11037,
+ "-Type": 11038,
+ "Ġfighting": 11039,
+ "Ġretail": 11040,
+ "Ġmodified": 11041,
+ "TF": 11042,
+ "Files": 11043,
+ "nie": 11044,
+ "_command": 11045,
+ "stone": 11046,
+ "ĠÑĤ": 11047,
+ "_thread": 11048,
+ "Ġbond": 11049,
+ "ĠDevelopment": 11050,
+ "Ġpt": 11051,
+ "FORM": 11052,
+ "plet": 11053,
+ "Ġidentified": 11054,
+ "cpp": 11055,
+ "206": 11056,
+ "225": 11057,
+ "Ġcoding": 11058,
+ "oked": 11059,
+ "ĠMaster": 11060,
+ "IDTH": 11061,
+ "Ġresidents": 11062,
+ "redit": 11063,
+ "ĠPhoto": 11064,
+ "=-": 11065,
+ "unte": 11066,
+ "ateur": 11067,
+ "159": 11068,
+ "_STATE": 11069,
+ "ĠSing": 11070,
+ "Ġsheet": 11071,
+ ".val": 11072,
+ "orse": 11073,
+ "Ġhers": 11074,
+ "Ġdetermined": 11075,
+ "Common": 11076,
+ "Ġwed": 11077,
+ "_queue": 11078,
+ "PH": 11079,
+ "ĠAtl": 11080,
+ "cred": 11081,
+ "/LICENSE": 11082,
+ "Ġmes": 11083,
+ "Ġadvanced": 11084,
+ ".java": 11085,
+ ".Sh": 11086,
+ "Go": 11087,
+ "kill": 11088,
+ "fp": 11089,
+ "_settings": 11090,
+ "Ġpal": 11091,
+ "Ġtruck": 11092,
+ "Ġcombined": 11093,
+ "Ġ\"${": 11094,
+ "ĠCorpor": 11095,
+ "Ġjoined": 11096,
+ "ĠJose": 11097,
+ "ĠCup": 11098,
+ "uns": 11099,
+ "estival": 11100,
+ "levision": 11101,
+ "Ġbroken": 11102,
+ "Ġmarriage": 11103,
+ "ĠWestern": 11104,
+ "Ġrepresents": 11105,
+ "ĠTitle": 11106,
+ "Ġss": 11107,
+ ".Ass": 11108,
+ "ongoose": 11109,
+ "iento": 11110,
+ "<>();Ċ": 11111,
+ "Ġabsolutely": 11112,
+ "Ġsmooth": 11113,
+ "TERN": 11114,
+ "ĠUnless": 11115,
+ "Word": 11116,
+ "Ġmerge": 11117,
+ "igan": 11118,
+ "ĠVol": 11119,
+ "Ġnn": 11120,
+ ".getId": 11121,
+ "Ġз": 11122,
+ "171": 11123,
+ "Ġsexy": 11124,
+ "Ġseeking": 11125,
+ "Single": 11126,
+ ".this": 11127,
+ "179": 11128,
+ "Ġkom": 11129,
+ "bound": 11130,
+ ";\"": 11131,
+ "ĠfontSize": 11132,
+ "_df": 11133,
+ "Ġinjury": 11134,
+ "(H": 11135,
+ "Ġissued": 11136,
+ "_END": 11137,
+ ":self": 11138,
+ "020": 11139,
+ "Ġpatch": 11140,
+ "Ġleaves": 11141,
+ "Ġadopt": 11142,
+ "FileName": 11143,
+ "ãĢIJ": 11144,
+ "Ġexecutive": 11145,
+ "ĠByte": 11146,
+ "]))Ċ": 11147,
+ "Ġnu": 11148,
+ "outing": 11149,
+ "cluding": 11150,
+ "-R": 11151,
+ ".options": 11152,
+ "Ġsubstant": 11153,
+ "avax": 11154,
+ "ĠBUT": 11155,
+ "Ġtechnical": 11156,
+ "Ġtwice": 11157,
+ "Ġmás": 11158,
+ "Ġunivers": 11159,
+ "yr": 11160,
+ "Ġdrag": 11161,
+ "ĠDC": 11162,
+ "Ġsed": 11163,
+ "Ġbot": 11164,
+ "ĠPal": 11165,
+ "ĠHall": 11166,
+ "forcement": 11167,
+ "Ġauch": 11168,
+ ".mod": 11169,
+ "notation": 11170,
+ "_files": 11171,
+ ".line": 11172,
+ "_flag": 11173,
+ "[name": 11174,
+ "Ġresolution": 11175,
+ "Ġbott": 11176,
+ "(\"[": 11177,
+ "ende": 11178,
+ "(arr": 11179,
+ "Free": 11180,
+ "(@\"": 11181,
+ "ĠDistrict": 11182,
+ "PEC": 11183,
+ ":-": 11184,
+ "Picker": 11185,
+ "ĠJo": 11186,
+ "ĠĠĠĠĠĊ": 11187,
+ "ĠRiver": 11188,
+ "_rows": 11189,
+ "Ġhelpful": 11190,
+ "Ġmassive": 11191,
+ "---Ċ": 11192,
+ "Ġmeasures": 11193,
+ "007": 11194,
+ "ĠRuntime": 11195,
+ "Ġworry": 11196,
+ "ĠSpec": 11197,
+ "ĉD": 11198,
+ "ãĢij": 11199,
+ "Ġ){Ċ": 11200,
+ "Ġworse": 11201,
+ "(filename": 11202,
+ "Ġlay": 11203,
+ "Ġmagic": 11204,
+ "ĠTheir": 11205,
+ "oul": 11206,
+ "stroy": 11207,
+ "ĠWhere": 11208,
+ "280": 11209,
+ "Ġsudden": 11210,
+ "Ġdefe": 11211,
+ "Ġbinding": 11212,
+ "Ġflight": 11213,
+ "ĠOnInit": 11214,
+ "ĠWomen": 11215,
+ "ĠPolicy": 11216,
+ "Ġdrugs": 11217,
+ "ishing": 11218,
+ "('../": 11219,
+ "ĠMel": 11220,
+ "peat": 11221,
+ "tor": 11222,
+ "Ġproposed": 11223,
+ "Ġstated": 11224,
+ "_RES": 11225,
+ "Ġeast": 11226,
+ "212": 11227,
+ "ĠCONDITION": 11228,
+ "_desc": 11229,
+ "Ġwinning": 11230,
+ "folio": 11231,
+ "Mapper": 11232,
+ "ĠPan": 11233,
+ "ĠAnge": 11234,
+ ".servlet": 11235,
+ "Ġcopies": 11236,
+ "LM": 11237,
+ "Ġvm": 11238,
+ "åį": 11239,
+ "Ġdictionary": 11240,
+ "Seg": 11241,
+ "177": 11242,
+ "elines": 11243,
+ "ĠSend": 11244,
+ "Ġiron": 11245,
+ "ĠFort": 11246,
+ "166": 11247,
+ ".domain": 11248,
+ "Ġdebate": 11249,
+ "NotNull": 11250,
+ "eq": 11251,
+ "acher": 11252,
+ "lf": 11253,
+ "ĉfmt": 11254,
+ "Ġlawy": 11255,
+ "178": 11256,
+ "ÄŁ": 11257,
+ "ĠMen": 11258,
+ "Ġtrim": 11259,
+ "(NULL": 11260,
+ "Ġ!!": 11261,
+ "Ġpad": 11262,
+ "Ġfollows": 11263,
+ "\"][\"": 11264,
+ "requ": 11265,
+ "ĠEp": 11266,
+ ".github": 11267,
+ "(img": 11268,
+ "eto": 11269,
+ "('\\": 11270,
+ "Services": 11271,
+ "umbnail": 11272,
+ "_main": 11273,
+ "pleted": 11274,
+ "fortunately": 11275,
+ "Ġwindows": 11276,
+ "Ġplane": 11277,
+ "ĠConnection": 11278,
+ ".local": 11279,
+ "uard": 11280,
+ "}\\": 11281,
+ "==\"": 11282,
+ "andon": 11283,
+ "ĠRoy": 11284,
+ "west": 11285,
+ "158": 11286,
+ "iginal": 11287,
+ "emies": 11288,
+ "itz": 11289,
+ "'):Ċ": 11290,
+ "ĠPeter": 11291,
+ "Ġtough": 11292,
+ "Ġreduced": 11293,
+ "Ġcalculate": 11294,
+ "Ġrapid": 11295,
+ "customer": 11296,
+ "Ġefficient": 11297,
+ "Ġmedium": 11298,
+ "Ġfell": 11299,
+ ".ref": 11300,
+ "ĠCas": 11301,
+ "Ġfeedback": 11302,
+ "Speed": 11303,
+ "(output": 11304,
+ "aje": 11305,
+ "Ġcategories": 11306,
+ "Ġfee": 11307,
+ "};": 11308,
+ "Ġdeleted": 11309,
+ "reh": 11310,
+ "Ġproof": 11311,
+ "Desc": 11312,
+ "Build": 11313,
+ "Ġsides": 11314,
+ ".ArrayList": 11315,
+ "-%": 11316,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 11317,
+ "ر": 11318,
+ ".match": 11319,
+ "ли": 11320,
+ "Ġfeels": 11321,
+ "Ġachieve": 11322,
+ "Ġclim": 11323,
+ "_ON": 11324,
+ "ĠCD": 11325,
+ "Ġteacher": 11326,
+ "_current": 11327,
+ "bn": 11328,
+ "_PL": 11329,
+ "isting": 11330,
+ "Enable": 11331,
+ "GEN": 11332,
+ "Ġtv": 11333,
+ "Ġsock": 11334,
+ "Ġplays": 11335,
+ "Ġdiscount": 11336,
+ "ĠKE": 11337,
+ "ĠDebug": 11338,
+ "Fore": 11339,
+ "ĠIraq": 11340,
+ "Ġappearance": 11341,
+ "Mon": 11342,
+ "Ġstyled": 11343,
+ "ĠHuman": 11344,
+ "iot": 11345,
+ "ĠHistory": 11346,
+ "Ġsac": 11347,
+ "ĠCollection": 11348,
+ "Ġrecommended": 11349,
+ ".Selected": 11350,
+ "Ġorganizations": 11351,
+ "Ġdiscovered": 11352,
+ "cohol": 11353,
+ "adas": 11354,
+ "ĠThomas": 11355,
+ "May": 11356,
+ "Ġconserv": 11357,
+ "Ġdomin": 11358,
+ "ĠFollow": 11359,
+ "ĠSection": 11360,
+ "ĠThanks": 11361,
+ "Username": 11362,
+ "Ġrecipe": 11363,
+ "Ġwonderful": 11364,
+ ".sleep": 11365,
+ "_if": 11366,
+ "ĉĊĉĊ": 11367,
+ "orno": 11368,
+ "Ġru": 11369,
+ "_target": 11370,
+ ".\"\"": 11371,
+ "à¦": 11372,
+ "EventArgs": 11373,
+ "Ġinputs": 11374,
+ "Ġfif": 11375,
+ "Ġvision": 11376,
+ "cy": 11377,
+ "ĠSeries": 11378,
+ ")(((": 11379,
+ "Ġtrading": 11380,
+ "Ġmarker": 11381,
+ "Begin": 11382,
+ "Ġtypically": 11383,
+ "Ġcauses": 11384,
+ "dropdown": 11385,
+ "_DEBUG": 11386,
+ "260": 11387,
+ "Ġdetect": 11388,
+ "country": 11389,
+ "!\");Ċ": 11390,
+ "ĉR": 11391,
+ "appy": 11392,
+ "Ġcref": 11393,
+ "('<": 11394,
+ "\"=>": 11395,
+ "ĠLE": 11396,
+ "reader": 11397,
+ "Ġadministr": 11398,
+ "õ": 11399,
+ "ucket": 11400,
+ "Ġfashion": 11401,
+ ".char": 11402,
+ "izar": 11403,
+ "Ġdisable": 11404,
+ "Ġsuc": 11405,
+ "ĠLive": 11406,
+ "issue": 11407,
+ "Ġmetadata": 11408,
+ "flags": 11409,
+ "ĠðŁ": 11410,
+ "Ġcommitted": 11411,
+ "Ġva": 11412,
+ "Ġrough": 11413,
+ "Ġ'''Ċ": 11414,
+ "Ġhighlight": 11415,
+ "_vars": 11416,
+ "VO": 11417,
+ "Ġencoding": 11418,
+ "-Z": 11419,
+ "_sign": 11420,
+ "$(\"#": 11421,
+ "Ġrain": 11422,
+ "reatest": 11423,
+ "ĠEND": 11424,
+ "Selection": 11425,
+ "Ġcandidates": 11426,
+ "Ġsav": 11427,
+ ".Empty": 11428,
+ "Ġdecisions": 11429,
+ "Ġcollabor": 11430,
+ "ridge": 11431,
+ "feed": 11432,
+ "ression": 11433,
+ "Ġpersons": 11434,
+ "VM": 11435,
+ "008": 11436,
+ "ega": 11437,
+ "_BIT": 11438,
+ "According": 11439,
+ "acked": 11440,
+ "Ġdollars": 11441,
+ "_loss": 11442,
+ "ĠCost": 11443,
+ "}\"Ċ": 11444,
+ "Notification": 11445,
+ "Ġprostit": 11446,
+ "Ġauthority": 11447,
+ ".rec": 11448,
+ "Ġspokes": 11449,
+ "ĠToday": 11450,
+ "istant": 11451,
+ "ĠHead": 11452,
+ "âĢĿ.": 11453,
+ "ertainment": 11454,
+ "cean": 11455,
+ "culate": 11456,
+ "Ġven": 11457,
+ "However": 11458,
+ "_arr": 11459,
+ "Ġtokens": 11460,
+ "Graph": 11461,
+ "ĠJud": 11462,
+ "ĠVirgin": 11463,
+ "ĠSerial": 11464,
+ "unning": 11465,
+ "Mutable": 11466,
+ "agers": 11467,
+ ".csv": 11468,
+ "Ġdeveloping": 11469,
+ "Ġinstructions": 11470,
+ "Ġpromise": 11471,
+ "Ġrequested": 11472,
+ "_encode": 11473,
+ "/\"": 11474,
+ "ĠIcon": 11475,
+ "uilt": 11476,
+ "-day": 11477,
+ "Ġintelligence": 11478,
+ ".IS": 11479,
+ "ĠObservable": 11480,
+ "ĠHard": 11481,
+ "Bool": 11482,
+ "211": 11483,
+ "idential": 11484,
+ ".Anchor": 11485,
+ "Ġselling": 11486,
+ "CI": 11487,
+ "AGES": 11488,
+ "tle": 11489,
+ "bur": 11490,
+ "UFFER": 11491,
+ "RY": 11492,
+ "Ġbigger": 11493,
+ "Ġrat": 11494,
+ "Ġfamous": 11495,
+ "Ġtypename": 11496,
+ "Ġexplained": 11497,
+ "}}Ċ": 11498,
+ "Ġnuclear": 11499,
+ "-N": 11500,
+ "Ġcrisis": 11501,
+ "ĠEnter": 11502,
+ "Ġanswers": 11503,
+ "/${": 11504,
+ "/pl": 11505,
+ "Ġsequ": 11506,
+ "_next": 11507,
+ "mask": 11508,
+ "Ġstanding": 11509,
+ "Ġplenty": 11510,
+ "ĠCross": 11511,
+ "ĉret": 11512,
+ "dro": 11513,
+ "ĠCast": 11514,
+ "167": 11515,
+ "=true": 11516,
+ "ĠChris": 11517,
+ "icio": 11518,
+ "ĠMike": 11519,
+ "Decimal": 11520,
+ "addComponent": 11521,
+ "Len": 11522,
+ "Ġcock": 11523,
+ "Ġ#{": 11524,
+ "URN": 11525,
+ "": 11657,
+ "Ġ*=": 11658,
+ "ĠPS": 11659,
+ "Ġdangerous": 11660,
+ "[p": 11661,
+ "OME": 11662,
+ "Other": 11663,
+ "ĠStringBuilder": 11664,
+ "Points": 11665,
+ "heading": 11666,
+ "Ġcurrency": 11667,
+ "Ġpercentage": 11668,
+ "_API": 11669,
+ "Ġclassic": 11670,
+ "thead": 11671,
+ "ĠMO": 11672,
+ "FE": 11673,
+ "Idx": 11674,
+ "await": 11675,
+ "Ġè": 11676,
+ "Ġaccident": 11677,
+ "Ġvariant": 11678,
+ "Ġmyst": 11679,
+ "ĠLand": 11680,
+ "ĠBre": 11681,
+ "Ġharm": 11682,
+ "ĠAcc": 11683,
+ "Ġcharged": 11684,
+ "iones": 11685,
+ "Visibility": 11686,
+ "arry": 11687,
+ "ĠLanguage": 11688,
+ "Ġwalking": 11689,
+ "\".ĊĊ": 11690,
+ "ifer": 11691,
+ "Ġleadership": 11692,
+ ".From": 11693,
+ "ynam": 11694,
+ "Ġtimestamp": 11695,
+ "ipt": 11696,
+ "ĠHas": 11697,
+ "REFER": 11698,
+ "ĠIts": 11699,
+ "Ġlistener": 11700,
+ "UTE": 11701,
+ "213": 11702,
+ "_description": 11703,
+ "Ġexperiences": 11704,
+ "Ġcreates": 11705,
+ "RS": 11706,
+ "cart": 11707,
+ "black": 11708,
+ "Ġchoices": 11709,
+ "war": 11710,
+ "750": 11711,
+ "Ġ'''": 11712,
+ "Ġordered": 11713,
+ "Ġevening": 11714,
+ "Ġpil": 11715,
+ "Ġtun": 11716,
+ "ĠBad": 11717,
+ "(app": 11718,
+ "random": 11719,
+ "Ġexplicit": 11720,
+ "Ġarrived": 11721,
+ "Ġfly": 11722,
+ "Ġeconom": 11723,
+ "-mail": 11724,
+ "Ġlists": 11725,
+ "Ġarchitect": 11726,
+ "234": 11727,
+ "ĠPay": 11728,
+ "Ġds": 11729,
+ "ĠSol": 11730,
+ "Ġvehicles": 11731,
+ "Hz": 11732,
+ "-com": 11733,
+ "Ġking": 11734,
+ "_equal": 11735,
+ "ĠHelp": 11736,
+ "Ġabuse": 11737,
+ "480": 11738,
+ "169": 11739,
+ "--;Ċ": 11740,
+ "Ġextr": 11741,
+ "Ġchemical": 11742,
+ "ä¿": 11743,
+ "Ġorient": 11744,
+ "Ġbreath": 11745,
+ "ĠSpace": 11746,
+ "(element": 11747,
+ "wait": 11748,
+ "DED": 11749,
+ "igma": 11750,
+ "Ġentr": 11751,
+ "Ġsob": 11752,
+ "-name": 11753,
+ "Ġaffected": 11754,
+ "ika": 11755,
+ "Ġcoal": 11756,
+ "_work": 11757,
+ "Ġhundreds": 11758,
+ "Ġpolitics": 11759,
+ "subject": 11760,
+ "Ġconsumer": 11761,
+ "ANGE": 11762,
+ "Ġrepeated": 11763,
+ "Send": 11764,
+ "Ġ#[": 11765,
+ "Ġprotocol": 11766,
+ "Ġleads": 11767,
+ "useum": 11768,
+ "Every": 11769,
+ "808": 11770,
+ "174": 11771,
+ "Import": 11772,
+ "(count": 11773,
+ "Ġchallenges": 11774,
+ "Ġnovel": 11775,
+ "Ġdepart": 11776,
+ "bits": 11777,
+ ".Current": 11778,
+ "Ġ`${": 11779,
+ "oting": 11780,
+ "(\\": 11781,
+ "Ġcreative": 11782,
+ "Ġbuff": 11783,
+ "Ġintroduced": 11784,
+ "usic": 11785,
+ "modules": 11786,
+ "Are": 11787,
+ "-doc": 11788,
+ "language": 11789,
+ "_cache": 11790,
+ "Ġtod": 11791,
+ "?>": 11792,
+ "omething": 11793,
+ "Ġhun": 11794,
+ "åº": 11795,
+ "aters": 11796,
+ "Intent": 11797,
+ "Ġimplemented": 11798,
+ "ĠCase": 11799,
+ "Children": 11800,
+ "Ġnotification": 11801,
+ "Renderer": 11802,
+ "Wrapper": 11803,
+ "Objects": 11804,
+ "tl": 11805,
+ ".Contains": 11806,
+ "Plugin": 11807,
+ ".row": 11808,
+ "Ġforg": 11809,
+ "Ġpermit": 11810,
+ "Ġtargets": 11811,
+ "ĠIF": 11812,
+ "Ġtip": 11813,
+ "sex": 11814,
+ "Ġsupports": 11815,
+ "Ġfold": 11816,
+ "photo": 11817,
+ "},čĊ": 11818,
+ "Ġgoogle": 11819,
+ "$('#": 11820,
+ "Ġsharing": 11821,
+ "Ġgoods": 11822,
+ "vs": 11823,
+ "ĠDan": 11824,
+ "Rate": 11825,
+ "ĠMartin": 11826,
+ "Ġmanner": 11827,
+ "lie": 11828,
+ ".The": 11829,
+ "Internal": 11830,
+ "ĠCONTR": 11831,
+ "Mock": 11832,
+ "RIGHT": 11833,
+ "Ġ'{": 11834,
+ "Ġcontrols": 11835,
+ "Mat": 11836,
+ "Ġmand": 11837,
+ "Ġextended": 11838,
+ "Ok": 11839,
+ "Ġembed": 11840,
+ "Ġplanet": 11841,
+ "ĠNon": 11842,
+ "-ch": 11843,
+ ")\",": 11844,
+ "epar": 11845,
+ "Ġbelieved": 11846,
+ "ĠEnvironment": 11847,
+ "ĠFriend": 11848,
+ "-res": 11849,
+ "Ġhandling": 11850,
+ "nic": 11851,
+ "-level": 11852,
+ "scri": 11853,
+ "Xml": 11854,
+ "BE": 11855,
+ "ungen": 11856,
+ "Ġalter": 11857,
+ "[idx": 11858,
+ "Pop": 11859,
+ "cam": 11860,
+ "Ġ(((": 11861,
+ "Ġshipping": 11862,
+ "Ġbattery": 11863,
+ "iddleware": 11864,
+ "MC": 11865,
+ "Ġimpl": 11866,
+ "otation": 11867,
+ "ĠLab": 11868,
+ "