From 5769b39f05a227482cc368beaa5850868a29490a Mon Sep 17 00:00:00 2001 From: luboslenco Date: Wed, 12 Nov 2025 19:59:27 +0100 Subject: [PATCH] Neural nodes progress --- paint/sources/box_preferences.ts | 95 +++++++++++++++++-- paint/sources/config.ts | 6 +- paint/sources/enums.ts | 16 ++++ paint/sources/neural_nodes/edit_image_node.ts | 17 +++- .../neural_nodes/image_to_depth_node.ts | 4 +- .../neural_nodes/image_to_normal_map_node.ts | 4 +- .../sources/neural_nodes/image_to_pbr_node.ts | 10 +- .../neural_nodes/inpaint_image_node.ts | 2 +- .../neural_nodes/text_to_image_node.ts | 46 ++++++--- paint/sources/neural_nodes/tile_image_node.ts | 2 +- .../neural_nodes/upscale_image_node.ts | 4 +- paint/sources/tab_plugins.ts | 2 +- 12 files changed, 165 insertions(+), 43 deletions(-) diff --git a/paint/sources/box_preferences.ts b/paint/sources/box_preferences.ts index aa041f35..b5bec055 100644 --- a/paint/sources/box_preferences.ts +++ b/paint/sources/box_preferences.ts @@ -464,15 +464,6 @@ function box_preferences_usage_tab() { make_material_parse_paint_material(); } ui.enabled = true; - - let h_gpu_inference: ui_handle_t = ui_handle(__ID__); - if (h_gpu_inference.init) { - h_gpu_inference.b = config_raw.gpu_inference; - } - config_raw.gpu_inference = ui_check(h_gpu_inference, tr("GPU Inference")); - if (ui.is_hovered) { - ui_tooltip(tr("Use GPU to accelerate neural node processing")); - } } function box_preferences_pen_tab() { @@ -696,6 +687,87 @@ function box_preferences_keymap_tab() { } } +function box_preferences_neural_tab() { + + ui_text(tr("All processing is done on device")); + + let inference_handle: ui_handle_t = ui_handle(__ID__); + if (inference_handle.init) { + inference_handle.i = config_raw.neural_backend; + } + let inference_combo: string[] = [ "CPU", "Vulkan" ]; + ui_combo(inference_handle, inference_combo, tr("Inference Backend"), true); + if (ui.is_hovered) { + ui_tooltip(tr("Backend for neural node processing")); + } + if (inference_handle.changed) { + config_raw.neural_backend = inference_handle.i; + } + + ui_text(tr("Models")); + + if (ui_panel(ui_handle(__ID__), "Stable Diffusion")) { // (downloaded) + // https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors + ui_text("license: openrail"); + ui_text("nodes: Text to Image"); + if (ui_button(tr("Download (4.0GB)"))) { + } + } + + if (ui_panel(ui_handle(__ID__), "Qwen Image")) { // (downloaded) + ui_text("source: https://huggingface.co/QuantStack/Qwen-Image-GGUF (apache-2.0)"); + ui_text("video ram: GB"); + ui_text("nodes: Text to Image"); + if (ui_button(tr("Download") + " (15.7GB)")) { + // https://huggingface.co/QuantStack/Qwen-Image-GGUF/resolve/main/Qwen_Image-Q4_K_S.gguf + // https://huggingface.co/QuantStack/Qwen-Image-GGUF/resolve/main/VAE/Qwen_Image-VAE.safetensors + // https://huggingface.co/unsloth/Qwen2.5-VL-7B-Instruct-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf + } + // if (ui_button(tr("Remove (15.7GB)"))) { + // } + } + + if (ui_panel(ui_handle(__ID__), "Qwen Image Edit")) { // (downloaded) + ui_text("license: apache-2.0"); + ui_text("nodes: Edit Image"); + if (ui_button(tr("Download (17.0GB)"))) { + // https://huggingface.co/QuantStack/Qwen-Image-Edit-2509-GGUF/resolve/main/Qwen-Image-Edit-2509-Q4_K_S.gguf + // https://huggingface.co/QuantStack/Qwen-Image-GGUF/resolve/main/VAE/Qwen_Image-VAE.safetensors + // https://huggingface.co/unsloth/Qwen2.5-VL-7B-Instruct-GGUF/resolve/main/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf + // https://huggingface.co/unsloth/Qwen2.5-VL-7B-Instruct-GGUF/resolve/main/mmproj-F16.gguf + } + } + + if (ui_panel(ui_handle(__ID__), "Wan")) { // (downloaded) + ui_text("license: apache-2.0"); + ui_text("nodes: Text to Image"); + if (ui_button(tr("Download (19.8GB)"))) { + // https://huggingface.co/QuantStack/Wan2.2-T2V-A14B-GGUF/resolve/main/LowNoise/Wan2.2-T2V-A14B-LowNoise-Q4_K_S.gguf + // https://huggingface.co/QuantStack/Wan2.2-T2V-A14B-GGUF/resolve/main/HighNoise/Wan2.2-T2V-A14B-HighNoise-Q4_K_S.gguf + // https://huggingface.co/QuantStack/Wan2.2-T2V-A14B-GGUF/resolve/main/VAE/Wan2.1_VAE.safetensors + // https://huggingface.co/city96/umt5-xxl-encoder-gguf/resolve/main/umt5-xxl-encoder-Q4_K_S.gguf + } + } + + if (ui_panel(ui_handle(__ID__), "Marigold")) { // (downloaded) + ui_text("license: openrail"); + ui_text("nodes: Image to PBR"); + if (ui_button(tr("Download (9.6GB)"))) { + // https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-depth-v1-1.q8_0.gguf + // https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-normals-v1-1.q8_0.gguf + // https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-iid-appearance-v1-1.q8_0.gguf + } + } + + if (ui_panel(ui_handle(__ID__), "RealESRGAN")) { // (downloaded) + // https://huggingface.co/armory3d/Real-ESRGAN/resolve/main/RealESRGAN_x4plus.pth + ui_text("license: bsd-3-clause"); + ui_text("nodes: Upscale Image"); + if (ui_button(tr("Download (0.06GB)"))) { + } + } +} + function box_preferences_plugins_tab() { ui_begin_sticky(); let row: f32[] = [ 1 / 4, 1 / 4 ]; @@ -828,11 +900,14 @@ function box_preferences_show() { if (ui_tab(box_preferences_htab, tr("Keymap"), true)) { box_preferences_keymap_tab(); } + if (ui_tab(box_preferences_htab, tr("Neural"), true)) { + box_preferences_neural_tab(); + } if (ui_tab(box_preferences_htab, tr("Plugins"), true)) { box_preferences_plugins_tab(); } }, - 620, config_raw.touch_ui ? 510 : 420, + 720, 520, function() { config_save(); }); diff --git a/paint/sources/config.ts b/paint/sources/config.ts index 58784ec6..a94a60aa 100644 --- a/paint/sources/config.ts +++ b/paint/sources/config.ts @@ -95,11 +95,11 @@ function config_save() { json_encode_bool("brush_depth_reject", config_raw.brush_depth_reject); json_encode_bool("brush_angle_reject", config_raw.brush_angle_reject); json_encode_i32("dilate_radius", config_raw.dilate_radius); - json_encode_bool("gpu_inference", config_raw.gpu_inference); json_encode_string("blender", config_raw.blender); json_encode_i32("scene_atlas_res", config_raw.scene_atlas_res); json_encode_bool("grid_snap", config_raw.grid_snap); json_encode_bool("experimental", config_raw.experimental); + json_encode_i32("neural_backend", config_raw.neural_backend); let config_json: string = json_encode_end(); let buffer: buffer_t = sys_string_to_buffer(config_json); @@ -200,12 +200,12 @@ function config_init() { config_raw.brush_live = false; config_raw.show_asset_names = false; config_raw.dilate_radius = 2; - config_raw.gpu_inference = true; config_raw.blender = ""; config_raw.scene_atlas_res = texture_res_t.RES8192; config_raw.pathtrace_mode = pathtrace_mode_t.FAST; config_raw.grid_snap = false; config_raw.experimental = false; + config_raw.neural_backend = neural_backend_t.VULKAN; } else { // Discard old config @@ -471,9 +471,9 @@ type config_t = { brush_depth_reject?: bool; brush_angle_reject?: bool; dilate_radius?: i32; - gpu_inference?: bool; blender?: string; scene_atlas_res?: i32; grid_snap?: bool; experimental?: bool; + neural_backend?: i32; }; diff --git a/paint/sources/enums.ts b/paint/sources/enums.ts index fba944bf..50938463 100644 --- a/paint/sources/enums.ts +++ b/paint/sources/enums.ts @@ -273,3 +273,19 @@ enum layout_size_t { STATUS_H = 5, HEADER = 6, // 0 - hidden, 1 - visible } + +enum neural_backend_t { + CPU = 0, + VULKAN = 1, +} + +enum preference_tab_t { + INTERFACE = 0, + THEME = 1, + USAGE = 2, + PEN = 3, + VIEWPORT = 4, + KEYMAP = 5, + NEURAL = 6, + PLUGINS = 7, +} diff --git a/paint/sources/neural_nodes/edit_image_node.ts b/paint/sources/neural_nodes/edit_image_node.ts index 49d86b2f..5abe7acd 100644 --- a/paint/sources/neural_nodes/edit_image_node.ts +++ b/paint/sources/neural_nodes/edit_image_node.ts @@ -28,10 +28,17 @@ function edit_image_node_button(node_id: i32) { let prompt: string = ui_text_area(ui_handle(__ID__), ui_align_t.LEFT, true, tr("prompt"), true); node.buttons[0].height = string_split(prompt, "\n").length + 2; + // let found: bool = false; + let found: bool = true; + if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button(tr("Processing...")); } - else if (ui_button("Run")) { + else if (!found && ui_button(tr("Setup"))) { + box_preferences_htab.i = preference_tab_t.NEURAL; + box_preferences_show(); + } + else if (found && ui_button(tr("Run"))) { let inp: ui_node_socket_t = node.inputs[0]; let from_node: ui_node_t = null; for (let i: i32 = 0; i < canvas.links.length; ++i) { @@ -54,9 +61,9 @@ function edit_image_node_button(node_id: i32) { iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); let argv: string[] = [ - dir + "/sd_qie", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/qwen_image_vae.safetensors", "--qwen2vl", - dir + "/Qwen2.5-VL-7B-Instruct-Q8_0.gguf", "--qwen2vl_vision", dir + "/Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf", "--sampling-method", "euler", - "--offload-to-cpu", "--diffusion-fa", "--steps", "50", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'", + dir + "/sd", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl", + dir + "/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", "--qwen2vl_vision", dir + "/mmproj-F16.gguf", "--sampling-method", "euler", + "--offload-to-cpu", "--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'", // "-n", // "'" + negative + "'", "-r", dir + "/input.png", "-o", dir + "/output.png", null diff --git a/paint/sources/neural_nodes/image_to_depth_node.ts b/paint/sources/neural_nodes/image_to_depth_node.ts index 6d223128..75d10503 100644 --- a/paint/sources/neural_nodes/image_to_depth_node.ts +++ b/paint/sources/neural_nodes/image_to_depth_node.ts @@ -26,7 +26,7 @@ function image_to_depth_node_button(node_id: i32) { let model: i32 = ui_combo(ui_handle(__ID__), models, tr("Model")); if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button("Processing..."); } else if (ui_button("Run")) { let inp: ui_node_socket_t = node.inputs[0]; @@ -50,7 +50,7 @@ function image_to_depth_node_button(node_id: i32) { } iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); - image_to_pbr_node_run_sd("marigold-depth-v1-1.safetensors", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-depth-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { image_to_depth_node_result = tex; }); } diff --git a/paint/sources/neural_nodes/image_to_normal_map_node.ts b/paint/sources/neural_nodes/image_to_normal_map_node.ts index f547b6f1..af9a4ad7 100644 --- a/paint/sources/neural_nodes/image_to_normal_map_node.ts +++ b/paint/sources/neural_nodes/image_to_normal_map_node.ts @@ -26,7 +26,7 @@ function image_to_normal_map_node_button(node_id: i32) { let model: i32 = ui_combo(ui_handle(__ID__), models, tr("Model")); if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button("Processing..."); } else if (ui_button("Run")) { let inp: ui_node_socket_t = node.inputs[0]; @@ -50,7 +50,7 @@ function image_to_normal_map_node_button(node_id: i32) { } iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); - image_to_pbr_node_run_sd("marigold-normals-v1-1.safetensors", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-normals-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { image_to_normal_map_node_result = tex; }); } diff --git a/paint/sources/neural_nodes/image_to_pbr_node.ts b/paint/sources/neural_nodes/image_to_pbr_node.ts index 01facf46..c424d273 100644 --- a/paint/sources/neural_nodes/image_to_pbr_node.ts +++ b/paint/sources/neural_nodes/image_to_pbr_node.ts @@ -102,7 +102,7 @@ function image_to_pbr_node_button(node_id: i32) { let model: i32 = ui_combo(ui_handle(__ID__), models, tr("Model")); if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button("Processing..."); } else if (ui_button("Run")) { let inp: ui_node_socket_t = node.inputs[0]; @@ -126,13 +126,13 @@ function image_to_pbr_node_button(node_id: i32) { } iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); - image_to_pbr_node_run_sd("marigold-normals-v1-1.safetensors", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-normals-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { image_to_pbr_node_result_normal = tex; - image_to_pbr_node_run_sd("marigold-depth-v1-1.safetensors", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-depth-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { image_to_pbr_node_result_height = tex; - image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.safetensors", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { image_to_pbr_node_result_base = tex; - image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.safetensors", "_roughness", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.q8_0.gguf", "_roughness", function(tex: gpu_texture_t) { image_to_pbr_node_result_roughness = tex; }); }); diff --git a/paint/sources/neural_nodes/inpaint_image_node.ts b/paint/sources/neural_nodes/inpaint_image_node.ts index acae1395..30678481 100644 --- a/paint/sources/neural_nodes/inpaint_image_node.ts +++ b/paint/sources/neural_nodes/inpaint_image_node.ts @@ -26,7 +26,7 @@ function inpaint_image_node_button(node_id: i32) { let model: i32 = ui_combo(ui_handle(__ID__), models, tr("Model")); if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button("Processing..."); } else if (ui_button("Run")) { let inp: ui_node_socket_t = node.inputs[0]; diff --git a/paint/sources/neural_nodes/text_to_image_node.ts b/paint/sources/neural_nodes/text_to_image_node.ts index 70f9faaa..7d7bc621 100644 --- a/paint/sources/neural_nodes/text_to_image_node.ts +++ b/paint/sources/neural_nodes/text_to_image_node.ts @@ -19,15 +19,38 @@ function text_to_image_node_vector(node: ui_node_t, socket: ui_node_socket_t): s return texstore + ".rgb"; } +function text_to_image_node_run_sd(dir: string, prompt: string): string[] { + let argv: string[] = [ + dir + "/sd", + "-m", + dir + "/v1-5-pruned-emaonly.safetensors", + "--offload-to-cpu", + "-W", + "512", + "-H", + "512", + "--steps", + "40", + "-s", + "-1", + "-o", + dir + "/output.png", + "-p", + "'" + prompt + "'", + null + ]; + return argv; +} + function text_to_image_node_run_qwen(dir: string, prompt: string): string[] { let argv: string[] = [ dir + "/sd", "--diffusion-model", - dir + "/qwen-image-Q8_0.gguf", + dir + "/Qwen_Image-Q4_K_S.gguf", "--vae", - dir + "/qwen_image_vae.safetensors", + dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl", - dir + "/Qwen2.5-VL-7B-Instruct-Q8_0.gguf", + dir + "/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", "--sampling-method", "euler", "--offload-to-cpu", @@ -54,13 +77,13 @@ function text_to_image_node_run_wan(dir: string, prompt: string): string[] { "-M", "vid_gen", "--diffusion-model", - dir + "/Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf", + dir + "/Wan2.2-T2V-A14B-LowNoise-Q4_K_S.gguf", "--high-noise-diffusion-model", - dir + "/Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf", + dir + "/Wan2.2-T2V-A14B-HighNoise-Q4_K_S.gguf", "--vae", - dir + "/wan_2.1_vae.safetensors", + dir + "/Wan2.1_VAE.safetensors", "--t5xxl", - dir + "/umt5-xxl-encoder-Q8_0.gguf", + dir + "/umt5-xxl-encoder-Q4_K_S.gguf", "--sampling-method", "euler", "--steps", @@ -88,16 +111,14 @@ function text_to_image_node_run_wan(dir: string, prompt: string): string[] { function text_to_image_node_button(node_id: i32) { let node: ui_node_t = ui_get_node(ui_nodes_get_canvas(true).nodes, node_id); - let models: string[] = [ "Qwen Image", "Wan" ]; + let models: string[] = [ "Stable Diffusion", "Qwen Image", "Wan" ]; let model: i32 = ui_combo(ui_handle(__ID__), models, tr("Model")); let prompt: string = ui_text_area(ui_handle(__ID__), ui_align_t.LEFT, true, tr("prompt"), true); node.buttons[0].height = string_split(prompt, "\n").length + 2; - // ui_button("Download"); - if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button("Processing..."); } else if (ui_button("Run")) { let dir: string; @@ -110,6 +131,9 @@ function text_to_image_node_button(node_id: i32) { let argv: string[]; if (model == 0) { + argv = text_to_image_node_run_sd(dir, prompt); + } + else if (model == 1) { argv = text_to_image_node_run_qwen(dir, prompt); } else { diff --git a/paint/sources/neural_nodes/tile_image_node.ts b/paint/sources/neural_nodes/tile_image_node.ts index 357f5472..b2738d99 100644 --- a/paint/sources/neural_nodes/tile_image_node.ts +++ b/paint/sources/neural_nodes/tile_image_node.ts @@ -26,7 +26,7 @@ function tile_image_node_button(node_id: i32) { let model: i32 = ui_combo(ui_handle(__ID__), models, tr("Model")); if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button("Processing..."); } else if (ui_button("Run")) { let inp: ui_node_socket_t = node.inputs[0]; diff --git a/paint/sources/neural_nodes/upscale_image_node.ts b/paint/sources/neural_nodes/upscale_image_node.ts index f8c8468c..6cb31a2a 100644 --- a/paint/sources/neural_nodes/upscale_image_node.ts +++ b/paint/sources/neural_nodes/upscale_image_node.ts @@ -22,11 +22,11 @@ function upscale_image_node_button(node_id: i32) { let canvas: ui_node_canvas_t = ui_nodes_get_canvas(true); let node: ui_node_t = ui_get_node(canvas.nodes, node_id); - let models: string[] = [ "RealESRGAN_X2", "RealESRGAN_X4" ]; + let models: string[] = [ "RealESRGAN_X4" ]; let model: i32 = ui_combo(ui_handle(__ID__), models, tr("Model")); if (iron_exec_async_done == 0) { - ui_button("Cancel..."); + ui_button("Processing..."); } else if (ui_button("Run")) { let inp: ui_node_socket_t = node.inputs[0]; diff --git a/paint/sources/tab_plugins.ts b/paint/sources/tab_plugins.ts index ff7d1b02..51b7ea0b 100644 --- a/paint/sources/tab_plugins.ts +++ b/paint/sources/tab_plugins.ts @@ -8,7 +8,7 @@ function tab_plugins_draw(htab: ui_handle_t) { ui_row(row); if (ui_button(tr("Manager"))) { - box_preferences_htab.i = 6; // Plugins + box_preferences_htab.i = preference_tab_t.PLUGINS; box_preferences_show(); } ui_end_sticky();