From b630829817038147eb0debf12ed3044c3ab985ea Mon Sep 17 00:00:00 2001 From: luboslenco Date: Fri, 14 Nov 2025 16:04:36 +0100 Subject: [PATCH] Neural nodes progress --- paint/assets/Scene.arm | Bin 168311 -> 169015 bytes paint/shaders/depth_to_ao_pass.kong | 123 ++++++++++++++++++ paint/sources/box_preferences.ts | 1 + paint/sources/neural_nodes/edit_image_node.ts | 6 +- .../neural_nodes/image_to_depth_node.ts | 4 +- .../neural_nodes/image_to_normal_map_node.ts | 4 +- .../sources/neural_nodes/image_to_pbr_node.ts | 51 +++++++- .../neural_nodes/inpaint_image_node.ts | 2 +- .../neural_nodes/text_to_image_node.ts | 8 +- paint/sources/neural_nodes/tile_image_node.ts | 2 +- .../neural_nodes/upscale_image_node.ts | 2 +- paint/sources/neural_nodes/vary_image_node.ts | 6 +- 12 files changed, 187 insertions(+), 22 deletions(-) create mode 100644 paint/shaders/depth_to_ao_pass.kong diff --git a/paint/assets/Scene.arm b/paint/assets/Scene.arm index 43eb552be78a6e7f96ef8e6c960013629d5fdb4a..533d948fef92c4c3ccf789ea8f736c3dc42d822b 100644 GIT binary patch delta 171 zcmeyqh->=_u7(!IEli887zMX4v0{3t!Y{zUz>tz!P?8Z}k{_R#KfTbKNex3#(Ss?5 zUldIxqYsljhM-~u)6eM&zDyj`4@NU(O@AN4BtHFvFOx7I7f^RbYG!&yNqlZ%!Sq0H WCP}bZUVc$-Vh&tvduu$?Cqn=swl%8& delta 27 jcmdnKg6sPtu7(!IEli887zMU3v0{3tvfVX_>60M 1.0 || coord.y < 0.0 || coord.y > 1.0) { + return 1000.0; + } + var sampled: float = sample_lod(height_map, sampler_linear, coord, 0.0).r * height_scale; + return sampled - hit_z; +} + +fun ray_cast(dir: float3, uv: float2, height: float): float { + var n_dir: float3 = normalize(dir); + var step_vec: float3 = n_dir * ray_step; + var hit_xy: float2 = float2(0.0, 0.0); + var hit_z: float = height + 0.001; + var hit_coord: float3 = float3(hit_xy, hit_z); + var prev_delta: float = get_delta_height(hit_xy, hit_z, uv); + var prev_t: float = 0.0; + hit_coord += step_vec; + var t: float = ray_step; + var i: int = 1; + while (i < int(max_steps)) { + var delta: float = get_delta_height(hit_coord.xy, hit_coord.z, uv); + if (delta > 0.0 && delta < 0.2) { + var prev_sdf: float = -prev_delta; + var sdf: float = -delta; + var fraction: float = prev_sdf / (prev_sdf - sdf); + return prev_t + ray_step * fraction; + } + if (t > 0.02) { // max_radius + return 0.02; // max_radius + } + prev_delta = delta; + prev_t = t; + hit_coord += step_vec; + t += ray_step; + i += 1; + } + return 0.02; // max_radius +} + +fun tangent(n: float3): float3 { + var t1: float3 = cross(n, float3(0.0, 0.0, 1.0)); + var t2: float3 = cross(n, float3(0.0, 1.0, 0.0)); + if (length(t1) > length(t2)) { + return normalize(t1); + } + return normalize(t2); +} + +fun depth_to_ao_pass_frag(input: vert_out): float4 { + var height: float = sample_lod(height_map, sampler_linear, input.tex, 0.0).r * height_scale; + var normal: float3 = sample_lod(normal_map, sampler_linear, input.tex, 0.0).rgb * 2.0 - 1.0; + var n: float3 = normalize(normal); + var o1: float3 = tangent(n); + var o2: float3 = cross(o1, n); + var c1: float3 = 0.5 * (o1 + o2); + var c2: float3 = 0.5 * (o1 - o2); + var col: float = ray_cast(lerp3(n, o1, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -o1, 0.5), input.tex, height); + col += ray_cast(lerp3(n, o2, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -o2, 0.5), input.tex, height); + col += ray_cast(lerp3(n, c1, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -c1, 0.5), input.tex, height); + col += ray_cast(lerp3(n, c2, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -c2, 0.5), input.tex, height); + + var d1: float3 = normalize(o1 + 0.5 * o2); + var d2: float3 = normalize(o1 - 0.5 * o2); + var d3: float3 = normalize(o2 + 0.5 * o1); + var d4: float3 = normalize(o2 - 0.5 * o1); + col += ray_cast(lerp3(n, d1, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -d1, 0.5), input.tex, height); + col += ray_cast(lerp3(n, d2, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -d2, 0.5), input.tex, height); + col += ray_cast(lerp3(n, d3, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -d3, 0.5), input.tex, height); + col += ray_cast(lerp3(n, d4, 0.5), input.tex, height); + col += ray_cast(lerp3(n, -d4, 0.5), input.tex, height); + var f: float = (col / (16.0 * 0.02)); // max_radius + return float4(f, f, f, 1.0); +} + +#[pipe] +struct pipe { + vertex = depth_to_ao_pass_vert; + fragment = depth_to_ao_pass_frag; +} diff --git a/paint/sources/box_preferences.ts b/paint/sources/box_preferences.ts index ed675d94..f7d41656 100644 --- a/paint/sources/box_preferences.ts +++ b/paint/sources/box_preferences.ts @@ -760,6 +760,7 @@ function box_preferences_neural_tab() { // https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-depth-v1-1.q8_0.gguf // https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-normals-v1-1.q8_0.gguf // https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-iid-appearance-v1-1.q8_0.gguf + // https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-iid-lighting-v1-1.q8_0.gguf } } diff --git a/paint/sources/neural_nodes/edit_image_node.ts b/paint/sources/neural_nodes/edit_image_node.ts index 5ea4e069..f8c28d3b 100644 --- a/paint/sources/neural_nodes/edit_image_node.ts +++ b/paint/sources/neural_nodes/edit_image_node.ts @@ -23,11 +23,11 @@ function edit_image_node_button(node_id: i32) { iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); let argv: string[] = [ - dir + "/sd", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl", + dir + "/sd_vulkan", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl", dir + "/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", "--qwen2vl_vision", dir + "/mmproj-F16.gguf", "--sampling-method", "euler", "--offload-to-cpu", - "--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'", + "--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", prompt, // "-n", - // "'" + negative + "'", + // negative, "-r", dir + "/input.png", "-o", dir + "/output.png", null ]; diff --git a/paint/sources/neural_nodes/image_to_depth_node.ts b/paint/sources/neural_nodes/image_to_depth_node.ts index 82af75eb..18841f7c 100644 --- a/paint/sources/neural_nodes/image_to_depth_node.ts +++ b/paint/sources/neural_nodes/image_to_depth_node.ts @@ -20,7 +20,7 @@ function image_to_depth_node_button(node_id: i32) { iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); let argv: string[] = [ - dir + "/sd", + dir + "/sd_vulkan", "-m", dir + "/marigold-depth-v1-1.q8_0.gguf", "--sampling-method", @@ -34,7 +34,7 @@ function image_to_depth_node_button(node_id: i32) { "-H", "768", "-p", - "' '", + " ", "-i", dir + "/input.png", "-o", diff --git a/paint/sources/neural_nodes/image_to_normal_map_node.ts b/paint/sources/neural_nodes/image_to_normal_map_node.ts index 45c80e51..ef521402 100644 --- a/paint/sources/neural_nodes/image_to_normal_map_node.ts +++ b/paint/sources/neural_nodes/image_to_normal_map_node.ts @@ -20,7 +20,7 @@ function image_to_normal_map_node_button(node_id: i32) { iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); let argv: string[] = [ - dir + "/sd", + dir + "/sd_vulkan", "-m", dir + "/marigold-normals-v1-1.q8_0.gguf", "--sampling-method", @@ -34,7 +34,7 @@ function image_to_normal_map_node_button(node_id: i32) { "-H", "768", "-p", - "' '", + " ", "-i", dir + "/input.png", "-o", diff --git a/paint/sources/neural_nodes/image_to_pbr_node.ts b/paint/sources/neural_nodes/image_to_pbr_node.ts index 3c55f628..0ff8d469 100644 --- a/paint/sources/neural_nodes/image_to_pbr_node.ts +++ b/paint/sources/neural_nodes/image_to_pbr_node.ts @@ -62,7 +62,7 @@ function image_to_pbr_node_run_sd(model: string, prompt: string, done: (tex: gpu let dir: string = neural_node_dir(); let argv: string[] = [ - dir + "/sd", + dir + "/sd_vulkan", "-m", dir + "/" + model, "--sampling-method", @@ -76,7 +76,7 @@ function image_to_pbr_node_run_sd(model: string, prompt: string, done: (tex: gpu "-H", "768", "-p", - "'" + prompt + "'", + prompt, "-i", dir + "/input.png", "-o", @@ -102,14 +102,54 @@ function image_to_pbr_node_button(node_id: i32) { let dir: string = neural_node_dir(); iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); - image_to_pbr_node_run_sd("marigold-normals-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-normals-v1-1.q8_0.gguf", "_normals", function(tex: gpu_texture_t) { image_to_pbr_node_result_normal = tex; - image_to_pbr_node_run_sd("marigold-depth-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-depth-v1-1.q8_0.gguf", "_height", function(tex: gpu_texture_t) { image_to_pbr_node_result_height = tex; - image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) { + image_to_pbr_node_run_sd("marigold-iid-lighting-v1-1.q8_0.gguf", "_base", function(tex: gpu_texture_t) { image_to_pbr_node_result_base = tex; image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.q8_0.gguf", "_roughness", function(tex: gpu_texture_t) { + // image_to_pbr_node_run_sd("marigold-iid-lighting-v1-1.q8_0.gguf", "_diffuse_shading", function(tex: gpu_texture_t) { image_to_pbr_node_result_roughness = tex; + + sys_notify_on_next_frame(function() { + let occmap: render_target_t; + { + let t: render_target_t = render_target_create(); + t.name = "occmap"; + t.width = 2048; + t.height = 2048; + t.format = "RGBA32"; + render_path_create_render_target(t); + occmap = t; + } + { + let t: render_target_t = render_target_create(); + t.name = "_height_map"; + t.width = 768; + t.height = 768; + t.format = "RGBA32"; + t._image = image_to_pbr_node_result_height; + map_set(render_path_render_targets, t.name, t); + } + { + let t: render_target_t = render_target_create(); + t.name = "_normal_map"; + t.width = 768; + t.height = 768; + t.format = "RGBA32"; + t._image = image_to_pbr_node_result_normal; + map_set(render_path_render_targets, t.name, t); + } + + render_path_load_shader("Scene/depth_to_ao_pass/depth_to_ao_pass"); + render_path_set_target("occmap"); + render_path_bind_target("_height_map", "height_map"); + render_path_bind_target("_normal_map", "normal_map"); + render_path_draw_shader("Scene/depth_to_ao_pass/depth_to_ao_pass"); + image_to_pbr_node_result_occlusion = occmap._image; + // blur + }); }); }); }); @@ -119,6 +159,7 @@ function image_to_pbr_node_button(node_id: i32) { } function image_to_pbr_node_check_result(done: (tex: gpu_texture_t) => void) { + iron_delay_idle_sleep(); if (iron_exec_async_done == 1) { let dir: string = neural_node_dir(); let file: string = dir + path_sep + "output.png"; diff --git a/paint/sources/neural_nodes/inpaint_image_node.ts b/paint/sources/neural_nodes/inpaint_image_node.ts index c39f9d2b..526f427c 100644 --- a/paint/sources/neural_nodes/inpaint_image_node.ts +++ b/paint/sources/neural_nodes/inpaint_image_node.ts @@ -35,7 +35,7 @@ function inpaint_image_node_button(node_id: i32) { "--diffusion-fa", // "--vae-tiling", // "--vae-on-cpu", - "--steps", "40", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'", "-n", "'" + negative + "'", "-r", dir + "/input.png", "-o", + "--steps", "40", "-s", "-1", "-W", "512", "-H", "512", "-p", prompt, "-n", negative, "-r", dir + "/input.png", "-o", dir + "/output.png", null ]; diff --git a/paint/sources/neural_nodes/text_to_image_node.ts b/paint/sources/neural_nodes/text_to_image_node.ts index 1f51992a..3b6891bf 100644 --- a/paint/sources/neural_nodes/text_to_image_node.ts +++ b/paint/sources/neural_nodes/text_to_image_node.ts @@ -7,7 +7,7 @@ function text_to_image_node_init() { function text_to_image_node_sd_args(dir: string, prompt: string): string[] { let argv: string[] = [ - dir + "/sd", + dir + "/sd_vulkan", "-m", dir + "/v1-5-pruned-emaonly.safetensors", "--offload-to-cpu", @@ -30,7 +30,7 @@ function text_to_image_node_sd_args(dir: string, prompt: string): string[] { function text_to_image_node_qwen_args(dir: string, prompt: string): string[] { let argv: string[] = [ - dir + "/sd", + dir + "/sd_vulkan", "--diffusion-model", dir + "/Qwen_Image-Q4_K_S.gguf", "--vae", @@ -59,7 +59,7 @@ function text_to_image_node_qwen_args(dir: string, prompt: string): string[] { function text_to_image_node_wan_args(dir: string, prompt: string): string[] { let argv: string[] = [ - dir + "/sd", + dir + "/sd_vulkan", "-M", "vid_gen", "--diffusion-model", @@ -88,7 +88,7 @@ function text_to_image_node_wan_args(dir: string, prompt: string): string[] { "-o", dir + "/output.png", "-p", - "'" + prompt + "'", + prompt, null ]; return argv; diff --git a/paint/sources/neural_nodes/tile_image_node.ts b/paint/sources/neural_nodes/tile_image_node.ts index 3ad7de03..a90ec462 100644 --- a/paint/sources/neural_nodes/tile_image_node.ts +++ b/paint/sources/neural_nodes/tile_image_node.ts @@ -59,7 +59,7 @@ function tile_image_node_button(node_id: i32) { "-H", "512", "-p", - "'" + prompt + "'", + prompt, "-n", "'" + negative + "'", "-r", diff --git a/paint/sources/neural_nodes/upscale_image_node.ts b/paint/sources/neural_nodes/upscale_image_node.ts index 70cbb179..87da8511 100644 --- a/paint/sources/neural_nodes/upscale_image_node.ts +++ b/paint/sources/neural_nodes/upscale_image_node.ts @@ -20,7 +20,7 @@ function upscale_image_node_button(node_id: i32) { iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); let argv: string[] = - [ dir + "/sd", "-M", "upscale", "--upscale-model", dir + "/RealESRGAN_x4plus.pth", "-i", dir + "/input.png", "-o", dir + "/output.png", null ]; + [ dir + "/sd_vulkan", "-M", "upscale", "--upscale-model", dir + "/RealESRGAN_x4plus.pth", "-i", dir + "/input.png", "-o", dir + "/output.png", null ]; iron_exec_async(argv[0], argv.buffer); sys_notify_on_update(neural_node_check_result, node); } diff --git a/paint/sources/neural_nodes/vary_image_node.ts b/paint/sources/neural_nodes/vary_image_node.ts index a79ac486..645cfcbb 100644 --- a/paint/sources/neural_nodes/vary_image_node.ts +++ b/paint/sources/neural_nodes/vary_image_node.ts @@ -23,11 +23,11 @@ function vary_image_node_button(node_id: i32) { iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0); let argv: string[] = [ - dir + "/sd", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl", + dir + "/sd_vulkan", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl", dir + "/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", "--qwen2vl_vision", dir + "/mmproj-F16.gguf", "--sampling-method", "euler", - "--offload-to-cpu", "--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'", + "--offload-to-cpu", "--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", prompt, // "-n", - // "'" + negative + "'", + // negative, "-r", dir + "/input.png", "-o", dir + "/output.png", null ];