Neural nodes progress

paint/shaders/depth_to_ao_pass.kong

@@ -0,0 +1,123 @@
+
+#[set(everything)]
+const constants: {
+	empty: float4;
+};
+
+#[set(everything)]
+const sampler_linear: sampler;
+
+#[set(everything)]
+const height_map: tex2d;
+
+#[set(everything)]
+const normal_map: tex2d;
+
+const max_steps: float = 100.0;
+const ray_step: float = 0.005;
+const height_scale: float = 1.0;
+
+struct vert_in {
+	pos: float2;
+}
+
+struct vert_out {
+	pos: float4;
+	tex: float2;
+}
+
+fun depth_to_ao_pass_vert(input: vert_in): vert_out {
+	var output: vert_out;
+	output.tex = input.pos.xy * 0.5 + 0.5;
+	output.tex.y = 1.0 - output.tex.y;
+	output.pos = float4(input.pos.xy, 0.0, 1.0);
+	return output;
+}
+
+fun get_delta_height(hit_xy: float2, hit_z: float, uv: float2): float {
+	var coord: float2 = uv + hit_xy;
+	if (coord.x < 0.0 || coord.x > 1.0 || coord.y < 0.0 || coord.y > 1.0) {
+		return 1000.0;
+	}
+	var sampled: float = sample_lod(height_map, sampler_linear, coord, 0.0).r * height_scale;
+	return sampled - hit_z;
+}
+
+fun ray_cast(dir: float3, uv: float2, height: float): float {
+	var n_dir: float3 = normalize(dir);
+	var step_vec: float3 = n_dir * ray_step;
+	var hit_xy: float2 = float2(0.0, 0.0);
+	var hit_z: float = height + 0.001;
+	var hit_coord: float3 = float3(hit_xy, hit_z);
+	var prev_delta: float = get_delta_height(hit_xy, hit_z, uv);
+	var prev_t: float = 0.0;
+	hit_coord += step_vec;
+	var t: float = ray_step;
+	var i: int = 1;
+	while (i < int(max_steps)) {
+		var delta: float = get_delta_height(hit_coord.xy, hit_coord.z, uv);
+		if (delta > 0.0 && delta < 0.2) {
+			var prev_sdf: float = -prev_delta;
+			var sdf: float = -delta;
+			var fraction: float = prev_sdf / (prev_sdf - sdf);
+			return prev_t + ray_step * fraction;
+		}
+		if (t > 0.02) { // max_radius
+			return 0.02; // max_radius
+		}
+		prev_delta = delta;
+		prev_t = t;
+		hit_coord += step_vec;
+		t += ray_step;
+		i += 1;
+	}
+	return 0.02; // max_radius
+}
+
+fun tangent(n: float3): float3 {
+	var t1: float3 = cross(n, float3(0.0, 0.0, 1.0));
+	var t2: float3 = cross(n, float3(0.0, 1.0, 0.0));
+	if (length(t1) > length(t2)) {
+		return normalize(t1);
+	}
+	return normalize(t2);
+}
+
+fun depth_to_ao_pass_frag(input: vert_out): float4 {
+	var height: float = sample_lod(height_map, sampler_linear, input.tex, 0.0).r * height_scale;
+	var normal: float3 = sample_lod(normal_map, sampler_linear, input.tex, 0.0).rgb * 2.0 - 1.0;
+	var n: float3 = normalize(normal);
+	var o1: float3 = tangent(n);
+	var o2: float3 = cross(o1, n);
+	var c1: float3 = 0.5 * (o1 + o2);
+	var c2: float3 = 0.5 * (o1 - o2);
+	var col: float = ray_cast(lerp3(n, o1, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -o1, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, o2, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -o2, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, c1, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -c1, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, c2, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -c2, 0.5), input.tex, height);
+
+	var d1: float3 = normalize(o1 + 0.5 * o2);
+	var d2: float3 = normalize(o1 - 0.5 * o2);
+	var d3: float3 = normalize(o2 + 0.5 * o1);
+	var d4: float3 = normalize(o2 - 0.5 * o1);
+	col += ray_cast(lerp3(n, d1, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -d1, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, d2, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -d2, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, d3, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -d3, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, d4, 0.5), input.tex, height);
+	col += ray_cast(lerp3(n, -d4, 0.5), input.tex, height);
+	var f: float = (col / (16.0 * 0.02)); // max_radius
+	return float4(f, f, f, 1.0);
+}
+
+#[pipe]
+struct pipe {
+	vertex = depth_to_ao_pass_vert;
+	fragment = depth_to_ao_pass_frag;
+}

paint/sources/box_preferences.ts

@@ -760,6 +760,7 @@ function box_preferences_neural_tab() {
 			// https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-depth-v1-1.q8_0.gguf
 			// https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-normals-v1-1.q8_0.gguf
 			// https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-iid-appearance-v1-1.q8_0.gguf
+			// https://huggingface.co/armory3d/marigold-v1-1-gguf/resolve/main/marigold-iid-lighting-v1-1.q8_0.gguf
 		}
 	}
 

paint/sources/neural_nodes/edit_image_node.ts

@@ -23,11 +23,11 @@ function edit_image_node_button(node_id: i32) {
 			iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0);
 
 			let argv: string[] = [
-				dir + "/sd", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl",
+				dir + "/sd_vulkan", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl",
 				dir + "/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", "--qwen2vl_vision", dir + "/mmproj-F16.gguf", "--sampling-method", "euler", "--offload-to-cpu",
-				"--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'",
+				"--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", prompt,
 				// "-n",
-				// "'" + negative + "'",
+				// negative,
 				"-r", dir + "/input.png", "-o", dir + "/output.png", null
 			];
 

paint/sources/neural_nodes/image_to_depth_node.ts

@@ -20,7 +20,7 @@ function image_to_depth_node_button(node_id: i32) {
 			iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0);
 
 			let argv: string[] = [
-				dir + "/sd",
+				dir + "/sd_vulkan",
 				"-m",
 				dir + "/marigold-depth-v1-1.q8_0.gguf",
 				"--sampling-method",
@@ -34,7 +34,7 @@ function image_to_depth_node_button(node_id: i32) {
 				"-H",
 				"768",
 				"-p",
-				"' '",
+				" ",
 				"-i",
 				dir + "/input.png",
 				"-o",

paint/sources/neural_nodes/image_to_normal_map_node.ts

@@ -20,7 +20,7 @@ function image_to_normal_map_node_button(node_id: i32) {
 			iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0);
 
 			let argv: string[] = [
-				dir + "/sd",
+				dir + "/sd_vulkan",
 				"-m",
 				dir + "/marigold-normals-v1-1.q8_0.gguf",
 				"--sampling-method",
@@ -34,7 +34,7 @@ function image_to_normal_map_node_button(node_id: i32) {
 				"-H",
 				"768",
 				"-p",
-				"' '",
+				" ",
 				"-i",
 				dir + "/input.png",
 				"-o",

paint/sources/neural_nodes/image_to_pbr_node.ts

@@ -62,7 +62,7 @@ function image_to_pbr_node_run_sd(model: string, prompt: string, done: (tex: gpu
 	let dir: string = neural_node_dir();
 
 	let argv: string[] = [
-		dir + "/sd",
+		dir + "/sd_vulkan",
 		"-m",
 		dir + "/" + model,
 		"--sampling-method",
@@ -76,7 +76,7 @@ function image_to_pbr_node_run_sd(model: string, prompt: string, done: (tex: gpu
 		"-H",
 		"768",
 		"-p",
-		"'" + prompt + "'",
+		prompt,
 		"-i",
 		dir + "/input.png",
 		"-o",
@@ -102,14 +102,54 @@ function image_to_pbr_node_button(node_id: i32) {
 			let dir: string = neural_node_dir();
 			iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0);
 
-			image_to_pbr_node_run_sd("marigold-normals-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) {
+			image_to_pbr_node_run_sd("marigold-normals-v1-1.q8_0.gguf", "_normals", function(tex: gpu_texture_t) {
 				image_to_pbr_node_result_normal = tex;
-				image_to_pbr_node_run_sd("marigold-depth-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) {
+				image_to_pbr_node_run_sd("marigold-depth-v1-1.q8_0.gguf", "_height", function(tex: gpu_texture_t) {
 					image_to_pbr_node_result_height = tex;
-					image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.q8_0.gguf", " ", function(tex: gpu_texture_t) {
+					image_to_pbr_node_run_sd("marigold-iid-lighting-v1-1.q8_0.gguf", "_base", function(tex: gpu_texture_t) {
 						image_to_pbr_node_result_base = tex;
 						image_to_pbr_node_run_sd("marigold-iid-appearance-v1-1.q8_0.gguf", "_roughness", function(tex: gpu_texture_t) {
+						// image_to_pbr_node_run_sd("marigold-iid-lighting-v1-1.q8_0.gguf", "_diffuse_shading", function(tex: gpu_texture_t) {
 							image_to_pbr_node_result_roughness = tex;
+
+							sys_notify_on_next_frame(function() {
+								let occmap: render_target_t;
+								{
+									let t: render_target_t = render_target_create();
+									t.name                 = "occmap";
+									t.width                = 2048;
+									t.height               = 2048;
+									t.format               = "RGBA32";
+									render_path_create_render_target(t);
+									occmap = t;
+								}
+								{
+									let t: render_target_t = render_target_create();
+									t.name                 = "_height_map";
+									t.width                = 768;
+									t.height               = 768;
+									t.format               = "RGBA32";
+									t._image = image_to_pbr_node_result_height;
+									map_set(render_path_render_targets, t.name, t);
+								}
+								{
+									let t: render_target_t = render_target_create();
+									t.name                 = "_normal_map";
+									t.width                = 768;
+									t.height               = 768;
+									t.format               = "RGBA32";
+									t._image = image_to_pbr_node_result_normal;
+									map_set(render_path_render_targets, t.name, t);
+								}
+
+								render_path_load_shader("Scene/depth_to_ao_pass/depth_to_ao_pass");
+								render_path_set_target("occmap");
+								render_path_bind_target("_height_map", "height_map");
+								render_path_bind_target("_normal_map", "normal_map");
+								render_path_draw_shader("Scene/depth_to_ao_pass/depth_to_ao_pass");
+								image_to_pbr_node_result_occlusion = occmap._image;
+								// blur
+							});
 						});
 					});
 				});
@@ -119,6 +159,7 @@ function image_to_pbr_node_button(node_id: i32) {
 }
 
 function image_to_pbr_node_check_result(done: (tex: gpu_texture_t) => void) {
+	iron_delay_idle_sleep();
 	if (iron_exec_async_done == 1) {
 		let dir: string = neural_node_dir();
 		let file: string = dir + path_sep + "output.png";

paint/sources/neural_nodes/inpaint_image_node.ts

@@ -35,7 +35,7 @@ function inpaint_image_node_button(node_id: i32) {
 					"--diffusion-fa",
 					// "--vae-tiling",
 					// "--vae-on-cpu",
-					"--steps", "40", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'", "-n", "'" + negative + "'", "-r", dir + "/input.png", "-o",
+					"--steps", "40", "-s", "-1", "-W", "512", "-H", "512", "-p", prompt, "-n", negative, "-r", dir + "/input.png", "-o",
 					dir + "/output.png", null
 				];
 

paint/sources/neural_nodes/text_to_image_node.ts

@@ -7,7 +7,7 @@ function text_to_image_node_init() {
 
 function text_to_image_node_sd_args(dir: string, prompt: string): string[] {
 	let argv: string[] = [
-		dir + "/sd",
+		dir + "/sd_vulkan",
 		"-m",
 		dir + "/v1-5-pruned-emaonly.safetensors",
 		"--offload-to-cpu",
@@ -30,7 +30,7 @@ function text_to_image_node_sd_args(dir: string, prompt: string): string[] {
 
 function text_to_image_node_qwen_args(dir: string, prompt: string): string[] {
 	let argv: string[] = [
-		dir + "/sd",
+		dir + "/sd_vulkan",
 		"--diffusion-model",
 		dir + "/Qwen_Image-Q4_K_S.gguf",
 		"--vae",
@@ -59,7 +59,7 @@ function text_to_image_node_qwen_args(dir: string, prompt: string): string[] {
 
 function text_to_image_node_wan_args(dir: string, prompt: string): string[] {
 	let argv: string[] = [
-		dir + "/sd",
+		dir + "/sd_vulkan",
 		"-M",
 		"vid_gen",
 		"--diffusion-model",
@@ -88,7 +88,7 @@ function text_to_image_node_wan_args(dir: string, prompt: string): string[] {
 		"-o",
 		dir + "/output.png",
 		"-p",
-		"'" + prompt + "'",
+		prompt,
 		null
 	];
 	return argv;

paint/sources/neural_nodes/tile_image_node.ts

@@ -59,7 +59,7 @@ function tile_image_node_button(node_id: i32) {
 					"-H",
 					"512",
 					"-p",
-					"'" + prompt + "'",
+					prompt,
 					"-n",
 					"'" + negative + "'",
 					"-r",

paint/sources/neural_nodes/upscale_image_node.ts

@@ -20,7 +20,7 @@ function upscale_image_node_button(node_id: i32) {
 			iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0);
 
 			let argv: string[] =
-			    [ dir + "/sd", "-M", "upscale", "--upscale-model", dir + "/RealESRGAN_x4plus.pth", "-i", dir + "/input.png", "-o", dir + "/output.png", null ];
+			    [ dir + "/sd_vulkan", "-M", "upscale", "--upscale-model", dir + "/RealESRGAN_x4plus.pth", "-i", dir + "/input.png", "-o", dir + "/output.png", null ];
 			iron_exec_async(argv[0], argv.buffer);
 			sys_notify_on_update(neural_node_check_result, node);
 		}

paint/sources/neural_nodes/vary_image_node.ts

@@ -23,11 +23,11 @@ function vary_image_node_button(node_id: i32) {
 			iron_write_png(dir + path_sep + "input.png", gpu_get_texture_pixels(input), input.width, input.height, 0);
 
 			let argv: string[] = [
-				dir + "/sd", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl",
+				dir + "/sd_vulkan", "--diffusion-model", dir + "/Qwen-Image-Edit-2509-Q4_K_S.gguf", "--vae", dir + "/Qwen_Image-VAE.safetensors", "--qwen2vl",
 				dir + "/Qwen2.5-VL-7B-Instruct-Q4_K_S.gguf", "--qwen2vl_vision", dir + "/mmproj-F16.gguf", "--sampling-method", "euler",
-				"--offload-to-cpu", "--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", "'" + prompt + "'",
+				"--offload-to-cpu", "--diffusion-fa", "--steps", "30", "-s", "-1", "-W", "512", "-H", "512", "-p", prompt,
 				// "-n",
-				// "'" + negative + "'",
+				// negative,
 				"-r", dir + "/input.png", "-o", dir + "/output.png", null
 			];