Add a level mesh viewer (--viewer cmdline arg) for better debugging

CMakeLists.txt

@@ -81,13 +81,17 @@ set(ZDRAY_SOURCES
 	src/lightmapper/gpuraytracer.h
 	src/lightmapper/stacktrace.cpp
 	src/lightmapper/stacktrace.h
+	src/lightmapper/levelmeshviewer.cpp
+	src/lightmapper/levelmeshviewer.h
 	src/lightmapper/glsl/binding_lightmapper.glsl.h
 	src/lightmapper/glsl/binding_raytrace.glsl.h
 	src/lightmapper/glsl/binding_textures.glsl.h
+	src/lightmapper/glsl/binding_viewer.glsl.h
 	src/lightmapper/glsl/frag_blur.glsl.h
 	src/lightmapper/glsl/frag_copy.glsl.h
 	src/lightmapper/glsl/frag_raytrace.glsl.h
 	src/lightmapper/glsl/frag_resolve.glsl.h
+	src/lightmapper/glsl/frag_viewer.glsl.h
 	src/lightmapper/glsl/montecarlo.glsl.h
 	src/lightmapper/glsl/polyfill_rayquery.glsl.h
 	src/lightmapper/glsl/trace_ambient_occlusion.glsl.h
@@ -98,6 +102,7 @@ set(ZDRAY_SOURCES
 	src/lightmapper/glsl/vert_copy.glsl.h
 	src/lightmapper/glsl/vert_raytrace.glsl.h
 	src/lightmapper/glsl/vert_screenquad.glsl.h
+	src/lightmapper/glsl/vert_viewer.glsl.h
 	src/models/model.cpp
 	src/models/model.h
 	src/models/model_md2.h

src/lightmapper/glsl/binding_viewer.glsl.h

@@ -0,0 +1,97 @@
+static const char* binding_viewer_glsl = R"glsl(
+
+struct SurfaceInfo
+{
+	vec3 Normal;
+	float Sky;
+	uint PortalIndex;
+	int TextureIndex;
+	float Alpha;
+	float Padding0;
+	uint LightStart;
+	uint LightEnd;
+	uint Padding1;
+	uint Padding2;
+};
+
+struct PortalInfo
+{
+	mat4 Transformation;
+};
+
+struct LightInfo
+{
+	vec3 Origin;
+	float Padding0;
+	vec3 RelativeOrigin;
+	float Padding1;
+	float Radius;
+	float Intensity;
+	float InnerAngleCos;
+	float OuterAngleCos;
+	vec3 SpotDir;
+	float SourceRadius;
+	vec3 Color;
+	float Padding3;
+};
+
+layout(set = 0, binding = 0, std430) buffer SurfaceIndexBuffer { uint surfaceIndices[]; };
+layout(set = 0, binding = 1, std430) buffer SurfaceBuffer { SurfaceInfo surfaces[]; };
+layout(set = 0, binding = 2, std430) buffer LightBuffer { LightInfo lights[]; };
+layout(set = 0, binding = 3, std430) buffer LightIndexBuffer { int lightIndexes[]; };
+layout(set = 0, binding = 4, std430) buffer PortalBuffer { PortalInfo portals[]; };
+
+#if defined(USE_RAYQUERY)
+
+layout(set = 0, binding = 5) uniform accelerationStructureEXT acc;
+
+#else
+
+struct CollisionNode
+{
+	vec3 center;
+	float padding1;
+	vec3 extents;
+	float padding2;
+	int left;
+	int right;
+	int element_index;
+	int padding3;
+};
+
+layout(set = 0, binding = 5, std430) buffer NodeBuffer
+{
+	int nodesRoot;
+	int nodebufferPadding1;
+	int nodebufferPadding2;
+	int nodebufferPadding3;
+	CollisionNode nodes[];
+};
+
+#endif
+
+struct SurfaceVertex // Note: this must always match the FFlatVertex struct
+{
+	vec3 pos;
+	float lindex;
+	vec2 uv;
+	vec2 luv;
+};
+
+layout(set = 0, binding = 6, std430) buffer VertexBuffer { SurfaceVertex vertices[]; };
+layout(set = 0, binding = 7, std430) buffer ElementBuffer { int elements[]; };
+
+layout(set = 1, binding = 0) uniform sampler2D textures[];
+
+layout(push_constant) uniform PushConstants
+{
+	mat4 ViewToWorld;
+	vec3 CameraPos;
+	float ProjX;
+	vec3 SunDir;
+	float ProjY;
+	vec3 SunColor;
+	float SunIntensity;
+};
+
+)glsl";

src/lightmapper/glsl/frag_viewer.glsl.h

@@ -0,0 +1,51 @@
+static const char* frag_viewer_glsl = R"glsl(
+
+#include <shaders/lightmap/binding_viewer.glsl>
+#include <shaders/lightmap/polyfill_rayquery.glsl>
+#include <shaders/lightmap/trace_levelmesh.glsl>
+#include <shaders/lightmap/trace_sunlight.glsl>
+#include <shaders/lightmap/trace_light.glsl>
+#include <shaders/lightmap/trace_ambient_occlusion.glsl>
+
+layout(location = 0) in vec3 FragRay;
+layout(location = 0) out vec4 fragcolor;
+
+void main()
+{
+	vec3 incoming = vec3(0.1);
+
+	vec3 origin = CameraPos;
+	vec3 L = normalize(FragRay);
+	TraceResult result = TraceFirstHit(origin, 0.0, L, 10000.0);
+	if (result.primitiveIndex != -1)
+	{
+		SurfaceInfo surface = GetSurface(result.primitiveIndex);
+		vec3 surfacepos = origin + L * result.t;
+
+		if (surface.Sky == 0.0)
+		{
+			incoming += TraceSunLight(surfacepos, surface.Normal);
+
+			uint LightStart = surface.LightStart;
+			uint LightEnd = surface.LightEnd;
+			for (uint j = LightStart; j < LightEnd; j++)
+			{
+				incoming += TraceLight(surfacepos, surface.Normal, lights[lightIndexes[j]], 0.0);
+			}
+
+			// incoming *= TraceAmbientOcclusion(surfacepos, surface.Normal);
+		}
+		else
+		{
+			incoming = SunColor;
+		}
+	}
+	else
+	{
+		incoming = vec3(1.0, 0.0, 0.0);
+	}
+
+	fragcolor = vec4(incoming, 1.0);
+}
+
+)glsl";

src/lightmapper/glsl/vert_viewer.glsl.h

@@ -0,0 +1,24 @@
+static const char* vert_viewer_glsl = R"glsl(
+
+#include <shaders/lightmap/binding_viewer.glsl>
+
+layout(location = 0) out vec3 FragRay;
+
+vec2 positions[4] = vec2[](
+	vec2(0.0, 0.0),
+	vec2(1.0, 0.0),
+	vec2(0.0, 1.0),
+	vec2(1.0, 1.0)
+);
+
+void main()
+{
+	vec4 viewpos = vec4(positions[gl_VertexIndex] * 2.0 - 1.0, 1.0, 1.0);
+	viewpos.x /= ProjX;
+	viewpos.y = -viewpos.y / ProjY;
+	FragRay = (ViewToWorld * viewpos).xyz - CameraPos;
+
+	gl_Position = vec4(positions[gl_VertexIndex] * 2.0 - 1.0, 1.0, 1.0);
+}
+
+)glsl";

src/lightmapper/gpuraytracer.cpp

@@ -5,17 +5,22 @@
 #include "vk_lightmapper.h"
 #include "renderdoc_app.h"
 #include "doom_levelmesh.h"
+#include "levelmeshviewer.h"
 
 #ifndef _WIN32
 #include <dlfcn.h>
 #endif
 
 static RENDERDOC_API_1_4_2* rdoc_api;
 
+extern bool showviewer;
+
 GPURaytracer::GPURaytracer()
 {
 	LoadRenderDoc();
-	mDevice = std::make_unique<VulkanRenderDevice>();
+	if (showviewer)
+		mViewer = std::make_unique<LevelMeshViewer>();
+	mDevice = std::make_unique<VulkanRenderDevice>(mViewer.get());
 	PrintVulkanInfo();
 }
 
@@ -77,6 +82,9 @@ void GPURaytracer::Raytrace(DoomLevelMesh* mesh)
 		{
 			mDevice->GetTextureManager()->DownloadLightmap(arrayIndex, mesh->LMTextureData.Data() + arrayIndex * mesh->LMTextureSize * mesh->LMTextureSize * 4);
 		}
+
+		if (mViewer)
+			mViewer->Exec(mDevice.get(), mesh->SunDirection, mesh->SunColor, 1.0f);
 	}
 	catch (...)
 	{

src/lightmapper/gpuraytracer.h

@@ -3,6 +3,7 @@
 #include "vk_renderdevice.h"
 
 class DoomLevelMesh;
+class LevelMeshViewer;
 
 class GPURaytracer
 {
@@ -16,5 +17,6 @@ class GPURaytracer
 	void PrintVulkanInfo();
 	void LoadRenderDoc();
 
+	std::unique_ptr<LevelMeshViewer> mViewer;
 	std::unique_ptr<VulkanRenderDevice> mDevice;
 };

src/lightmapper/hw_levelmesh.cpp

@@ -9,6 +9,15 @@ LevelMesh::LevelMesh()
 
 	AddEmptyMesh();
 	UpdateCollision();
+
+	Mesh.MaxVertices = std::max(Mesh.Vertices.Size() * 2, (unsigned int)10000);
+	Mesh.MaxIndexes = std::max(Mesh.Indexes.Size() * 2, (unsigned int)10000);
+	Mesh.MaxSurfaces = std::max(Mesh.SurfaceIndexes.Size() * 2, (unsigned int)10000);
+	Mesh.MaxUniforms = std::max(Mesh.Uniforms.Size() * 2, (unsigned int)10000);
+	Mesh.MaxSurfaceIndexes = std::max(Mesh.SurfaceIndexes.Size() * 2, (unsigned int)10000);
+	Mesh.MaxNodes = (int)std::max(Collision->get_nodes().size() * 2, (size_t)10000);
+	Mesh.MaxLights = 100'000;
+	Mesh.MaxLightIndexes = 4 * 1024 * 1024;
 }
 
 void LevelMesh::AddEmptyMesh()