Gui Stencil Mask Optimization

libopenage/presenter/presenter.cpp

@@ -1,4 +1,4 @@
-// Copyright 2019-2024 the openage authors. See copying.md for legal info.
+// Copyright 2019-2025 the openage authors. See copying.md for legal info.
 
 #include "presenter.h"
 
@@ -166,9 +166,18 @@ void Presenter::init_graphics(const renderer::window_settings &window_settings)
 		this->time_loop->get_clock());
 	this->render_passes.push_back(this->hud_renderer->get_render_pass());
 
+	for (auto &render_pass : render_passes) {
+		render_pass->set_stencil_state(renderer::StencilState::USE_STENCIL_TEST);
+	}
+
 	this->init_gui();
 	this->init_final_render_pass();
 
+	// set passes indices
+	this->index_gui_stencil_pass = this->render_passes.size() - 3;
+	this->index_gui_render_pass = this->render_passes.size() - 2;
+	this->index_final_render_pass = this->render_passes.size() - 1;
+
 	if (this->simulation) {
 		auto render_factory = std::make_shared<renderer::RenderFactory>(this->terrain_renderer, this->world_renderer);
 		this->simulation->attach_renderer(render_factory);
@@ -213,6 +222,9 @@ void Presenter::init_gui() {
 		this->renderer // openage renderer
 	);
 
+	auto stencil_pass = this->gui->get_stencil_pass();
+	this->render_passes.push_back(stencil_pass);
+
 	auto gui_pass = this->gui->get_render_pass();
 	this->render_passes.push_back(gui_pass);
 }
@@ -313,11 +325,18 @@ void Presenter::render() {
 	this->terrain_renderer->update();
 	this->world_renderer->update();
 	this->hud_renderer->update();
+
 	this->gui->render();
+	this->renderer->render(this->render_passes[this->index_gui_stencil_pass]);
 
-	for (auto &pass : this->render_passes) {
-		this->renderer->render(pass);
+	for (size_t i = 0; i < this->index_gui_stencil_pass; ++i) {
+		this->renderer->render(this->render_passes[i]);
 	}
+
+	this->gui->render();
+	this->renderer->render(this->render_passes[this->index_gui_render_pass]);
+
+	this->renderer->render(this->render_passes[this->index_final_render_pass]);
 }
 
 } // namespace openage::presenter

libopenage/presenter/presenter.h

@@ -1,4 +1,4 @@
-// Copyright 2019-2024 the openage authors. See copying.md for legal info.
+// Copyright 2019-2025 the openage authors. See copying.md for legal info.
 
 #pragma once
 
@@ -230,6 +230,13 @@ class Presenter {
 	 * Input manager.
 	 */
 	std::shared_ptr<input::InputManager> input_manager;
+
+	/**
+	 * Pass indices.
+	 */
+	size_t index_gui_stencil_pass;
+	size_t index_gui_render_pass;
+	size_t index_final_render_pass;
 };
 
 } // namespace presenter

libopenage/renderer/gui/gui.cpp

@@ -1,4 +1,4 @@
-// Copyright 2015-2024 the openage authors. See copying.md for legal info.
+// Copyright 2015-2025 the openage authors. See copying.md for legal info.
 
 #include "gui.h"
 
@@ -62,6 +62,10 @@ std::shared_ptr<renderer::RenderPass> GUI::get_render_pass() const {
 	return this->render_pass;
 }
 
+std::shared_ptr<renderer::RenderPass> GUI::get_stencil_pass() const {
+	return this->stencil_pass;
+}
+
 void GUI::initialize_render_pass(size_t width,
                                  size_t height,
                                  const util::Path &shaderdir) {
@@ -98,6 +102,10 @@ void GUI::initialize_render_pass(size_t width,
 	// TODO: Rendering into the FBO is a bit redundant right now because we
 	// just copy the GUI texture into the output texture
 	this->render_pass = renderer->add_render_pass({display_obj}, fbo);
+
+	auto stencil_pass = renderer->add_render_pass({display_obj}, fbo);
+	stencil_pass->set_stencil_state(StencilState::WRITE_STENCIL_MASK);
+	this->stencil_pass = stencil_pass;
 }
 
 

libopenage/renderer/gui/gui.h

@@ -1,4 +1,4 @@
-// Copyright 2015-2024 the openage authors. See copying.md for legal info.
+// Copyright 2015-2025 the openage authors. See copying.md for legal info.
 
 #pragma once
 
@@ -63,6 +63,13 @@ class GUI {
 	 */
 	std::shared_ptr<renderer::RenderPass> get_render_pass() const;
 
+	/**
+	 * Get the stencil render pass of the GUI.
+	 *
+	 * @return stencil render pass of the GUI.
+	 */
+	std::shared_ptr<renderer::RenderPass> get_stencil_pass() const;
+
 	/**
 	 * Render the GUI texture.
 	 */
@@ -139,6 +146,11 @@ class GUI {
 	 * this pass via a \p renderer::resources::Renderable.
 	 */
 	std::shared_ptr<renderer::RenderPass> render_pass;
+
+	/**
+	 * Render pass for the stencil mask of the GUI.
+	 */
+	std::shared_ptr<renderer::RenderPass> stencil_pass;
 };
 
 } // namespace gui

libopenage/renderer/opengl/renderer.cpp

@@ -162,6 +162,30 @@ void GlRenderer::optimize(const std::shared_ptr<GlRenderPass> &pass) {
 	}
 }
 
+void GlRenderer::setupStencilWrite() {
+	glEnable(GL_STENCIL_TEST);
+	glStencilFunc(GL_ALWAYS, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+	glStencilMask(0xFF);
+	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+	glDepthMask(GL_FALSE);
+}
+
+void GlRenderer::setupStencilTest() {
+	glEnable(GL_STENCIL_TEST);
+	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+	glStencilMask(0x00);
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	glDepthMask(GL_TRUE);
+}
+
+void GlRenderer::disableStencilTest() {
+	glDisable(GL_STENCIL_TEST);
+	glStencilMask(0xFF);
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	glDepthMask(GL_TRUE);
+}
+
 void GlRenderer::check_error() {
 	// thanks for the global state, opengl!
 	GlContext::check_error();
@@ -176,7 +200,12 @@ void GlRenderer::render(const std::shared_ptr<RenderPass> &pass) {
 	// see https://www.khronos.org/opengl/wiki/Vertex_Rendering#Causes_of_rendering_failure
 	shared_quad_vao->bind();
 
-	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
+	// ensure that an (empty) VAO is bound before rendering geometry
+	// a bound VAO is required to render bufferless quad geometries by OpenGL
+	// see https://www.khronos.org/opengl/wiki/Vertex_Rendering#Causes_of_rendering_failure
+	shared_quad_vao->bind();
+
+	glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 
 	// TODO: Option for face culling
 	// glEnable(GL_CULL_FACE);
@@ -198,6 +227,20 @@ void GlRenderer::render(const std::shared_ptr<RenderPass> &pass) {
 			glClear(GL_DEPTH_BUFFER_BIT);
 		}
 
+		switch (layer.stencil_state) {
+		case StencilState::WRITE_STENCIL_MASK:
+			setupStencilWrite();
+			break;
+
+		case StencilState::USE_STENCIL_TEST:
+			setupStencilTest();
+			break;
+
+		case StencilState::DISABLE_STENCIL:
+			disableStencilTest();
+			break;
+		}
+
 		for (auto const &obj : objects) {
 			if (obj.alpha_blending) {
 				glEnable(GL_BLEND);

libopenage/renderer/opengl/renderer.h

@@ -63,6 +63,12 @@ class GlRenderer final : public Renderer {
 	/// Optimize the render pass by reordering stuff
 	static void optimize(const std::shared_ptr<GlRenderPass> &pass);
 
+	void setupStencilWrite();
+
+	void setupStencilTest();
+
+	void disableStencilTest();
+
 	/// The GL context.
 	std::shared_ptr<GlContext> gl_context;
 

libopenage/renderer/render_pass.cpp

@@ -1,4 +1,4 @@
-// Copyright 2024-2024 the openage authors. See copying.md for legal info.
+// Copyright 2024-2025 the openage authors. See copying.md for legal info.
 
 #include "render_pass.h"
 
@@ -83,7 +83,7 @@ void RenderPass::add_renderables(Renderable &&renderable, int64_t priority) {
 	this->add_renderables(std::vector<Renderable>{std::move(renderable)}, priority);
 }
 
-void RenderPass::add_layer(int64_t priority, bool clear_depth) {
+void RenderPass::add_layer(int64_t priority, bool clear_depth, StencilState stencil_state) {
 	size_t layer_index = 0;
 	for (const auto &layer : this->layers) {
 		if (layer.priority > priority) {
@@ -92,14 +92,20 @@ void RenderPass::add_layer(int64_t priority, bool clear_depth) {
 		layer_index++;
 	}
 
-	this->add_layer(layer_index, priority, clear_depth);
+	this->add_layer(layer_index, priority, clear_depth, stencil_state);
 }
 
-void RenderPass::add_layer(size_t index, int64_t priority, bool clear_depth) {
-	this->layers.insert(this->layers.begin() + index, Layer{priority, clear_depth});
+void RenderPass::add_layer(size_t index, int64_t priority, bool clear_depth, StencilState stencil_state) {
+	this->layers.insert(this->layers.begin() + index, Layer{priority, clear_depth, stencil_state});
 	this->renderables.insert(this->renderables.begin() + index, std::vector<Renderable>{});
 }
 
+void RenderPass::set_stencil_state(StencilState state) {
+	for (auto &layer : this->layers) {
+		layer.stencil_state = state;
+	}
+}
+
 void RenderPass::clear_renderables() {
 	// Keep layer definitions, but reset the length of each layer to 0
 	for (size_t i = 0; i < this->layers.size(); i++) {

libopenage/renderer/render_pass.h

@@ -1,4 +1,4 @@
-// Copyright 2024-2024 the openage authors. See copying.md for legal info.
+// Copyright 2024-2025 the openage authors. See copying.md for legal info.
 
 #pragma once
 
@@ -14,6 +14,18 @@ namespace openage {
 namespace renderer {
 class RenderTarget;
 
+/**
+ * Stencil states for the render pass.
+ */
+enum class StencilState {
+	/// State for writing GUI elements to stencil buffer.
+	WRITE_STENCIL_MASK,
+	/// State for using the mask when rendering scene.
+	USE_STENCIL_TEST,
+	/// State for normal rendering (GUI rendering).
+	DISABLE_STENCIL
+};
+
 /**
  * Defines a layer in the render pass. A layer is a slice of the renderables
  * that have the same priority. Each layer can have its own settings.
@@ -27,6 +39,8 @@ struct Layer {
 	int64_t priority;
 	/// Whether to clear the depth buffer before rendering this layer.
 	bool clear_depth = true;
+	/// The state of the stencil buffer for the render pass.
+	StencilState stencil_state;
 };
 
 /**
@@ -95,8 +109,16 @@ class RenderPass {
 	 *
 	 * @param priority Priority of the layer. Layers with higher priority are drawn first.
 	 * @param clear_depth If true clears the depth buffer before rendering this layer.
+	 * @param stencil_state State of the stencil buffer, using to do stencil test.
+	 */
+	void add_layer(int64_t priority, bool clear_depth = true, StencilState stencil_state = StencilState::DISABLE_STENCIL);
+
+	/**
+	 * Set the stencil state for the render pass.
+	 *
+	 * @param state The new stencil state.
 	 */
-	void add_layer(int64_t priority, bool clear_depth = true);
+	void set_stencil_state(StencilState state);
 
 	/**
 	 * Clear the list of renderables
@@ -138,8 +160,9 @@ class RenderPass {
 	 * @param index Index in \p layers member to insert the new layer.
 	 * @param priority Priority of the layer. Layers with higher priority are drawn first.
 	 * @param clear_depth If true clears the depth buffer before rendering this layer.
+	 * @param stencil_state State of the stencil buffer, using to do stencil test.
 	 */
-	void add_layer(size_t index, int64_t priority, bool clear_depth = true);
+	void add_layer(size_t index, int64_t priority, bool clear_depth = true, StencilState stencil_state = StencilState::DISABLE_STENCIL);
 
 	/**
 	 * Render target to write to.