From 3828bc2d649e84133c9f059be5bd98a7b4e01381 Mon Sep 17 00:00:00 2001 From: Stephen Nancekivell Date: Wed, 9 Oct 2024 11:29:19 +1100 Subject: [PATCH 1/2] dont drop tables in incremental append runs --- .../materializations/incremental/incremental.sql | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql b/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql index c4dcb1b8..2afca102 100644 --- a/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql +++ b/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql @@ -20,9 +20,14 @@ {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%} {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} - - {{ drop_relation_if_exists(preexisting_intermediate_relation) }} - {{ drop_relation_if_exists(preexisting_backup_relation) }} + {% set incremental_strategy = adapter.calculate_incremental_strategy(config.get('incremental_strategy')) %} + + {% if not full_refresh_mode and incremental_strategy == 'append' %} + -- skip dropping tables for incremental append runs. + {% else %} + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + {% endif %} {{ run_hooks(pre_hooks, inside_transaction=False) }} {{ run_hooks(pre_hooks, inside_transaction=True) }} @@ -55,7 +60,6 @@ {% else %} {% set column_changes = none %} - {% set incremental_strategy = adapter.calculate_incremental_strategy(config.get('incremental_strategy')) %} {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} {%- if on_schema_change != 'ignore' %} {%- set column_changes = adapter.check_incremental_schema_changes(on_schema_change, existing_relation, sql) -%} From 84fafc96b2a2e087124b62e6a61ee72734d03de3 Mon Sep 17 00:00:00 2001 From: Stephen Nancekivell Date: Thu, 13 Feb 2025 14:37:48 +1100 Subject: [PATCH 2/2] add feature flag --- .../macros/materializations/incremental/incremental.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql b/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql index 2afca102..868ea6a7 100644 --- a/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql +++ b/dbt/include/clickhouse/macros/materializations/incremental/incremental.sql @@ -22,7 +22,7 @@ {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} {% set incremental_strategy = adapter.calculate_incremental_strategy(config.get('incremental_strategy')) %} - {% if not full_refresh_mode and incremental_strategy == 'append' %} + {% if not full_refresh_mode and incremental_strategy == 'append' and config.get('incremental_append_skip_drop_table', default='false') == 'true' %} -- skip dropping tables for incremental append runs. {% else %} {{ drop_relation_if_exists(preexisting_intermediate_relation) }}