Merge branch 'devel' into fix/3269-fixes-athena-refresh-mode

Author: rudolfix

.github/workflows/build_docs.yml

@@ -21,16 +21,14 @@ jobs:
         with:
           node-version: '22'
 
-      - name: Setup Python
-        uses: actions/setup-python@v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
           python-version: "3.11"
+          activate-environment: true
 
       - name: Install node dependencies
         run: cd docs/website && npm install
 
       - name: Install python dependencies
-        run: cd docs/website && pip install -r requirements.txt
-
-      - name: Build docs
-        run: cd docs/website && npm run build:cloudflare
+        run: cd docs/website && npm run build

.github/workflows/main.yml

@@ -27,9 +27,10 @@ jobs:
 
   # testing and linting of docs snippets is always run to catch problems in the docs
   # NOTE: we could splint linting and testing of docs so linter can also always run for fork PRs
-  test_docs_snippets:
-    name: test snippets in docs
-    uses: ./.github/workflows/test_docs_snippets.yml
+  test_docs:
+    name: lint and test snippets, examples and notebooks in docs, lints tools
+    uses: ./.github/workflows/test_docs.yml
+    secrets: inherit
 
   # NOTE: we build docs the same way as on cloudflare, so we can catch problems early
   build_docs:
@@ -86,12 +87,6 @@ jobs:
   # so we depend on authorize and forward secrets
   #
 
-  test_examples:
-    name: test examples
-    needs: [lint, authorize_run_from_fork]
-    uses: ./.github/workflows/test_examples.yml
-    secrets: inherit
-
   test_destinations_remote:
     name: test remote destinations with secrets
     needs: [authorize_run_from_fork, test_common]

.github/workflows/test_docs_snippets.yml renamed to .github/workflows/test_docs.yml

@@ -1,5 +1,5 @@
 
-name: docs | snippets
+name: docs | lint and test tools, snippets and examples
 
 on:
   workflow_call:
@@ -12,6 +12,16 @@ env:
   # detect if the workflow is executed in a repo fork
   IS_FORK: ${{ github.event.pull_request.head.repo.fork }}
 
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
+  # RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+
+  # Slack hook for chess in production example
+  RUNTIME__SLACK_INCOMING_HOOK: ${{ secrets.RUNTIME__SLACK_INCOMING_HOOK }}
+  # Path to local qdrant database
+  DESTINATION__QDRANT__CREDENTIALS__PATH: zendesk.qdb
+
+
 jobs:
 
   run_lint:
@@ -65,24 +75,34 @@ jobs:
           activate-environment: true
           enable-cache: true
 
+      - name: Start weaviate
+        run:  docker compose -f "tests/load/weaviate/docker-compose.yml" up -d
+
+
       - name: Install dependencies
-        run: uv sync --extra duckdb --extra parquet --extra postgres --extra s3 --extra workspace --group docs --group sentry-sdk --group ibis --group providers
+        run: cd docs && make dev
 
       - name: Install dlthub incl alpha releases
-        run: uv run pip install --pre dlthub
+        run: cd docs && uv run pip install --pre dlthub
 
-      - name: run docs preprocessor
-        run: make preprocess-docs
+      - name: lint docs
+        run: cd docs && make lint
+
+      - name: lint embedded snippets
+        run: cd docs && make lint-embedded-snippets
 
       - name: Create secrets.toml for snippets
         run: |
           cp tests/.dlt/dev.secrets.toml docs/website/docs/.dlt/secrets.toml
 
-      - name: Run linter and tests on snippets
-        run: make lint-and-test-snippets
+      - name: test snippets
+        run: cd docs && make test-snippets
 
-      - name: Test notebooks
-        run: cd docs/notebooks && make test
+      - name: create secrets.toml for examples
+        run: pwd && echo "$DLT_SECRETS_TOML" > docs/examples/.dlt/secrets.toml
 
-      - name: Lint education notebooks
-        run: make lint-notebooks
+      - name: test examples
+        run: cd docs && make test-examples
+
+      - name: run docs preprocessor
+        run: cd docs && make preprocess-docs

Makefile

@@ -44,10 +44,10 @@ has-uv:
 	uv --version
 
 dev: has-uv
-	uv sync --all-extras --group docs --group dev --group providers --group pipeline --group sources --group sentry-sdk --group ibis --group adbc --group dashboard-tests
+	uv sync --all-extras --group dev --group providers --group pipeline --group sources --group sentry-sdk --group ibis --group adbc --group dashboard-tests
 
 dev-airflow: has-uv
-	uv sync --all-extras --group docs --group providers --group pipeline --group sources --group sentry-sdk --group ibis --group airflow
+	uv sync --all-extras --group providers --group pipeline --group sources --group sentry-sdk --group ibis --group airflow
 
 lint: lint-core lint-security lint-docstrings
 
@@ -60,45 +60,14 @@ lint-core:
 	# NOTE: we exclude all D lint errors (docstrings)
 	uv run flake8 --extend-ignore=D --max-line-length=200 dlt
 	uv run flake8 --extend-ignore=D --max-line-length=200 tests --exclude tests/reflection/module_cases,tests/common/reflection/cases/modules/
-	uv run black dlt docs tests --check --diff --color --extend-exclude=".*syntax_error.py"
 
 format:
-	uv run black dlt docs tests --extend-exclude='.*syntax_error.py|_storage/.*'
-	uv run black docs/education --ipynb --extend-exclude='.*syntax_error.py|_storage/.*'
-
-lint-snippets:
-	cd docs/tools && uv run python check_embedded_snippets.py full
-	# TODO: re-enable transformation snippets tests when dlthub dep is available
-	uv pip install docstring_parser_fork --reinstall
-	uv run mypy --config-file mypy.ini docs/website docs/tools --exclude docs/tools/lint_setup --exclude docs/website/docs_processed --exclude docs/website/versioned_docs/
-	uv run ruff check
-	uv run flake8 --max-line-length=200 docs/website docs/tools --exclude docs/website/.dlt-repo,docs/website/node_modules
-
-lint-and-test-snippets: lint-snippets
-	cd docs/website/docs && uv run pytest --ignore=node_modules
-
-lint-and-test-examples:
-	uv pip install docstring_parser_fork --reinstall
-	cd docs/tools && uv run python prepare_examples_tests.py
-	uv run ruff check
-	uv run flake8 --max-line-length=200 docs/examples
-	uv run mypy --config-file mypy.ini docs/examples
-	cd docs/examples && uv run pytest
-
-test-examples:
-	cd docs/examples && uv run pytest
+	uv run black dlt tests --extend-exclude='.*syntax_error.py|_storage/.*'
 
 lint-security:
 	# go for ll by cleaning up eval and SQL warnings.
 	uv run bandit -r dlt/ -n 3 -lll
 
-lint-notebooks:
-	uv run nbqa flake8 docs/education --extend-ignore=D,F704 --max-line-length=200
-	uv run nbqa mypy docs/education \
-	--ignore-missing-imports \
-	--disable-error-code=no-redef \
-	--disable-error-code=top-level-await
-
 # check docstrings for all important public classes and functions
 lint-docstrings:
 	uv run flake8 --count \
@@ -146,10 +115,6 @@ test-build-images: build-library
 	docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell uv version --short)" .
 	docker build -f deploy/dlt/Dockerfile.minimal --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell uv version --short)" .
 
-preprocess-docs:
-	# run docs preprocessing to run a few checks and ensure examples can be parsed
-	cd docs/website && npm i && npm run preprocess-docs
-
 start-test-containers:
 	docker compose -f "tests/load/dremio/docker-compose.yml" up -d
 	docker compose -f "tests/load/postgres/docker-compose.yml" up -d

dlt/_workspace/helpers/dashboard/dlt_dashboard.py

@@ -1,3 +1,7 @@
+# /// script
+# [tool.marimo.display]
+# theme = "light"
+# ///
 # flake8: noqa: F841
 # mypy: disable-error-code=no-untyped-def
 
@@ -103,20 +107,32 @@ def home(
     else:
         _buttons: List[Any] = []
         _buttons.append(dlt_refresh_button)
+        _pipeline_execution_exception: List[Any] = []
+        _pipeline_execution_summary: mo.Html = None
+        _last_load_packages_info: mo.Html = None
         if dlt_pipeline:
             _buttons.append(
                 mo.ui.button(
                     label="<small>Open pipeline working dir</small>",
                     on_click=lambda _: utils.open_local_folder(dlt_pipeline.working_dir),
                 )
             )
-        if dlt_pipeline and (local_dir := utils.get_local_data_path(dlt_pipeline)):
-            _buttons.append(
-                mo.ui.button(
-                    label="<small>Open local data location</small>",
-                    on_click=lambda _: utils.open_local_folder(local_dir),
+            if local_dir := utils.get_local_data_path(dlt_pipeline):
+                _buttons.append(
+                    mo.ui.button(
+                        label="<small>Open local data location</small>",
+                        on_click=lambda _: utils.open_local_folder(local_dir),
+                    )
                 )
-            )
+            if trace := dlt_pipeline.last_trace:
+                _pipeline_execution_summary = utils.build_pipeline_execution_visualization(trace)
+                _last_load_packages_info = mo.vstack(
+                    [
+                        mo.md(f"<small>{strings.view_load_packages_text}</small>"),
+                        utils.load_package_status_labels(trace),
+                    ]
+                )
+            _pipeline_execution_exception = utils.build_exception_section(dlt_pipeline)
         _stack = [
             mo.vstack(
                 [
@@ -182,10 +198,16 @@ def home(
                         ],
                         justify="space-between",
                     ),
-                    mo.hstack(_buttons, justify="start"),
                 ]
-            )
+            ),
+            mo.hstack(_buttons, justify="start"),
         ]
+        if _pipeline_execution_summary:
+            _stack.append(_pipeline_execution_summary)
+        if _last_load_packages_info:
+            _stack.append(_last_load_packages_info)
+        if _pipeline_execution_exception:
+            _stack.extend(_pipeline_execution_exception)
         if not dlt_pipeline and dlt_pipeline_name:
             _stack.append(
                 mo.callout(
@@ -195,7 +217,6 @@ def home(
                     kind="warn",
                 )
             )
-
     mo.vstack(_stack)
     return (dlt_pipeline,)
 
@@ -221,8 +242,6 @@ def section_overview(
     )
 
     if dlt_pipeline and dlt_section_overview_switch.value:
-        if _exception_section := utils.build_exception_section(dlt_pipeline):
-            _result.extend(_exception_section)
         _result += [
             mo.ui.table(
                 utils.pipeline_details(dlt_config, dlt_pipeline, dlt_pipelines_dir),

dlt/_workspace/helpers/dashboard/dlt_dashboard_styles.css

@@ -7,12 +7,22 @@
   --dlt-color-lime: #c6d300;
   --dlt-color-aqua: #59c1d5;
   --dlt-color-purple: #191937;
+  --dlt-color-pink: #EE746D;
 
   /* background colors */
   --dlt-color-lime-background: #c6d30006;
   --dlt-color-aqua-background: #59c1d503;
   --dlt-color-purple-background: #19193706;
 
+  /* badge colors */
+  --grey-bg: #E0E0E0;
+  --grey-text: #555555;
+  --yellow-bg: #F2D6A2;
+  --yellow-text: #AD7F37;
+  --green-bg: #a8e4ac;
+  --green-text: #468A47;
+  --red-bg: #D47672;
+  --red-text: #FBE8E8;
 }
 
 /* Make headings bold */
@@ -87,8 +97,82 @@
   margin-top: 0.5rem;
 }
 
-
 marimo-callout-output .border {
   margin-top: 0.5rem;
   margin-bottom: 0.5rem;
+}
+
+/* Pipeline execution visualization styles */
+.pipeline-execution-container {
+  padding-top: 16px;
+  padding-bottom: 10px;
+}
+
+.pipeline-execution-layout {
+  display: flex;
+  flex-direction: row;
+  justify-content: space-between;
+  align-items: center;
+  gap: 16px;
+}
+
+.pipeline-execution-info {
+  display: flex;
+  flex-direction: column;
+}
+
+.pipeline-execution-timeline {
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  flex: 0 0 40%;
+}
+
+.pipeline-execution-timeline-bar {
+  display: flex;
+  flex-direction: row;
+  justify-content: center;
+  height: 16px;
+}
+
+/* Note: .pipeline-execution-timeline-segment uses inline styles for dynamic width, color, and border-radius */
+
+.pipeline-execution-labels {
+  display: flex;
+  flex-direction: row;
+  justify-content: space-between;
+}
+
+.pipeline-execution-badges {
+  display: flex;
+  flex-direction: row;
+  justify-content: space-between;
+  gap: 16px;
+}
+
+/* Status badge styles */
+.status-badge {
+  padding: 6px 16px;
+  border-radius: 6px;
+  display: inline-block;
+}
+
+.status-badge-yellow {
+  background-color: var(--yellow-bg);
+  color: var(--yellow-text);
+}
+
+.status-badge-green {
+  background-color: var(--green-bg);
+  color: var(--green-text);
+}
+
+.status-badge-red {
+  background-color: var(--red-bg);
+  color: var(--red-text);
+}
+
+.status-badge-grey {
+  background-color: var(--grey-bg);
+  color: var(--grey-text);
 }

dlt/_workspace/helpers/dashboard/strings.py

@@ -90,6 +90,9 @@
 ## Pipeline `{}`
 """
 
+view_load_packages_text = "Status of load packages from last execution"
+
+
 #
 # Overview section
 #