Skip to content

Added support for NBTest, for testing notebooks with assertions in the Multi-software test CI pipeline #1772

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: develop
Choose a base branch
from
Open

Added support for NBTest, for testing notebooks with assertions in the Multi-software test CI pipeline #1772

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ab7591d
Added NBTest assertions in the test notebooks, and modified the CI an…
vrnimje Jul 24, 2025
0e69fea
Reordered shell commands in Makefile to test notebooks first
vrnimje Jul 24, 2025
59d1978
Removing nbtest branch from CI activation, which is no longer needed
vrnimje Jul 24, 2025
23c85a4
Added coverage shell commands for nbtest
vrnimje Jul 25, 2025
e58d93c
Removed nbval from test dependencies
vrnimje Jul 25, 2025
57db2d7
Reverting spurious formatting changes
vrnimje Jul 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,4 +169,3 @@ jobs:
make install
pip install ".[test]"
make test_spark

4 changes: 2 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,19 @@ docs:
mkdocs build

test:
pytest --nbtest tests/notebooks/
pytest tests/unit/
pytest tests/issues/
pytest --nbval tests/notebooks/
ydata_profiling -h

test_spark:
pytest tests/backends/spark_backend/
ydata_profiling -h

test_cov:
pytest --cov=. --cov-append --nbtest tests/notebooks/
pytest --cov=. tests/unit/
pytest --cov=. --cov-append tests/issues/
pytest --cov=. --cov-append --nbval tests/notebooks/
ydata_profiling -h

examples:
Expand Down
2 changes: 1 addition & 1 deletion make.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ IF "%1%" == "docs" (
IF "%1" == "test" (
pytest tests/unit/
pytest tests/issues/
pytest --nbval tests/notebooks/
pytest --nbtest tests/notebooks/
ECHO "Tests completed!"
GOTO end
)
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ test = [
"coverage>=6.5, <8",
"codecov",
"pytest-cov",
"nbval",
"nbtest-gen",
"pyarrow",
"twine>=3.1.1",
"kaggle",
Expand Down Expand Up @@ -150,4 +150,4 @@ ydata_profiling = ["py.typed"]
universal = true

[tool.setuptools.package-dir]
"" = "src"
"" = "src"
60 changes: 39 additions & 21 deletions tests/notebooks/lazy_pipeline.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,15 @@
"\n",
"# Our package\n",
"from ydata_profiling import ProfileReport\n",
"from ydata_profiling.utils.cache import cache_file"
"from ydata_profiling.utils.cache import cache_file\n",
"\n",
"import nbtest\n",
"\n",
"# The tests in this notebook only run in the continuous integration pipeline\n",
"# in order to run manually uncomment the following two lines:\n",
"\n",
"# import os\n",
"# os.environ['NBTEST_RUN_ASSERTS'] = '1'"
]
},
{
Expand All @@ -48,10 +56,8 @@
"with capture_output() as out:\n",
" profile = ProfileReport(df, title=\"Titanic Dataset\", progress_bar=True, lazy=False)\n",
"\n",
"assert all(\n",
" any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs\n",
")\n",
"assert len(out.outputs) == 2"
"nbtest.assert_true(all(any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs))\n",
"nbtest.assert_equal(len(out.outputs), 2)"
]
},
{
Expand All @@ -69,22 +75,20 @@
" lazy=True,\n",
" )\n",
"\n",
"assert len(out.outputs) == 0\n",
"nbtest.assert_equal(len(out.outputs), 0)\n",
"\n",
"with capture_output() as out:\n",
" _ = profile.to_html()\n",
"\n",
"\n",
"assert all(\n",
" any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs\n",
")\n",
"assert len(out.outputs) == 3\n",
"nbtest.assert_true(all(any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs))\n",
"nbtest.assert_equal(len(out.outputs), 3)\n",
"\n",
"with capture_output() as out:\n",
" _ = profile.to_file(\"/tmp/tmpfile.html\")\n",
"\n",
"assert \"Export report to file\" in out.outputs[0].data[\"text/plain\"]\n",
"assert len(out.outputs) == 1"
"nbtest.assert_in(\"Export report to file\", out.outputs[0].data[\"text/plain\"])\n",
"nbtest.assert_equal(len(out.outputs), 1)"
]
},
{
Expand All @@ -96,46 +100,60 @@
"# Test caching of the iterative building process\n",
"with capture_output() as out:\n",
" profile = ProfileReport(df, title=\"Titanic Dataset\", progress_bar=True, lazy=True)\n",
"assert len(out.outputs) == 0\n",
"nbtest.assert_equal(len(out.outputs), 0)\n",
"\n",
"with capture_output() as out:\n",
" profile.description_set\n",
"assert len(out.outputs) == 1\n",
"nbtest.assert_equal(len(out.outputs), 1)\n",
"\n",
"with capture_output() as out:\n",
" profile.report\n",
"assert len(out.outputs) == 1\n",
"nbtest.assert_equal(len(out.outputs), 1)\n",
"\n",
"with capture_output() as out:\n",
" profile.html\n",
"assert len(out.outputs) == 1\n",
"nbtest.assert_equal(len(out.outputs), 1)\n",
"\n",
"with capture_output() as out:\n",
" profile.config.html.style.theme = \"united\"\n",
" profile.invalidate_cache(\"rendering\")\n",
" profile.to_file(\"/tmp/cache1.html\")\n",
"assert len(out.outputs) == 2\n",
"nbtest.assert_equal(len(out.outputs), 2)\n",
"\n",
"with capture_output() as out:\n",
" profile.config.pool_size = 1\n",
" profile.html\n",
"assert len(out.outputs) == 0\n",
"nbtest.assert_equal(len(out.outputs), 0)\n",
"\n",
"with capture_output() as out:\n",
" profile.config.pool_size = 0\n",
" profile.config.samples.head = 5\n",
" profile.config.samples.tail = 15\n",
" profile.invalidate_cache()\n",
" profile.to_file(\"/tmp/cache2.html\")\n",
"assert len(out.outputs) == 4"
"nbtest.assert_equal(len(out.outputs), 4)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python"
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
38 changes: 21 additions & 17 deletions tests/notebooks/meteorites.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,15 @@
"from IPython.utils.capture import capture_output\n",
"\n",
"import ydata_profiling\n",
"from ydata_profiling.utils.cache import cache_file"
"from ydata_profiling.utils.cache import cache_file\n",
"\n",
"import nbtest\n",
"\n",
"# The tests in this notebook only run in the continuous integration pipeline\n",
"# in order to run manually uncomment the following two lines:\n",
"\n",
"# import os\n",
"# os.environ['NBTEST_RUN_ASSERTS'] = '1'"
]
},
{
Expand Down Expand Up @@ -83,9 +91,9 @@
" )\n",
" display(pr)\n",
"\n",
"assert len(out.outputs) == 2\n",
"assert out.outputs[0].data[\"text/plain\"] == \"<IPython.core.display.HTML object>\"\n",
"assert out.outputs[1].data[\"text/plain\"] == \"\""
"nbtest.assert_equal(len(out.outputs), 2)\n",
"nbtest.assert_equal(out.outputs[0].data[\"text/plain\"], \"<IPython.core.display.HTML object>\")\n",
"nbtest.assert_equal(out.outputs[1].data[\"text/plain\"], \"\")"
]
},
{
Expand All @@ -103,10 +111,8 @@
" lazy=False,\n",
" )\n",
"\n",
"assert all(\n",
" any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs\n",
")\n",
"assert len(out.outputs) == 2"
"nbtest.assert_true(all(any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs))\n",
"nbtest.assert_equal(len(out.outputs), 2)"
]
},
{
Expand All @@ -119,10 +125,8 @@
"with capture_output() as out:\n",
" pfr.to_file(\"/tmp/example.html\")\n",
"\n",
"assert all(\n",
" any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs\n",
")\n",
"assert len(out.outputs) == 2"
"nbtest.assert_true(all(any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs))\n",
"nbtest.assert_equal(len(out.outputs), 2)"
]
},
{
Expand All @@ -135,9 +139,9 @@
"with capture_output() as out:\n",
" display(pfr)\n",
"\n",
"assert len(out.outputs) == 2\n",
"assert out.outputs[0].data[\"text/plain\"] == \"<IPython.core.display.HTML object>\"\n",
"assert out.outputs[1].data[\"text/plain\"] == \"\""
"nbtest.assert_equal(len(out.outputs), 2)\n",
"nbtest.assert_equal(out.outputs[0].data[\"text/plain\"], \"<IPython.core.display.HTML object>\")\n",
"nbtest.assert_equal(out.outputs[1].data[\"text/plain\"], \"\")"
]
}
],
Expand All @@ -157,9 +161,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
39 changes: 23 additions & 16 deletions tests/notebooks/titanic.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,15 @@
"from ipywidgets import widgets\n",
"\n",
"from ydata_profiling import ProfileReport\n",
"from ydata_profiling.utils.cache import cache_file"
"from ydata_profiling.utils.cache import cache_file\n",
"\n",
"import nbtest\n",
"\n",
"# The tests in this notebook only run in the continuous integration pipeline\n",
"# in order to run manually uncomment the following two lines:\n",
"\n",
"# import os\n",
"# os.environ['NBTEST_RUN_ASSERTS'] = '1'"
]
},
{
Expand Down Expand Up @@ -54,10 +62,9 @@
" lazy=False,\n",
" )\n",
"\n",
"assert all(\n",
" any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs\n",
")\n",
"assert len(out.outputs) == 2"
"nbtest.assert_true(all(any(v in s.data[\"text/plain\"] for v in [\"%|\", \"FloatProgress\"]) for s in out.outputs))\n",
"\n",
"nbtest.assert_equal(len(out.outputs), 2)"
]
},
{
Expand All @@ -76,7 +83,7 @@
" lazy=False,\n",
" )\n",
"\n",
"assert len(out.outputs) == 0"
"nbtest.assert_equal(len(out.outputs), 0)"
]
},
{
Expand All @@ -87,14 +94,14 @@
"source": [
"# Waiting on issue: https://github.com/computationalmodelling/nbval/issues/136\n",
"\n",
"# The Notebook Widgets Interface\n",
"# The Notebook Widgets Interface - faced execution error here, hence, the tests below have been commented\n",
"# with capture_output() as out:\n",
"# profile.to_widgets()\n",
"\n",
"# assert len(out.outputs) == 2\n",
"# assert out.outputs[0].data['text/plain'].startswith('Tab(children=(HTML(value=')\n",
"# assert out.outputs[1].data['text/plain'] == '<IPython.display.HTML object>'\n",
"# assert 'ydata-profiling' in out.outputs[1].data['text/html']"
"# nbtest.assert_equal(len(out.outputs), 2)\n",
"# nbtest.assert_true(out.outputs[0].data['text/plain'].startswith('Tab(children=(HTML(value='))\n",
"# nbtest.assert_equal(out.outputs[1].data['text/plain'], '<IPython.display.HTML object>')\n",
"# nbtest.assert_in('ydata-profiling', out.outputs[1].data['text/html'])"
]
},
{
Expand All @@ -107,14 +114,14 @@
"with capture_output() as out:\n",
" profile.to_notebook_iframe()\n",
"\n",
"assert len(out.outputs) == 1\n",
"assert out.outputs[0].data[\"text/plain\"] == \"<IPython.core.display.HTML object>\""
"nbtest.assert_equal(len(out.outputs), 1)\n",
"nbtest.assert_equal(out.outputs[0].data[\"text/plain\"], \"<IPython.core.display.HTML object>\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand All @@ -128,9 +135,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.6"
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}