From a982f50f5575fbaaa67eb3e144a7c3e7955806c3 Mon Sep 17 00:00:00 2001 From: Chris B Date: Thu, 11 Jun 2020 12:37:49 +0100 Subject: [PATCH 1/4] Add test of magics handling. --- tests/test_magics.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/test_magics.py diff --git a/tests/test_magics.py b/tests/test_magics.py new file mode 100644 index 0000000..918d884 --- /dev/null +++ b/tests/test_magics.py @@ -0,0 +1,42 @@ +import os + +import nbformat + +import pytest + +from utils import build_nb + + +pytest_plugins = "pytester" + +@pytest.mark.parametrize("magic, expected_passes", [ + (r"%dirs", [True]), + (r"%this_magic_does_not_exist", [False]) +]) +def test_magics(testdir, magic, expected_passes): + # Setup notebook to test: + sources = [ + # In [1]: + magic, + ] + nb = build_nb(sources) + + # Write notebook to test dir + nbformat.write(nb, os.path.join( + str(testdir.tmpdir), 'test_magics.ipynb')) + + # Run tests + result = testdir.inline_run('--nbval-lax', '--current-env', '-s') + reports = result.getreports('pytest_runtest_logreport') + + # Setup and teardown of cells should have no issues: + setup_teardown = [r for r in reports if r.when != 'call'] + for r in setup_teardown: + assert r.passed + + reports = [r for r in reports if r.when == 'call'] + + assert len(reports) == len(expected_passes) + + for actual_report, expected_pass in zip(reports, expected_passes): + assert actual_report.passed is expected_pass From f45a39f3e1b46d67dfd4cad66bb1d39230e77e11 Mon Sep 17 00:00:00 2001 From: Chris B Date: Thu, 11 Jun 2020 14:31:31 +0100 Subject: [PATCH 2/4] Add another type of bad magic to test --- tests/test_magics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_magics.py b/tests/test_magics.py index 918d884..b64fddf 100644 --- a/tests/test_magics.py +++ b/tests/test_magics.py @@ -11,6 +11,7 @@ @pytest.mark.parametrize("magic, expected_passes", [ (r"%dirs", [True]), + (r"%precision bad", [False]), (r"%this_magic_does_not_exist", [False]) ]) def test_magics(testdir, magic, expected_passes): From ebb1d6743b9b4345c2bcdeb509b988930e41ebd8 Mon Sep 17 00:00:00 2001 From: Chris B Date: Thu, 11 Jun 2020 14:32:19 +0100 Subject: [PATCH 3/4] Have pip summarize what's installed before running the tests. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e091143..fcdb216 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ install: fi - pip install doit - doit install_test_deps + - pip freeze # command to run tests script: doit test From 14e00b1340f105441a0b6e34ffe412c97db48add Mon Sep 17 00:00:00 2001 From: Chris B Date: Fri, 11 Sep 2020 15:13:05 +0100 Subject: [PATCH 4/4] Use subprocess for running notebooks. --- tests/test_magics.py | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/tests/test_magics.py b/tests/test_magics.py index b64fddf..fe58986 100644 --- a/tests/test_magics.py +++ b/tests/test_magics.py @@ -9,35 +9,25 @@ pytest_plugins = "pytester" -@pytest.mark.parametrize("magic, expected_passes", [ - (r"%dirs", [True]), - (r"%precision bad", [False]), - (r"%this_magic_does_not_exist", [False]) +@pytest.mark.parametrize("magic, expected_pass", [ + (r"%dirs", True), + (r"%precision bad", False), + (r"%this_magic_does_not_exist", False) ]) -def test_magics(testdir, magic, expected_passes): - # Setup notebook to test: - sources = [ +def test_magics(testdir, magic, expected_pass): + nb = build_nb([ # In [1]: magic, - ] - nb = build_nb(sources) - - # Write notebook to test dir + ]) + nb_name = 'test_magics' nbformat.write(nb, os.path.join( - str(testdir.tmpdir), 'test_magics.ipynb')) - - # Run tests - result = testdir.inline_run('--nbval-lax', '--current-env', '-s') - reports = result.getreports('pytest_runtest_logreport') - - # Setup and teardown of cells should have no issues: - setup_teardown = [r for r in reports if r.when != 'call'] - for r in setup_teardown: - assert r.passed + str(testdir.tmpdir), nb_name+".ipynb")) - reports = [r for r in reports if r.when == 'call'] + # using subprocess because otherwise second and subsequent tests always fail (some state left over somewhere in the jupyter stack) + result = testdir.runpytest_subprocess('--nbval-lax', '--current-env', '-s', '-v') - assert len(reports) == len(expected_passes) + assert result.ret == (not expected_pass) - for actual_report, expected_pass in zip(reports, expected_passes): - assert actual_report.passed is expected_pass + result.stdout.fnmatch_lines_random( + ["*collected 1 item*", + "{nb_name}::ipynb::Cell 0 {result}".format(nb_name=nb_name, result="PASSED" if expected_pass else "FAILED")])