From 0c5b9ee646a63bab79b32200a5d32f84aaa08f04 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 12:56:18 +0300 Subject: [PATCH 01/28] entry points --- notebooker/_entrypoints.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/notebooker/_entrypoints.py b/notebooker/_entrypoints.py index 23f0d58..e1b729e 100644 --- a/notebooker/_entrypoints.py +++ b/notebooker/_entrypoints.py @@ -70,6 +70,12 @@ def filesystem_default_value(dirname): is_flag=True, help="If selected, notebooker set current working directory to absolute path of the notebook to keep it local context available", ) +@click.option( + "--categorization", + default=False, + is_flag=True, + help="If selected, discovers only templates with the 'category=example' tags set to any cell and groups notebooks by their category names", +) @click.option( "--default-mailfrom", default=DEFAULT_MAILFROM_ADDRESS, help="Set a new value for the default mailfrom setting." ) @@ -91,6 +97,7 @@ def base_notebooker( py_template_subdir, notebooker_disable_git, execute_at_origin, + categorization, default_mailfrom, running_timeout, serializer_cls, @@ -106,6 +113,7 @@ def base_notebooker( PY_TEMPLATE_SUBDIR=py_template_subdir, NOTEBOOKER_DISABLE_GIT=notebooker_disable_git, EXECUTE_AT_ORIGIN=execute_at_origin, + CATEGORIZATION=categorization, DEFAULT_MAILFROM=default_mailfrom, RUNNING_TIMEOUT=running_timeout, ) @@ -180,6 +188,7 @@ def start_webapp( @base_notebooker.command() @click.option("--report-name", help="The name of the template to execute, relative to the template directory.") +@click.option("--category", default="", help="Category of the template.") @click.option( "--overrides-as-json", default="{}", help="The parameters to inject into the notebook template, in JSON format." ) @@ -230,6 +239,7 @@ def start_webapp( def execute_notebook( config: BaseConfig, report_name, + category, overrides_as_json, iterate_override_values_of, report_title, @@ -250,6 +260,7 @@ def execute_notebook( return execute_notebook_entrypoint( config, report_name, + category, overrides_as_json, iterate_override_values_of, report_title, From c8e81f447a6851c5e60036be4eeadf40ca6e31d4 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 12:58:11 +0300 Subject: [PATCH 02/28] add templates --- tests/unit/utils/templates/cat1_nb.ipynb | 41 +++++++++++++++++++ tests/unit/utils/templates/cat2_nb.ipynb | 41 +++++++++++++++++++ tests/unit/utils/templates/no_cat.ipynb | 41 +++++++++++++++++++ .../templates/subdir/cat1_nb_subdir.ipynb | 41 +++++++++++++++++++ .../templates/subdir/cat2_nb_subdir.ipynb | 41 +++++++++++++++++++ .../templates/subdir/no_cat_nb_subdir.ipynb | 41 +++++++++++++++++++ 6 files changed, 246 insertions(+) create mode 100644 tests/unit/utils/templates/cat1_nb.ipynb create mode 100644 tests/unit/utils/templates/cat2_nb.ipynb create mode 100644 tests/unit/utils/templates/no_cat.ipynb create mode 100644 tests/unit/utils/templates/subdir/cat1_nb_subdir.ipynb create mode 100644 tests/unit/utils/templates/subdir/cat2_nb_subdir.ipynb create mode 100644 tests/unit/utils/templates/subdir/no_cat_nb_subdir.ipynb diff --git a/tests/unit/utils/templates/cat1_nb.ipynb b/tests/unit/utils/templates/cat1_nb.ipynb new file mode 100644 index 0000000..082a65f --- /dev/null +++ b/tests/unit/utils/templates/cat1_nb.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aab4d036-a4e4-4ab0-bb49-6aceaff8525a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": ["category=cat1"] + }, + "outputs": [], + "source": [ + "Test notebook without notebooker tag" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/unit/utils/templates/cat2_nb.ipynb b/tests/unit/utils/templates/cat2_nb.ipynb new file mode 100644 index 0000000..bc37ab0 --- /dev/null +++ b/tests/unit/utils/templates/cat2_nb.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aab4d036-a4e4-4ab0-bb49-6aceaff8525a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": ["category=cat2","parameters"] + }, + "outputs": [], + "source": [ + "Test notebook without notebooker tag" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/unit/utils/templates/no_cat.ipynb b/tests/unit/utils/templates/no_cat.ipynb new file mode 100644 index 0000000..dacdb67 --- /dev/null +++ b/tests/unit/utils/templates/no_cat.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aab4d036-a4e4-4ab0-bb49-6aceaff8525a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "Test notebook without notebooker tag" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/unit/utils/templates/subdir/cat1_nb_subdir.ipynb b/tests/unit/utils/templates/subdir/cat1_nb_subdir.ipynb new file mode 100644 index 0000000..81f5f8f --- /dev/null +++ b/tests/unit/utils/templates/subdir/cat1_nb_subdir.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aab4d036-a4e4-4ab0-bb49-6aceaff8525a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": ["parameters","category=cat1"] + }, + "outputs": [], + "source": [ + "Test notebook without notebooker tag" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/unit/utils/templates/subdir/cat2_nb_subdir.ipynb b/tests/unit/utils/templates/subdir/cat2_nb_subdir.ipynb new file mode 100644 index 0000000..d6a4ff0 --- /dev/null +++ b/tests/unit/utils/templates/subdir/cat2_nb_subdir.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aab4d036-a4e4-4ab0-bb49-6aceaff8525a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": ["category = cat2"] + }, + "outputs": [], + "source": [ + "Test notebook without notebooker tag" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/unit/utils/templates/subdir/no_cat_nb_subdir.ipynb b/tests/unit/utils/templates/subdir/no_cat_nb_subdir.ipynb new file mode 100644 index 0000000..dacdb67 --- /dev/null +++ b/tests/unit/utils/templates/subdir/no_cat_nb_subdir.ipynb @@ -0,0 +1,41 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "aab4d036-a4e4-4ab0-bb49-6aceaff8525a", + "metadata": { + "editable": true, + "slideshow": { + "slide_type": "" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "Test notebook without notebooker tag" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.8 (py38)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From fc3d2a77b34469f9fcf5849d4335b2cff1284cd4 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 12:59:16 +0300 Subject: [PATCH 03/28] constants.py --- notebooker/constants.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/notebooker/constants.py b/notebooker/constants.py index a957a06..8b3df5c 100644 --- a/notebooker/constants.py +++ b/notebooker/constants.py @@ -88,6 +88,7 @@ class NotebookResultBase(object): mailfrom = attr.ib(default=None) email_subject = attr.ib(default=None) is_slideshow = attr.ib(default=False) + category = attr.ib(default=None) def saveable_output(self): out = attr.asdict(self) @@ -164,6 +165,7 @@ class NotebookResultComplete(NotebookResultBase): scheduler_job_id = attr.ib(default=None) mailfrom = attr.ib(default=None) is_slideshow = attr.ib(default=False) + category = attr.ib(default=None) def html_resources(self): """We have to save the raw images using Mongo GridFS - figure out where they will go here""" @@ -197,6 +199,7 @@ def saveable_output(self): "raw_html": "", # backwards compatibility for versions<0.3.1 "mailfrom": self.mailfrom, "is_slideshow": self.is_slideshow, + "category": self.category, } def __repr__(self): @@ -205,7 +208,7 @@ def __repr__(self): "job_start_time={job_start_time}, job_finish_time={job_finish_time}, update_time={update_time}, " "report_title={report_title}, overrides={overrides}, mailto={mailto}, error_mailto={error_mailto}, " "mailfrom={mailfrom}, email_subject={email_subject}, generate_pdf_output={generate_pdf_output}, " - "hide_code={hide_code}, scheduler_job_id={scheduler_job_id}, is_slideshow={is_slideshow})".format( + "hide_code={hide_code}, scheduler_job_id={scheduler_job_id}, is_slideshow={is_slideshow}, category={category})".format( job_id=self.job_id, status=self.status, report_name=self.report_name, @@ -222,5 +225,6 @@ def __repr__(self): hide_code=self.hide_code, scheduler_job_id=self.scheduler_job_id, is_slideshow=self.is_slideshow, + category=self.category, ) ) From 2f114bc2f5f98603028bf9cf95d95eb121f2541a Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:00:27 +0300 Subject: [PATCH 04/28] execute_notebook.py --- notebooker/execute_notebook.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/notebooker/execute_notebook.py b/notebooker/execute_notebook.py index bc6ba10..ce1770d 100644 --- a/notebooker/execute_notebook.py +++ b/notebooker/execute_notebook.py @@ -54,6 +54,7 @@ def _run_checks( scheduler_job_id: Optional[str] = None, mailfrom: Optional[str] = None, is_slideshow: bool = False, + category: Optional[str] = None, ) -> NotebookResultComplete: """ This is the actual method which executes a notebook, whether running in the webapp or via the entrypoint. @@ -152,6 +153,7 @@ def _run_checks( generate_pdf_output=generate_pdf_output, report_name=template_name, report_title=report_title, + category=category, overrides=overrides, scheduler_job_id=scheduler_job_id, mailfrom=mailfrom, @@ -164,6 +166,7 @@ def _run_checks( def run_report( job_submit_time, report_name, + category, overrides, result_serializer, report_title="", @@ -222,6 +225,7 @@ def run_report( scheduler_job_id=scheduler_job_id, mailfrom=mailfrom, is_slideshow=is_slideshow, + category=category, ) logger.info("Successfully got result.") result_serializer.save_check_result(result) @@ -234,6 +238,7 @@ def run_report( job_start_time=job_submit_time, report_name=report_name, report_title=report_title, + category=category, error_info=error_info, overrides=overrides, mailto=mailto, @@ -257,6 +262,7 @@ def run_report( return run_report( job_submit_time, report_name, + category, overrides, result_serializer, report_title=report_title, @@ -351,6 +357,7 @@ def _get_overrides(overrides_as_json: AnyStr, iterate_override_values_of: Option def execute_notebook_entrypoint( config: BaseConfig, report_name: str, + category: str, overrides_as_json: str, iterate_override_values_of: Union[List[str], str], report_title: str, @@ -377,6 +384,7 @@ def execute_notebook_entrypoint( start_time = datetime.datetime.now() logger.info("Running a report with these parameters:") logger.info("report_name = %s", report_name) + logger.info("category = %s", category) logger.info("overrides_as_json = %s", overrides_as_json) logger.info("iterate_override_values_of = %s", iterate_override_values_of) logger.info("report_title = %s", report_title) @@ -407,6 +415,7 @@ def execute_notebook_entrypoint( result = run_report( start_time, report_name, + category, overrides, result_serializer, report_title=report_title, @@ -495,6 +504,7 @@ def run_report_in_subprocess( email_subject=None, n_retries=3, is_slideshow=False, + category=None, ) -> str: """ Execute the Notebooker report in a subprocess. @@ -513,6 +523,7 @@ def run_report_in_subprocess( :param email_subject: `str` if passed, then this string will be used in the email subject :param n_retries: The number of retries to attempt. :param is_slideshow: Whether the notebook is a reveal.js slideshow or not. + :param category: Category of the notebook :return: The unique job_id. """ if error_mailto is None: @@ -535,6 +546,7 @@ def run_report_in_subprocess( is_slideshow=is_slideshow, email_subject=email_subject, mailfrom=mailfrom, + category=category, ) command = ( @@ -578,6 +590,7 @@ def run_report_in_subprocess( + (["--is-slideshow"] if is_slideshow else []) + ([f"--scheduler-job-id={scheduler_job_id}"] if scheduler_job_id is not None else []) + ([f"--mailfrom={mailfrom}"] if mailfrom is not None else []) + + ([f"--category={category}"] if category is not None else []) + ([f"--email-subject={email_subject}"] if email_subject else []) ) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 733573fe13c07ae38189dfd8d8b918823f332b72 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:36:23 +0300 Subject: [PATCH 05/28] changes to header.html --- notebooker/web/templates/header.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooker/web/templates/header.html b/notebooker/web/templates/header.html index 19097fe..fdfc767 100644 --- a/notebooker/web/templates/header.html +++ b/notebooker/web/templates/header.html @@ -82,7 +82,7 @@

Execute a notebook:

{% for report, subreports in all_reports|dictsort() recursive %} {% if subreports is none %} - {{ report }} + {{ report.split('/')[-1] }} {% else %}
@@ -96,7 +96,7 @@

Execute a notebook:

{% endfor %} - Add a new notebook + How to add a new notebook
From 97a90f50907689589dccb48e13ec3dbc8e3b4e34 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:37:11 +0300 Subject: [PATCH 06/28] changes to index.js --- notebooker/web/static/notebooker/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooker/web/static/notebooker/index.js b/notebooker/web/static/notebooker/index.js index 095e9bc..e5e3e9d 100644 --- a/notebooker/web/static/notebooker/index.js +++ b/notebooker/web/static/notebooker/index.js @@ -66,7 +66,7 @@ const updateContents = (currentFolder, entries) => { const displayName = entryAfterLevel(report, level); const stats = entries[report]; reportParts.push( - `` + + `` + '
' + `

${displayName}

\n` + '
\n' + @@ -81,7 +81,7 @@ const updateContents = (currentFolder, entries) => { " " + "
" + '
' + - ` Original report name: ${stats.report_name}\n` + + ` Original report name: ${stats.original_report}\n` + "
" + "" ); From d74373e6e8524106a24b6f0d698bce0aec7c0642 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:41:06 +0300 Subject: [PATCH 07/28] changes to index.py --- notebooker/web/routes/index.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/notebooker/web/routes/index.py b/notebooker/web/routes/index.py index 6d4df3e..b4825a6 100644 --- a/notebooker/web/routes/index.py +++ b/notebooker/web/routes/index.py @@ -41,6 +41,10 @@ def result_listing(report_name): result_limit = int(request.args.get("limit") or DEFAULT_RESULT_LIMIT) all_reports = get_all_possible_templates() with current_app.app_context(): + if "PATH_TO_CATEGORY_DICT" in current_app.config and report_name in current_app.config["PATH_TO_CATEGORY_DICT"]: + title_name = current_app.config["PATH_TO_CATEGORY_DICT"][report_name] + "/" + report_name.split("/")[-1] + else: + title_name = inflection.titleize(report_name) result = render_template( "result_listing.html", all_reports=all_reports, @@ -49,7 +53,7 @@ def result_listing(report_name): report_name=report_name, result_limit=result_limit, n_results_available=get_serializer().n_all_results_for_report_name(report_name), - titleised_report_name=inflection.titleize(report_name), + titleised_report_name=title_name, readonly_mode=current_app.config["READONLY_MODE"], scheduler_disabled=current_app.config["DISABLE_SCHEDULER"], ) From c31e8a53b13a7e38a30228bf4724c5cbdbb44894 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:42:02 +0300 Subject: [PATCH 08/28] changes to mongo.py --- notebooker/serialization/mongo.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/notebooker/serialization/mongo.py b/notebooker/serialization/mongo.py index 4ddf97b..64ef82b 100644 --- a/notebooker/serialization/mongo.py +++ b/notebooker/serialization/mongo.py @@ -209,6 +209,7 @@ def save_check_stub( is_slideshow: bool = False, email_subject: Optional[str] = None, mailfrom: Optional[str] = None, + category: Optional[str] = None, ) -> None: """Call this when we are just starting a check. Saves a "pending" job into storage.""" job_start_time = job_start_time or datetime.datetime.now() @@ -228,6 +229,7 @@ def save_check_stub( scheduler_job_id=scheduler_job_id, is_slideshow=is_slideshow, mailfrom=mailfrom, + category=category, ) self._save_to_db(pending_result) @@ -325,6 +327,7 @@ def _convert_result( scheduler_job_id=result.get("scheduler_job_id", None), is_slideshow=result.get("is_slideshow", False), email_subject=result.get("email_subject", None), + category=result.get("category", None), ) elif cls == NotebookResultPending: return NotebookResultPending( @@ -344,6 +347,7 @@ def _convert_result( stdout=result.get("stdout", []), scheduler_job_id=result.get("scheduler_job_id", None), is_slideshow=result.get("is_slideshow", False), + category=result.get("category", None), ) elif cls == NotebookResultError: @@ -370,6 +374,7 @@ def _convert_result( stdout=result.get("stdout", []), scheduler_job_id=result.get("scheduler_job_id", False), is_slideshow=result.get("is_slideshow", False), + category=result.get("category", None), ) else: raise ValueError("Could not deserialise {} into result object.".format(result)) @@ -397,10 +402,17 @@ def _get_result_count(self, base_filter): def get_count_and_latest_time_per_report(self, subfolder: Optional[str]): base_filer = {} if not subfolder else {"report_name": {"$regex": subfolder + ".*"}} + return self.fetch_reports(base_filer) + + def get_count_and_latest_time_per_report_per_category(self, category: Optional[str]): + base_filer = {} if not category else {"category": category} + return self.fetch_reports(base_filer) + + def fetch_reports(self, base_filer: Dict[str, Any]): reports = list( self._get_raw_results( base_filter=base_filer, - projection={"report_name": 1, "job_start_time": 1, "scheduler_job_id": 1, "_id": 0}, + projection={"report_name": 1, "job_start_time": 1, "scheduler_job_id": 1, "category": 1, "_id": 0}, limit=0, ) ) @@ -411,7 +423,7 @@ def get_count_and_latest_time_per_report(self, subfolder: Optional[str]): for report, all_runs in jobs_by_name.items(): latest_start_time = max(r["job_start_time"] for r in all_runs) scheduled_runs = len([x for x in all_runs if x.get("scheduler_job_id")]) - output[report] = {"count": len(all_runs), "latest_run": latest_start_time, "scheduler_runs": scheduled_runs} + output[report] = {"count": len(all_runs), "latest_run": latest_start_time, "scheduler_runs": scheduled_runs, "category": r["category"]} return output def get_all_results( From fd8523f2bcf06294f299d28e394c3d9866cf9a6b Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:43:19 +0300 Subject: [PATCH 09/28] changes to report_execution.py --- notebooker/web/routes/report_execution.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/notebooker/web/routes/report_execution.py b/notebooker/web/routes/report_execution.py index d837a17..8e5a426 100644 --- a/notebooker/web/routes/report_execution.py +++ b/notebooker/web/routes/report_execution.py @@ -81,6 +81,10 @@ def run_report_http(report_name): :returns: An HTML template which is the Run Report interface. """ + if "PATH_TO_CATEGORY_DICT" in current_app.config and report_name in current_app.config["PATH_TO_CATEGORY_DICT"]: + category = current_app.config["PATH_TO_CATEGORY_DICT"][report_name] + else: + category = "" report_name = convert_report_name_url_to_path(report_name) json_params = request.args.get("json_params") initial_python_parameters = json_to_python(json_params) or "" @@ -95,6 +99,7 @@ def run_report_http(report_name): has_prefix=False, has_suffix=False, report_name=report_name, + category=category, all_reports=get_all_possible_templates(), initialPythonParameters={}, readonly_mode=current_app.config["READONLY_MODE"], @@ -111,6 +116,7 @@ def run_report_http(report_name): has_prefix=has_prefix, has_suffix=has_suffix, report_name=report_name, + category=category, all_reports=get_all_possible_templates(), initialPythonParameters=initial_python_parameters, default_mailfrom=current_app.config["DEFAULT_MAILFROM"], @@ -129,12 +135,14 @@ class RunReportParams(NamedTuple): scheduler_job_id: Optional[str] is_slideshow: bool email_subject: Optional[str] + category: Optional[str] def validate_run_params(report_name, params, issues: List[str]) -> RunReportParams: logger.info(f"Validating input params: {params} for {report_name}") # Find and cleanse the title of the report - report_title = validate_title(params.get("report_title") or report_name, issues) + category = params.get("category", "") + report_title = validate_title(params.get("report_title") or (category + "/" + report_name.strip("/").split("/")[-1] if category else report_name), issues) # Get mailto email address mailto = validate_mailto(params.get("mailto"), issues) error_mailto = validate_mailto(params.get("error_mailto"), issues) @@ -145,6 +153,7 @@ def validate_run_params(report_name, params, issues: List[str]) -> RunReportPara is_slideshow = params.get("is_slideshow") in ("on", "True", True) email_subject = validate_title(params.get("email_subject") or "", issues) + out = RunReportParams( report_title=report_title, mailto=mailto, @@ -155,6 +164,7 @@ def validate_run_params(report_name, params, issues: List[str]) -> RunReportPara scheduler_job_id=params.get("scheduler_job_id"), is_slideshow=is_slideshow, email_subject=email_subject, + category=category, ) logger.info(f"Validated params: {out}") return out @@ -179,6 +189,7 @@ def _handle_run_report( f"mailfrom={params.mailfrom} " f"email_subject={params.email_subject} " f"is_slideshow={params.is_slideshow} " + f"category={params.category} " ) try: with current_app.app_context(): @@ -196,6 +207,7 @@ def _handle_run_report( mailfrom=params.mailfrom, email_subject=params.email_subject, is_slideshow=params.is_slideshow, + category=params.category, ) return ( jsonify({"id": job_id}), From f6631783e26c6a99e03b158a50eb20032d4afb0e Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:43:47 +0300 Subject: [PATCH 10/28] changes to report.py --- notebooker/utils/results.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/notebooker/utils/results.py b/notebooker/utils/results.py index 85eb851..4241b21 100644 --- a/notebooker/utils/results.py +++ b/notebooker/utils/results.py @@ -6,7 +6,7 @@ import babel.dates import inflection -from flask import url_for +from flask import url_for,current_app from notebooker import constants from notebooker.exceptions import NotebookRunException @@ -140,12 +140,20 @@ def get_all_available_results_json( def get_count_and_latest_time_per_report(serializer: MongoResultSerializer, subfolder: Optional[str] = None): - reports = serializer.get_count_and_latest_time_per_report(subfolder) + if subfolder and current_app.config["CATEGORIZATION"]: + category = subfolder.rstrip("/") + reports = serializer.get_count_and_latest_time_per_report_per_category(category) + else: + reports = serializer.get_count_and_latest_time_per_report(subfolder) output = {} for report_name, metadata in sorted(reports.items(), key=lambda x: x[1]["latest_run"], reverse=True): - metadata["report_name"] = report_name + title_name = report_name + if "PATH_TO_CATEGORY_DICT" in current_app.config and report_name in current_app.config["PATH_TO_CATEGORY_DICT"]: + title_name = current_app.config["PATH_TO_CATEGORY_DICT"][report_name] + "/" + report_name.split("/")[-1] + metadata["report_name"] = title_name + metadata["original_report"] = report_name metadata["time_diff"] = babel.dates.format_timedelta(datetime.datetime.now() - metadata["latest_run"]) - output[inflection.titleize(report_name)] = metadata + output[inflection.titleize(title_name)] = metadata return output From 33e7d0e804f41d7bde31448e423e850e8968e89a Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:44:29 +0300 Subject: [PATCH 11/28] changes to run_report.html --- notebooker/web/templates/run_report.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/notebooker/web/templates/run_report.html b/notebooker/web/templates/run_report.html index 9ddc5dc..a47542e 100644 --- a/notebooker/web/templates/run_report.html +++ b/notebooker/web/templates/run_report.html @@ -16,7 +16,7 @@

Customise your report

Report Title:

- +

Existing parameters:

@@ -34,6 +34,10 @@

Override parameters:

+
+ + +

Email result to:

Date: Tue, 8 Oct 2024 13:45:34 +0300 Subject: [PATCH 12/28] changes to scheduler.js --- notebooker/web/static/notebooker/scheduler.js | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/notebooker/web/static/notebooker/scheduler.js b/notebooker/web/static/notebooker/scheduler.js index fba014d..1bce25e 100644 --- a/notebooker/web/static/notebooker/scheduler.js +++ b/notebooker/web/static/notebooker/scheduler.js @@ -80,9 +80,25 @@ load_all_templates = (callback) => { dataType: "json", success: (result) => { let templates = Array(); - for (let i = 0; i < result.result.length; i++) { - let value = result.result[i]; - templates = templates.concat({ name: value, value: value }); + if (Array.isArray(result.result)) { + // This logic handles basic templates + for (let i = 0; i < result.result.length; i++) { + let value = result.result[i]; + templates = templates.concat({ name: value, value: value }); + } + } else if (typeof result.result === "object") { + // This logic handles categorized templates + for (let key in result.result) { + if (result.result.hasOwnProperty(key)) { + let value = result.result[key]; + for (let subKey in value) { + if (value.hasOwnProperty(subKey)) { + let lastPart = subKey.split("/").pop(); + templates = templates.concat({ name: key + "/" + lastPart, value: subKey }); + } + } + } + } } $(".selection.dropdown").dropdown({ values: templates, @@ -295,21 +311,21 @@ $(document).ready(() => { const schedulerTable = $("#schedulerTable"); const schedulerDataTable = schedulerTable.DataTable({ columns: [ - { - title: "Report Unique ID", - name: "id", - data: "id", - }, { title: "Report Title", name: "report_title", data: "params.report_title", }, { - title: "Report Name", - name: "report_name", - data: "params.report_name", + title: "Report Unique ID", + name: "id", + data: "id", }, + // { + // title: "Report Name", + // name: "report_name", + // data: "params.report_name", + // }, { title: "Cron Schedule", name: "cron_schedule", From bda09942e991b189914d8d6cbc2144ba0080ac55 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:50:36 +0300 Subject: [PATCH 13/28] changes to scheduler.js --- notebooker/web/scheduler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notebooker/web/scheduler.py b/notebooker/web/scheduler.py index 2a31125..7c5f4e5 100644 --- a/notebooker/web/scheduler.py +++ b/notebooker/web/scheduler.py @@ -24,6 +24,7 @@ def run_report( is_slideshow: bool = False, error_mailto: Optional[str] = None, email_subject: Optional[str] = None, + category: Optional[str] = None, ): """ This is the entrypoint of the scheduler; APScheduler has to @@ -46,6 +47,7 @@ def run_report( n_retries=0, is_slideshow=is_slideshow, email_subject=email_subject, + category=category, ) else: # Fall back to using API. This will not work in readonly mode. From eee88d0ffd28ac9e4cf8f692762a0773e22ad913 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 13:51:04 +0300 Subject: [PATCH 14/28] changes to scheduling.py --- notebooker/web/routes/scheduling.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/notebooker/web/routes/scheduling.py b/notebooker/web/routes/scheduling.py index 9bfc26c..01e241c 100644 --- a/notebooker/web/routes/scheduling.py +++ b/notebooker/web/routes/scheduling.py @@ -52,6 +52,8 @@ def remove_schedule(job_id): def get_job_id(report_name: str, report_title: str) -> str: + if "PATH_TO_CATEGORY_DICT" in current_app.config and report_name in current_app.config["PATH_TO_CATEGORY_DICT"]: + report_name = current_app.config["PATH_TO_CATEGORY_DICT"][report_name] + "/" + report_name.split("/")[-1] return f"{report_name}_{report_title}" @@ -68,7 +70,10 @@ def update_schedule(report_name): overrides_dict = handle_overrides(request.values.get("overrides", ""), issues) if issues: return jsonify({"status": "Failed", "content": ("\n".join(issues))}) - + if "PATH_TO_CATEGORY_DICT" in current_app.config and report_name in current_app.config["PATH_TO_CATEGORY_DICT"]: + category = current_app.config["PATH_TO_CATEGORY_DICT"][report_name] + else: + category = "" params = { "report_name": report_name, "overrides": overrides_dict, @@ -81,6 +86,7 @@ def update_schedule(report_name): "hide_code": params.hide_code, "is_slideshow": params.is_slideshow, "scheduler_job_id": job_id, + "category": category, } job.modify(trigger=trigger, kwargs=params) current_app.apscheduler.reschedule_job(job_id, jobstore="mongo", trigger=trigger) @@ -103,6 +109,10 @@ def create_schedule(report_name): if issues: return jsonify({"status": "Failed", "content": ("\n".join(issues))}) job_id = get_job_id(report_name, params.report_title) + if "PATH_TO_CATEGORY_DICT" in current_app.config and report_name in current_app.config["PATH_TO_CATEGORY_DICT"]: + category = current_app.config["PATH_TO_CATEGORY_DICT"][report_name] + else: + category = "" dict_params = { "report_name": report_name, "overrides": overrides_dict, @@ -115,6 +125,7 @@ def create_schedule(report_name): "hide_code": params.hide_code, "scheduler_job_id": job_id, "is_slideshow": params.is_slideshow, + "category": category, } logger.info(f"Creating job with params: {dict_params}") try: From fd947d89e20542b63c836c7c5cb98f3594c0a654 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:07:01 +0300 Subject: [PATCH 15/28] changes to settings.py --- notebooker/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/notebooker/settings.py b/notebooker/settings.py index daf2d01..eeea35b 100644 --- a/notebooker/settings.py +++ b/notebooker/settings.py @@ -28,6 +28,9 @@ class BaseConfig: # A boolean flag to dictate whether we should execute the notebook at the origin or not. EXECUTE_AT_ORIGIN: bool = False + # A boolean flag to dictate whether we should discover and group notebooker by their category tags. + CATEGORIZATION: bool = False + # The serializer class we are using for storage, e.g. PyMongoResultSerializer SERIALIZER_CLS: DEFAULT_SERIALIZER = None # The dictionary of parameters which are used to initialize the serializer class above From 6e73cdcddd423a78dfad8896a1aef74ce7200312 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:09:23 +0300 Subject: [PATCH 16/28] changes to templates.py --- notebooker/utils/templates.py | 18 ++++++++++++++++-- notebooker/web/routes/templates.py | 4 ++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/notebooker/utils/templates.py b/notebooker/utils/templates.py index 86e86fe..8893859 100644 --- a/notebooker/utils/templates.py +++ b/notebooker/utils/templates.py @@ -21,6 +21,20 @@ def _valid_dirname(d): def _valid_filename(f): return (f.endswith(".py") or f.endswith(".ipynb")) and "__init__" not in f and "__pycache__" not in f +def _extract_category(path) -> Optional[str]: + if path.endswith(".ipynb"): + nb = nbformat.read(path, as_version=nbformat.v4.nbformat) + return _get_category(nb) + return None + +def _get_category(notebook: nbformat.NotebookNode) -> Optional[int]: + for idx, cell in enumerate(notebook["cells"]): + tags = cell.get("metadata", {}).get("tags", []) + for tag in tags: + clean_tag = tag.translate({ord(' '): None}) # Remove spaces + if clean_tag.startswith('category='): + return clean_tag.split('=')[1] + return None def _get_parameters_cell_idx(notebook: nbformat.NotebookNode) -> Optional[int]: for idx, cell in enumerate(notebook["cells"]): @@ -31,7 +45,7 @@ def _get_parameters_cell_idx(notebook: nbformat.NotebookNode) -> Optional[int]: def template_name_to_notebook_node( - template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True + template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True ) -> nbformat.NotebookNode: path = generate_ipynb_from_py( get_template_dir(), template_name, notebooker_disable_git, py_template_dir, warn_on_local=warn_on_local @@ -41,7 +55,7 @@ def template_name_to_notebook_node( def _get_preview( - template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True + template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True ) -> str: """Returns an HTML render of a report template, with parameters highlighted.""" cached = get_cache(("preview", template_name)) diff --git a/notebooker/web/routes/templates.py b/notebooker/web/routes/templates.py index 597dddf..51d2102 100644 --- a/notebooker/web/routes/templates.py +++ b/notebooker/web/routes/templates.py @@ -5,7 +5,7 @@ from notebooker.utils.results import get_count_and_latest_time_per_report from notebooker.utils.web import convert_report_name_url_to_path from notebooker.web.routes.report_execution import get_report_parameters_html -from notebooker.web.utils import all_templates_flattened, get_all_possible_templates, get_serializer +from notebooker.web.utils import get_all_possible_templates, get_serializer templates_bp = Blueprint("templates_bp", __name__) @@ -41,7 +41,7 @@ def all_possible_templates_flattened(): :returns: A JSON which is a list of all possible templates with their full names. """ - return jsonify({"result": all_templates_flattened()}) + return jsonify({"result": get_all_templates()}) @templates_bp.route("/core/get_template_parameters/", methods=["GET"]) From 9fad0e9b4d0c3265899323c1a0580e8dacc8ed57 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:10:26 +0300 Subject: [PATCH 17/28] changes to utils.py --- notebooker/web/routes/templates.py | 2 +- notebooker/web/utils.py | 122 ++++++++++++++++++++++++++--- 2 files changed, 111 insertions(+), 13 deletions(-) diff --git a/notebooker/web/routes/templates.py b/notebooker/web/routes/templates.py index 51d2102..d13b951 100644 --- a/notebooker/web/routes/templates.py +++ b/notebooker/web/routes/templates.py @@ -5,7 +5,7 @@ from notebooker.utils.results import get_count_and_latest_time_per_report from notebooker.utils.web import convert_report_name_url_to_path from notebooker.web.routes.report_execution import get_report_parameters_html -from notebooker.web.utils import get_all_possible_templates, get_serializer +from notebooker.web.utils import get_all_possible_templates, get_serializer, get_all_templates templates_bp = Blueprint("templates_bp", __name__) diff --git a/notebooker/web/utils.py b/notebooker/web/utils.py index 903c189..e2454ab 100644 --- a/notebooker/web/utils.py +++ b/notebooker/web/utils.py @@ -9,7 +9,7 @@ from notebooker.constants import python_template_dir from notebooker.serialization.mongo import MongoResultSerializer from notebooker.serialization.serialization import get_serializer_from_cls -from notebooker.utils.templates import _valid_dirname, _valid_filename, _gen_all_templates +from notebooker.utils.templates import _valid_dirname, _valid_filename, _gen_all_templates, _extract_category logger = getLogger(__name__) @@ -46,24 +46,122 @@ def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, U """ Creates a nested dictionary that represents the folder structure of rootdir """ + categorization = current_app.config.get("CATEGORIZATION", False) starting_point = starting_point or _get_python_template_dir() all_dirs = {} rootdir = starting_point.rstrip(os.sep) start = rootdir.rfind(os.sep) + 1 - for path, dirs, files in os.walk(rootdir): + + for path, _, files in os.walk(rootdir): if not _valid_dirname(path): continue folders = path[start:].split(os.sep) - subdir = { - os.sep.join(folders[1:] + [f.replace(".ipynb", "").replace(".py", "")]): None - for f in files - if _valid_filename(f) - } - parent = reduce(dict.get, folders[:-1], all_dirs) - parent[folders[-1]] = subdir - return all_dirs[rootdir[start:]] + subdir = {} + parent = all_dirs + + for f in files: + full_path = os.path.join(starting_point, *folders[1:], f) + if _valid_filename(f): + if categorization: + category = _extract_category(full_path) + if category: + parent.setdefault(rootdir.split(os.sep)[-1], {}).setdefault(category, {})[os.path.join(*folders[1:], f)] = None + else: + subdir[os.path.join(*folders[1:], f)] = None + + for folder in folders[:-1]: + if folder not in parent: + parent[folder] = {} + parent = parent[folder] + if not categorization: + parent[folders[-1]] = subdir + + if categorization: + all_dirs = filter_for_code_files(all_dirs) + path_to_category_name = {name: original_key for original_key, sub_dict in all_dirs.get(rootdir.split(os.sep)[-1], {}).items() + for name, value in sub_dict.items() if value is None} + current_app.config["PATH_TO_CATEGORY_DICT"] = path_to_category_name + + stripped = strip_extensions(all_dirs) + logger.info("Stripped directory structure %s", stripped) + return stripped.get(rootdir[start:], {}) + + +def strip_extensions(d): + def strip_extension(item): + """Strips .py or .ipynb extension from a given item, if present.""" + for ext in ('.py', '.ipynb'): + if item.endswith(ext): + return item[:-len(ext)] + return item + + """ + Recursively removes .ipynb and .py extensions from all keys and values in the dictionary. + """ + + def process_dict(sub_d): + """ + Recursively processes each item in the dictionary to strip extensions from keys and values. + """ + new_dict = {} + for k, v in sub_d.items(): + new_key = strip_extension(k) + if isinstance(v, dict): + new_dict[new_key] = process_dict(v) # Recursive call for sub-dictionaries + elif isinstance(v, str): + new_dict[new_key] = strip_extension(v) # Strip extension from values if string + else: + new_dict[new_key] = v # Copy other values directly + return new_dict + + return process_dict(d) + + +def filter_for_code_files(d): + """ + Recursively filters a dictionary to retain only items that are either + .py or .ipynb files or directories leading to such files. + """ + + def has_code_files(sub_d): + """ + Determines whether a dictionary or its nested dictionaries contain + any .py or .ipynb files. + """ + if not isinstance(sub_d, dict): + return False + for k, v in sub_d.items(): + if isinstance(v, dict) and has_code_files(v): + return True + if k.endswith(('.py', '.ipynb')): + return True + return False + + def filter_dict(sub_d): + """ + Recursively filters the dictionary to retain only keys leading to .py or .ipynb files, + or to dictionaries that do, direct or indirectly. + """ + new_dict = {} + for k, v in sub_d.items(): + if isinstance(v, dict): + filtered_sub_d = filter_dict(v) + if has_code_files(filtered_sub_d): # Retain if leads to code files + new_dict[k] = filtered_sub_d + elif k.endswith(('.py', '.ipynb')): + new_dict[k] = v + return new_dict + + filtered_dict = filter_dict(d) + return strip_extensions(filtered_dict) def all_templates_flattened(): - templates = list(_gen_all_templates(get_all_possible_templates(warn_on_local=False))) - return templates + return list(_gen_all_templates(get_all_possible_templates(warn_on_local=False))) + + +def get_all_templates(): + if current_app.config["CATEGORIZATION"]: + return get_all_possible_templates(warn_on_local=False) + else: + return all_templates_flattened() From ad644b5fd089a36f02a8579d2917a20f3ea56f65 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:11:00 +0300 Subject: [PATCH 18/28] added sort to get constant results in the test --- tests/integration/web/routes/test_core.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/web/routes/test_core.py b/tests/integration/web/routes/test_core.py index a92349c..5957995 100644 --- a/tests/integration/web/routes/test_core.py +++ b/tests/integration/web/routes/test_core.py @@ -14,7 +14,7 @@ def test_create_schedule(flask_app, setup_workspace): rv = client.get("/core/all_possible_templates_flattened") assert rv.status_code == 200 data = json.loads(rv.data) - assert data == {"result": ["fake/py_report", "fake/ipynb_report", "fake/report_failing"]} + assert sorted(data["result"]) == sorted(["fake/py_report", "fake/ipynb_report", "fake/report_failing"]) def test_version_number(flask_app, setup_workspace): @@ -67,6 +67,7 @@ def test_get_all_templates_with_results(flask_app, setup_workspace): status=JobStatus.DONE, overrides={"param1": "big"}, scheduler_job_id="ohboy_it's_a_schedule", + category=None, ), NotebookResultError( job_id="job2", @@ -85,10 +86,12 @@ def test_get_all_templates_with_results(flask_app, setup_workspace): data = json.loads(rv.data) assert data == { "Report Name": { + 'category': None, "count": 2, "scheduler_runs": 1, "report_name": "report_name", "latest_run": "Sat, 02 Jan 2021 00:00:00 GMT", + 'original_report': 'report_name', "time_diff": "1 month", } } @@ -160,6 +163,7 @@ def test_get_all_templates_with_results_then_delete(flask_app, setup_workspace): status=JobStatus.DONE, overrides={"param1": "big"}, scheduler_job_id="ohboy_it's_a_schedule", + category=None, ), NotebookResultError( job_id="job2", @@ -185,17 +189,21 @@ def test_get_all_templates_with_results_then_delete(flask_app, setup_workspace): data = json.loads(rv.data) assert data == { "Bad Report": { + 'category': None, "count": 1, "scheduler_runs": 0, "report_name": "BadReport", "latest_run": "Thu, 02 Jan 2014 00:00:00 GMT", + 'original_report': 'BadReport', "time_diff": "7 years", }, "Report Name": { + 'category': None, "count": 2, "scheduler_runs": 1, "report_name": "report_name", "latest_run": "Sat, 02 Jan 2021 00:00:00 GMT", + 'original_report': 'report_name', "time_diff": "1 month", }, } @@ -208,10 +216,12 @@ def test_get_all_templates_with_results_then_delete(flask_app, setup_workspace): data = json.loads(rv.data) assert data == { "Report Name": { + 'category': None, "count": 1, "scheduler_runs": 0, "report_name": "report_name", "latest_run": "Sat, 02 Jan 2021 00:00:00 GMT", + 'original_report': 'report_name', "time_diff": "1 month", } } From f1ddbe3c48e34893ca5b340cfacc304b831f0bb2 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:14:26 +0300 Subject: [PATCH 19/28] added test for execution with category --- tests/integration/test_execute_notebook.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_execute_notebook.py b/tests/integration/test_execute_notebook.py index 4af4462..8184695 100644 --- a/tests/integration/test_execute_notebook.py +++ b/tests/integration/test_execute_notebook.py @@ -97,6 +97,11 @@ def test_main(mongo_host, setup_and_cleanup_notebooker_filesystem, webapp_config "sad@email", "notebooker@example.com", ), + ( + ["--report-name", "crashyreport", "--category", "cat1", "--error-mailto", "sad@email", "--mailfrom", "notebooker@example.com"], + "sad@email", + "notebooker@example.com", + ), (["--report-name", "crashyreport", "--mailfrom", "notebooker@example.com"], None, "notebooker@example.com"), ], ) From 3263553fea49a248ecd8cf8dab759cee5edaf8d9 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:15:09 +0300 Subject: [PATCH 20/28] changes to test_run_report.py --- tests/integration/web/test_run_report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/web/test_run_report.py b/tests/integration/web/test_run_report.py index 0dcfa20..3be6ddb 100644 --- a/tests/integration/web/test_run_report.py +++ b/tests/integration/web/test_run_report.py @@ -19,6 +19,7 @@ def test_run_report_json_parameters(flask_app, setup_workspace): is_slideshow = True scheduler_job_id = "abc/123" email_subject = "Subject" + category = "" payload = { "overrides": json.dumps(overrides), "report_title": report_title, @@ -50,6 +51,7 @@ def test_run_report_json_parameters(flask_app, setup_workspace): mailfrom=mailfrom, is_slideshow=is_slideshow, email_subject=email_subject, + category=category, ) From ee43675d155601036438fb7f70479370bb0ab61a Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:15:27 +0300 Subject: [PATCH 21/28] changes to test_run_report.py --- tests/unit/test_run_report.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_run_report.py b/tests/unit/test_run_report.py index c0c054c..a692e6c 100644 --- a/tests/unit/test_run_report.py +++ b/tests/unit/test_run_report.py @@ -66,6 +66,7 @@ def test_validate_run_params(): mailfrom="test@example.com", is_slideshow=True, email_subject="Subject of the email", + category="", ) actual_output = validate_run_params("lovely_report_name", input_params, issues) assert issues == [] From 2cf120ed36505f1a0a03714ee869929a4069e731 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:15:54 +0300 Subject: [PATCH 22/28] changes to test_scheduling.py --- tests/integration/web/test_scheduling.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/web/test_scheduling.py b/tests/integration/web/test_scheduling.py index 2bb2e06..b7506d5 100644 --- a/tests/integration/web/test_scheduling.py +++ b/tests/integration/web/test_scheduling.py @@ -41,6 +41,7 @@ def test_create_schedule(flask_app, setup_workspace, report_name): "scheduler_job_id": f"{report_name}_test2", "mailfrom": "test@example.com", "email_subject": "Subject", + "category": "", }, "trigger": { "fields": { From 1563576ba2ebdda4843fc540b3d6448ed0248d36 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:16:41 +0300 Subject: [PATCH 23/28] added test to check categorized structure --- tests/integration/test_templates.py | 105 +++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_templates.py b/tests/integration/test_templates.py index 284f70b..ef59d5e 100644 --- a/tests/integration/test_templates.py +++ b/tests/integration/test_templates.py @@ -1,18 +1,91 @@ -from notebooker.web.utils import get_all_possible_templates - - -def test_get_all_possible_templates(flask_app): - flask_app.config["PY_TEMPLATE_BASE_DIR"] = None - with flask_app.app_context(): - assert get_all_possible_templates() == { - "sample": { - "sample/plot_random": None, - "sample/test_plotly": None, - "sample/plot_random_raw": None, - "sample/slideshow_test": None, - }, - "other_folder": { - "other_folder/other_slideshow_test": None, - "this one has spaces": {"other_folder/this one has spaces/plot_random2": None}, +import os +import pytest +import shutil +import tempfile + +from flask import Flask + +from notebooker.utils.filesystem import mkdir_p +from notebooker.utils.templates import _valid_dirname +from notebooker.web.utils import get_directory_structure + +@pytest.fixture +def app_context(): + app = Flask(__name__) + # Configure your app for testing here + ctx = app.app_context() + ctx.push() # Pushes the application context + + yield app # This makes the app available to the test functions + + ctx.pop() # Removes the application context after test completion + +@pytest.mark.parametrize( + "input_dirname, expected_result", + [ + ("./my_directory", True), + ("../hello_world/a/b/c/", True), + (".git/blah", False), + ("../.git/hello/world", False), + ("normal/path/to/something", True), + ("/absolute/path/.git", False), + ("/absolute/path/git", True), + ], +) +def test_valid_dirnames(input_dirname, expected_result): + assert _valid_dirname(input_dirname) is expected_result + + +def test_get_directory_structure(app_context): + temp_dir = tempfile.mkdtemp() + try: + paths = [ + "hello.py", + "goodbye.py", + "depth/1.py", + "this/is/very/deep.py", + "depth/2.py", + "this/is/deep.py", + "this/report.py", + "hello_again.ipynb", + "depth/3.ipynb", + ".hidden/4.ipynb", + ".hidden/visible/5.ipynb", + ".hidden/.more-hidden/6.ipynb", + "./visible/7.ipynb", + "this/is/../is/8.ipynb", + ] + for path in paths: + abspath = os.path.join(temp_dir, path) + if "/" in path: + mkdir_p(os.path.dirname(abspath)) + with open(abspath, "w") as f: + f.write("#hello") + expected_structure = { + "hello": None, + "goodbye": None, + "depth": {"depth/1": None, "depth/2": None, "depth/3": None}, + "this": { + "this/report": None, + "is": {"this/is/8": None, "this/is/deep": None, "very": {"this/is/very/deep": None}}, }, + "hello_again": None, + "visible": {"visible/7": None}, } + + assert get_directory_structure(temp_dir) == expected_structure + finally: + shutil.rmtree(temp_dir) + +def test_get_directory_structure_categorized(app_context): + app_context.config['CATEGORIZATION'] = True + temp_dir = tempfile.mkdtemp() + try: + expected_structure = {'cat1': {'cat1_nb': None, 'subdir/cat1_nb_subdir': None}, + 'cat2': {'cat2_nb': None, 'subdir/cat2_nb_subdir': None}} + + templates_path = os.path.join(os.path.dirname(__file__), 'templates') + actual_structure = get_directory_structure(templates_path) + assert actual_structure == expected_structure + finally: + shutil.rmtree(temp_dir) \ No newline at end of file From 0b0b71399671114fb77e1f15e3c8b8f9dedd6371 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:36:50 +0300 Subject: [PATCH 24/28] updated CHANGELOG.md --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d2278c..a79df99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +0.6.4 (2024-10-08) +------------------ +* Feature: Categorization of the notebooks, allowing set special category tag to notebooks for easy grouping +* This feature solves issues with huge repo's where only limited number of notebooks are used in the webapp: + * Very deep navigation tree in the ui for deepest notebook paths + * Super long names of the reports in scheduler and results pages + * Hard tile navigation for the reports +* Important: only categorized notebooks, those having 'category=..' tag are shown as options to select in the webapp + 0.6.3 (2024-07-11) ------------------ * Feature: Flag to preserve original working directory when running notebooks to make local imports and relative paths work. From 3eaefcd610aa1719aa56143f01705ecdc3ef64a2 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:42:08 +0300 Subject: [PATCH 25/28] blacked --- notebooker/serialization/mongo.py | 7 ++++++- notebooker/utils/results.py | 2 +- notebooker/utils/templates.py | 13 ++++++++----- notebooker/web/routes/report_execution.py | 7 +++++-- notebooker/web/utils.py | 20 +++++++++++++------- tests/integration/test_execute_notebook.py | 11 ++++++++++- tests/integration/test_templates.py | 15 ++++++++++----- tests/integration/web/routes/test_core.py | 16 ++++++++-------- 8 files changed, 61 insertions(+), 30 deletions(-) diff --git a/notebooker/serialization/mongo.py b/notebooker/serialization/mongo.py index 64ef82b..0ad1d8c 100644 --- a/notebooker/serialization/mongo.py +++ b/notebooker/serialization/mongo.py @@ -423,7 +423,12 @@ def fetch_reports(self, base_filer: Dict[str, Any]): for report, all_runs in jobs_by_name.items(): latest_start_time = max(r["job_start_time"] for r in all_runs) scheduled_runs = len([x for x in all_runs if x.get("scheduler_job_id")]) - output[report] = {"count": len(all_runs), "latest_run": latest_start_time, "scheduler_runs": scheduled_runs, "category": r["category"]} + output[report] = { + "count": len(all_runs), + "latest_run": latest_start_time, + "scheduler_runs": scheduled_runs, + "category": r["category"], + } return output def get_all_results( diff --git a/notebooker/utils/results.py b/notebooker/utils/results.py index 4241b21..f75ac27 100644 --- a/notebooker/utils/results.py +++ b/notebooker/utils/results.py @@ -6,7 +6,7 @@ import babel.dates import inflection -from flask import url_for,current_app +from flask import url_for, current_app from notebooker import constants from notebooker.exceptions import NotebookRunException diff --git a/notebooker/utils/templates.py b/notebooker/utils/templates.py index 8893859..a73aada 100644 --- a/notebooker/utils/templates.py +++ b/notebooker/utils/templates.py @@ -21,21 +21,24 @@ def _valid_dirname(d): def _valid_filename(f): return (f.endswith(".py") or f.endswith(".ipynb")) and "__init__" not in f and "__pycache__" not in f + def _extract_category(path) -> Optional[str]: if path.endswith(".ipynb"): nb = nbformat.read(path, as_version=nbformat.v4.nbformat) return _get_category(nb) return None + def _get_category(notebook: nbformat.NotebookNode) -> Optional[int]: for idx, cell in enumerate(notebook["cells"]): tags = cell.get("metadata", {}).get("tags", []) for tag in tags: - clean_tag = tag.translate({ord(' '): None}) # Remove spaces - if clean_tag.startswith('category='): - return clean_tag.split('=')[1] + clean_tag = tag.translate({ord(" "): None}) # Remove spaces + if clean_tag.startswith("category="): + return clean_tag.split("=")[1] return None + def _get_parameters_cell_idx(notebook: nbformat.NotebookNode) -> Optional[int]: for idx, cell in enumerate(notebook["cells"]): tags = cell.get("metadata", {}).get("tags", []) @@ -45,7 +48,7 @@ def _get_parameters_cell_idx(notebook: nbformat.NotebookNode) -> Optional[int]: def template_name_to_notebook_node( - template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True + template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True ) -> nbformat.NotebookNode: path = generate_ipynb_from_py( get_template_dir(), template_name, notebooker_disable_git, py_template_dir, warn_on_local=warn_on_local @@ -55,7 +58,7 @@ def template_name_to_notebook_node( def _get_preview( - template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True + template_name: str, notebooker_disable_git: bool, py_template_dir: str, warn_on_local: Optional[bool] = True ) -> str: """Returns an HTML render of a report template, with parameters highlighted.""" cached = get_cache(("preview", template_name)) diff --git a/notebooker/web/routes/report_execution.py b/notebooker/web/routes/report_execution.py index 8e5a426..6a76e6a 100644 --- a/notebooker/web/routes/report_execution.py +++ b/notebooker/web/routes/report_execution.py @@ -142,7 +142,11 @@ def validate_run_params(report_name, params, issues: List[str]) -> RunReportPara logger.info(f"Validating input params: {params} for {report_name}") # Find and cleanse the title of the report category = params.get("category", "") - report_title = validate_title(params.get("report_title") or (category + "/" + report_name.strip("/").split("/")[-1] if category else report_name), issues) + report_title = validate_title( + params.get("report_title") + or (category + "/" + report_name.strip("/").split("/")[-1] if category else report_name), + issues, + ) # Get mailto email address mailto = validate_mailto(params.get("mailto"), issues) error_mailto = validate_mailto(params.get("error_mailto"), issues) @@ -153,7 +157,6 @@ def validate_run_params(report_name, params, issues: List[str]) -> RunReportPara is_slideshow = params.get("is_slideshow") in ("on", "True", True) email_subject = validate_title(params.get("email_subject") or "", issues) - out = RunReportParams( report_title=report_title, mailto=mailto, diff --git a/notebooker/web/utils.py b/notebooker/web/utils.py index e2454ab..358266c 100644 --- a/notebooker/web/utils.py +++ b/notebooker/web/utils.py @@ -65,7 +65,9 @@ def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, U if categorization: category = _extract_category(full_path) if category: - parent.setdefault(rootdir.split(os.sep)[-1], {}).setdefault(category, {})[os.path.join(*folders[1:], f)] = None + parent.setdefault(rootdir.split(os.sep)[-1], {}).setdefault(category, {})[ + os.path.join(*folders[1:], f) + ] = None else: subdir[os.path.join(*folders[1:], f)] = None @@ -78,8 +80,12 @@ def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, U if categorization: all_dirs = filter_for_code_files(all_dirs) - path_to_category_name = {name: original_key for original_key, sub_dict in all_dirs.get(rootdir.split(os.sep)[-1], {}).items() - for name, value in sub_dict.items() if value is None} + path_to_category_name = { + name: original_key + for original_key, sub_dict in all_dirs.get(rootdir.split(os.sep)[-1], {}).items() + for name, value in sub_dict.items() + if value is None + } current_app.config["PATH_TO_CATEGORY_DICT"] = path_to_category_name stripped = strip_extensions(all_dirs) @@ -90,9 +96,9 @@ def get_directory_structure(starting_point: Optional[str] = None) -> Dict[str, U def strip_extensions(d): def strip_extension(item): """Strips .py or .ipynb extension from a given item, if present.""" - for ext in ('.py', '.ipynb'): + for ext in (".py", ".ipynb"): if item.endswith(ext): - return item[:-len(ext)] + return item[: -len(ext)] return item """ @@ -133,7 +139,7 @@ def has_code_files(sub_d): for k, v in sub_d.items(): if isinstance(v, dict) and has_code_files(v): return True - if k.endswith(('.py', '.ipynb')): + if k.endswith((".py", ".ipynb")): return True return False @@ -148,7 +154,7 @@ def filter_dict(sub_d): filtered_sub_d = filter_dict(v) if has_code_files(filtered_sub_d): # Retain if leads to code files new_dict[k] = filtered_sub_d - elif k.endswith(('.py', '.ipynb')): + elif k.endswith((".py", ".ipynb")): new_dict[k] = v return new_dict diff --git a/tests/integration/test_execute_notebook.py b/tests/integration/test_execute_notebook.py index 8184695..1ed991c 100644 --- a/tests/integration/test_execute_notebook.py +++ b/tests/integration/test_execute_notebook.py @@ -98,7 +98,16 @@ def test_main(mongo_host, setup_and_cleanup_notebooker_filesystem, webapp_config "notebooker@example.com", ), ( - ["--report-name", "crashyreport", "--category", "cat1", "--error-mailto", "sad@email", "--mailfrom", "notebooker@example.com"], + [ + "--report-name", + "crashyreport", + "--category", + "cat1", + "--error-mailto", + "sad@email", + "--mailfrom", + "notebooker@example.com", + ], "sad@email", "notebooker@example.com", ), diff --git a/tests/integration/test_templates.py b/tests/integration/test_templates.py index ef59d5e..10964db 100644 --- a/tests/integration/test_templates.py +++ b/tests/integration/test_templates.py @@ -9,6 +9,7 @@ from notebooker.utils.templates import _valid_dirname from notebooker.web.utils import get_directory_structure + @pytest.fixture def app_context(): app = Flask(__name__) @@ -20,6 +21,7 @@ def app_context(): ctx.pop() # Removes the application context after test completion + @pytest.mark.parametrize( "input_dirname, expected_result", [ @@ -77,15 +79,18 @@ def test_get_directory_structure(app_context): finally: shutil.rmtree(temp_dir) + def test_get_directory_structure_categorized(app_context): - app_context.config['CATEGORIZATION'] = True + app_context.config["CATEGORIZATION"] = True temp_dir = tempfile.mkdtemp() try: - expected_structure = {'cat1': {'cat1_nb': None, 'subdir/cat1_nb_subdir': None}, - 'cat2': {'cat2_nb': None, 'subdir/cat2_nb_subdir': None}} + expected_structure = { + "cat1": {"cat1_nb": None, "subdir/cat1_nb_subdir": None}, + "cat2": {"cat2_nb": None, "subdir/cat2_nb_subdir": None}, + } - templates_path = os.path.join(os.path.dirname(__file__), 'templates') + templates_path = os.path.join(os.path.dirname(__file__), "templates") actual_structure = get_directory_structure(templates_path) assert actual_structure == expected_structure finally: - shutil.rmtree(temp_dir) \ No newline at end of file + shutil.rmtree(temp_dir) diff --git a/tests/integration/web/routes/test_core.py b/tests/integration/web/routes/test_core.py index 5957995..6af22b3 100644 --- a/tests/integration/web/routes/test_core.py +++ b/tests/integration/web/routes/test_core.py @@ -86,12 +86,12 @@ def test_get_all_templates_with_results(flask_app, setup_workspace): data = json.loads(rv.data) assert data == { "Report Name": { - 'category': None, + "category": None, "count": 2, "scheduler_runs": 1, "report_name": "report_name", "latest_run": "Sat, 02 Jan 2021 00:00:00 GMT", - 'original_report': 'report_name', + "original_report": "report_name", "time_diff": "1 month", } } @@ -189,21 +189,21 @@ def test_get_all_templates_with_results_then_delete(flask_app, setup_workspace): data = json.loads(rv.data) assert data == { "Bad Report": { - 'category': None, + "category": None, "count": 1, "scheduler_runs": 0, "report_name": "BadReport", "latest_run": "Thu, 02 Jan 2014 00:00:00 GMT", - 'original_report': 'BadReport', + "original_report": "BadReport", "time_diff": "7 years", }, "Report Name": { - 'category': None, + "category": None, "count": 2, "scheduler_runs": 1, "report_name": "report_name", "latest_run": "Sat, 02 Jan 2021 00:00:00 GMT", - 'original_report': 'report_name', + "original_report": "report_name", "time_diff": "1 month", }, } @@ -216,12 +216,12 @@ def test_get_all_templates_with_results_then_delete(flask_app, setup_workspace): data = json.loads(rv.data) assert data == { "Report Name": { - 'category': None, + "category": None, "count": 1, "scheduler_runs": 0, "report_name": "report_name", "latest_run": "Sat, 02 Jan 2021 00:00:00 GMT", - 'original_report': 'report_name', + "original_report": "report_name", "time_diff": "1 month", } } From 05e210d0669a10f0d78edaa7c8d0682ec7078eb3 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 14:50:35 +0300 Subject: [PATCH 26/28] bump version + fix contribution doc --- .circleci/config.yml | 8 ++++---- CONTRIBUTING.md | 2 +- docs/conf.py | 2 +- notebooker/version.py | 2 +- notebooker/web/static/package.json | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 258575e..701fc35 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -220,7 +220,7 @@ jobs: PYTHON_VERSION: "3_6" CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_6 CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_6 - VERSION: 0.6.3 + VERSION: 0.6.4 PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases YARN_STATIC_DIR: notebooker/web/static/ IMAGE_NAME: mangroup/notebooker @@ -236,7 +236,7 @@ jobs: environment: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_7 CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_7 - VERSION: 0.6.3 + VERSION: 0.6.4 PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases YARN_STATIC_DIR: notebooker/web/static/ IMAGE_NAME: mangroup/notebooker @@ -250,7 +250,7 @@ jobs: environment: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_8 CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_8 - VERSION: 0.6.3 + VERSION: 0.6.4 PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases YARN_STATIC_DIR: notebooker/web/static/ IMAGE_NAME: mangroup/notebooker @@ -264,7 +264,7 @@ jobs: environment: CIRCLE_ARTIFACTS: /tmp/circleci-artifacts/3_11 CIRCLE_TEST_REPORTS: /tmp/circleci-test-results/3_11 - VERSION: 0.6.3 + VERSION: 0.6.4 PANDOC_RELEASES_URL: https://github.com/jgm/pandoc/releases YARN_STATIC_DIR: notebooker/web/static/ IMAGE_NAME: mangroup/notebooker diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c098c5f..dd958de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,7 +20,7 @@ Do also make sure to run the webapp and make sure you haven't broken anything. When releasing a new version, please increment the version number in: * `notebooker/version.py` * `.circleci/config.yml` -* `docs/config.yml` +* `docs/conf.py` * `notebooker/web/static/package.json` This build will validate that these numbers match those given in `.circleci/config.yml`. diff --git a/docs/conf.py b/docs/conf.py index 8b04cb8..d282a5c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ author = "Man Group Quant Tech" # The full version, including alpha/beta/rc tags -release = "0.6.3" +release = "0.6.4" # -- General configuration --------------------------------------------------- diff --git a/notebooker/version.py b/notebooker/version.py index 63af887..364e7ba 100644 --- a/notebooker/version.py +++ b/notebooker/version.py @@ -1 +1 @@ -__version__ = "0.6.3" +__version__ = "0.6.4" diff --git a/notebooker/web/static/package.json b/notebooker/web/static/package.json index 00facf8..859c451 100644 --- a/notebooker/web/static/package.json +++ b/notebooker/web/static/package.json @@ -1,6 +1,6 @@ { "name": "notebooker", - "version": "0.6.3", + "version": "0.6.4", "description": "Notebooker - Turn notebooks into reports", "dependencies": { "bootstrap-table": "1.20.2", From ced0ced36ad6970b08913b57e5ba14c6e6fa405c Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Tue, 8 Oct 2024 15:07:44 +0300 Subject: [PATCH 27/28] More details about enabling categorization --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a79df99..fd32532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ * Very deep navigation tree in the ui for deepest notebook paths * Super long names of the reports in scheduler and results pages * Hard tile navigation for the reports +* Enabling categorization of the notebooks using special flag: + * Add 'category=..' tag to the relevant notebooks metadata + * Execute notebooker with --categorization flag * Important: only categorized notebooks, those having 'category=..' tag are shown as options to select in the webapp +* Keeps original navigation by directory structure if categorization flag is not 0.6.3 (2024-07-11) ------------------ From df92fab22e53826f158cdb7edce437d6d1909366 Mon Sep 17 00:00:00 2001 From: Andrey Gourine Date: Wed, 9 Oct 2024 13:47:05 +0300 Subject: [PATCH 28/28] dummy commit --- tests/integration/test_templates.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_templates.py b/tests/integration/test_templates.py index 10964db..881546e 100644 --- a/tests/integration/test_templates.py +++ b/tests/integration/test_templates.py @@ -16,9 +16,7 @@ def app_context(): # Configure your app for testing here ctx = app.app_context() ctx.push() # Pushes the application context - yield app # This makes the app available to the test functions - ctx.pop() # Removes the application context after test completion