From 798be17a37b2bdbbc79a115fd1c8a530a7ef7468 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Fri, 16 Apr 2021 20:01:30 +0200 Subject: [PATCH 01/16] Test building docs with GHA --- .github/workflows/docs.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..e9d74f48 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,23 @@ +name: docs + +on: [push, pull_request] + +jobs: + docs: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python setup.py develop + pip install -r docs/requirements.txt + - name: Build docs + run: | + cd docs + make html From 67af15ff42c84cca404945760cac39a9f0ad1d0b Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Fri, 16 Apr 2021 20:13:13 +0200 Subject: [PATCH 02/16] Help finding warnings locally when building docs by treating warnings as errors (like readthedocs does). --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 168b9752..548031a8 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -10,7 +10,7 @@ BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ALLSPHINXOPTS = -v -T -E -W --keep-going -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . From 5ffdf0f2b1c9070227e7a3ac3033aa898e7221bc Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sat, 17 Apr 2021 11:03:00 +0200 Subject: [PATCH 03/16] Fix typos and reformat --- docs/conf.py | 6 +++++- pypuppetdb/types.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9ccc34c4..72bd8985 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,11 @@ # -- General configuration ---------------------------------------------------- -extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinx_rtd_theme"] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinx_rtd_theme", +] templates_path = ["_templates"] diff --git a/pypuppetdb/types.py b/pypuppetdb/types.py index 4299a791..bd55c866 100644 --- a/pypuppetdb/types.py +++ b/pypuppetdb/types.py @@ -831,7 +831,7 @@ class Inventory(object): :ivar environment: The environment associated with the inventory's certname. :ivar facts: The dictionary of key-value pairs for the nodes - assosciated facts. + associated facts. :ivar trusted: The trusted data from the node. """ From 3b2b18ca79272061c17a8314b7018a6dce9adfc9 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sat, 17 Apr 2021 11:21:34 +0200 Subject: [PATCH 04/16] Enable reading doc types from PEP 484 type hints --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 72bd8985..5f9d1caa 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,6 +16,7 @@ "sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinx_rtd_theme", + "sphinx_autodoc_typehints", ] templates_path = ["_templates"] From f7bb88bfda99b38d658d4e14dbb5b6fbfa51719d Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 22 Nov 2021 21:16:26 +0100 Subject: [PATCH 05/16] Use type hints for Event and Report and remove the :type entries from their docstrings thanks to the sphinx-autodoc-typehints extension. Also complete their docstrings for their constructors and the static creation methods. --- docs/developer.rst | 3 + docs/requirements.txt | 1 + pypuppetdb/types.py | 230 +++++++++++++++++------------------------- 3 files changed, 94 insertions(+), 140 deletions(-) diff --git a/docs/developer.rst b/docs/developer.rst index 5fdf100c..daa7a12b 100644 --- a/docs/developer.rst +++ b/docs/developer.rst @@ -125,8 +125,11 @@ endpoints data. .. autoclass:: pypuppetdb.types.Fact .. autoclass:: pypuppetdb.types.Resource .. autoclass:: pypuppetdb.types.Event + :members: + :special-members: __init__ .. autoclass:: pypuppetdb.types.Report :members: + :special-members: __init__ .. autoclass:: pypuppetdb.types.Catalog :members: .. autoclass:: pypuppetdb.types.Edge diff --git a/docs/requirements.txt b/docs/requirements.txt index cbf1e365..f852e84d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,3 @@ sphinx sphinx-rtd-theme +sphinx-autodoc-typehints diff --git a/pypuppetdb/types.py b/pypuppetdb/types.py index bd55c866..8f5b1542 100644 --- a/pypuppetdb/types.py +++ b/pypuppetdb/types.py @@ -3,61 +3,39 @@ import logging from datetime import timedelta - from pypuppetdb.QueryBuilder import (EqualsOperator, AndOperator) +from typing import List from pypuppetdb.utils import json_to_datetime log = logging.getLogger(__name__) class Event(object): - """This object represents an event. Unless otherwise specified all - parameters are required. - - :param node: The hostname of the node this event fired on. - :type node: :obj:`string` - :param status: The status for the event. - :type status: :obj:`string` - :param timestamp: A timestamp of when this event occured. - :type timestamp: :obj:`string` formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` - :param hash_: The hash of the report that contains this event. - :type hash_: :obj:`string` - :param title: The resource title this event was fired for. - :type title: :obj:`string` - :param property_: The property of the resource this event was fired for. - :type property_: :obj:`string` - :param message: A message associated with this event. - :type message: :obj:`string` - :param new_value: The new value/state of the resource. - :type new_value: :obj:`string` - :param old_value: The old value/state of the resource. - :type old_value: :obj:`string` - :param type_: The type of the resource this event fired for. - :type type_: :obj:`string` - :param class_: The class responsible for running this event. - :type class_: :obj:`string` - :param execution_path: The path used to reach this particular resource. - :type execution_path: :obj:`string` - :param source_file: The puppet source code file containing the class. - :type source_file: :obj:`string` - :param line_number: The line number in the source file containing the - definition responsible for triggering this event. - :type line_number: :obj:`int` - - :ivar node: A :obj:`string` of this event's node certname. - :ivar status: A :obj:`string` of this event's status. - :ivar failed: The :obj:`bool` equivalent of `status`. - :ivar timestamp: A :obj:`datetime.datetime` of when this event happend. - :ivar node: The hostname of the machine this event\ - occured on. - :ivar hash_: The hash of this event. - :ivar item: :obj:`dict` with information about the item/resource this\ - event was triggered for. + """This object represents an Event (action taken during a Puppet run, from the Report). """ - def __init__(self, node, status, timestamp, hash_, title, property_, - message, new_value, old_value, type_, class_, execution_path, - source_file, line_number): + def __init__(self, node: str, status: str, timestamp: str, hash_: str, title: str, + property_: str, message: str, new_value: str, old_value: str, type_: str, + class_: str, execution_path: str, source_file: str, line_number: int): + """Creates an Event object. + + :param node: The hostname of the node this event fired on. + :param status: The status for the event. + :param timestamp: A timestamp of when this event occurred. + Formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` + :param hash_: The hash of the report that contains this event. + :param title: The resource title this event was fired for. + :param property_: The property of the resource this event was fired for. + :param message: A message associated with this event. + :param new_value: The new value/state of the resource. + :param old_value: The old value/state of the resource. + :param type_: The type of the resource this event fired for. + :param class_: The class responsible for running this event. + :param execution_path: The path used to reach this particular resource. + :param source_file: The puppet source code file containing the class. + :param line_number: The line number in the source file containing the + definition responsible for triggering this event. + """ self.node = node self.status = status if self.status == 'failure': @@ -82,7 +60,13 @@ def __str__(self): return str('{0}').format(self.__string) @staticmethod - def create_from_dict(event): + def create_from_dict(event: dict) -> 'Event': + """Create an Event object from a JSON object (dict) representing an Event + returned by PuppetDB API events endpoint: + https://puppet.com/docs/puppetdb/5.2/api/query/v4/events.html#response-format . + + :param event: JSON object (dict) representing an Event + """ return Event( node=event['certname'], status=event['status'], @@ -102,93 +86,50 @@ def create_from_dict(event): class Report(object): - """This object represents a report. Unless otherwise specified all - parameters are required. - - :param api: API object - :type api: :class:`pypuppetdb.api.BaseAPI` - :param node: The hostname of the node this report originated on. - :type node: :obj:`string` - :param hash_: A string uniquely identifying this report. - :type hash_: :obj:`string` - :param start: The start time of the agent run. - :type start: :obj:`string` formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` - :param end: The time the agent finished its run. - :type end: :obj:`string` formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` - :param received: The time PuppetDB received the report. - :type received: :obj:`string` formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` - :param version: The catalog / configuration version. - :type version: :obj:`string` - :param format_: The catalog format version. - :type format_: :obj:`int` - :param agent_version: The Puppet agent version. - :type agent_version: :obj:`string` - :param transaction: The UUID of this transaction. - :type transaction: :obj:`string` - :param environment: (Optional) The environment assigned to the node that\ - submitted this report. - :type environment: :obj:`string` - :param status: (Optional) The status associated to this report's node. - :type status: :obj:`string` - :param noop: (Default `False`) A flag indicating weather the report was\ - produced by a noop run. - :type noop: :obj:`bool` - :param noop_pending: (Default `False`) A flag indicating weather the \ - report had pending changes produced by a noop run. - :type noop_pending: :obj:`bool` - :param metrics: (Optional) All metrics associated with this report. - :type metrics: :obj:`list` containing :obj:`dict` with Metrics - :param logs: (Optional) All logs associated with this report. - :type logs: :obj:`list` containing :obj:`dict` of logs - :param code_id: (Optional) Ties the catalog to the Puppet Code that\ - generated the catalog. - :type code_id: :obj:`string` - :param catalog_uuid: (Optional) Ties the report to the catalog used\ - from that Puppet run. - :type catalog_uuid: :obj:`string` - :param cached_catalog_status: (Optional) Identifies if the Puppet run\ - used a cached catalog and weather or not it was used due to an\ - error. Can be one of 'explicitly_requested', 'on_failure',\ - 'not_used' not 'null'. - :type cached_catalog_status: :obj:`string` - :param producer: (Optional) The certname of the Puppet Master that\ - sent the report to PuppetDB - :type producer: :obj:`string` - - :ivar node: The hostname this report originated from. - :ivar hash_: Unique identifier of this report. - :ivar start: :obj:`datetime.datetime` when the Puppet agent run started. - :ivar end: :obj:`datetime.datetime` when the Puppet agent run ended. - :ivar received: :obj:`datetime.datetime` when the report finished uploading. - :ivar version: :obj:`string` catalog configuration version. - :ivar format_: :obj:`int` catalog format version. - :ivar agent_version: :obj:`string` Puppet Agent version. - :ivar run_time: :obj:`datetime.timedelta` of **end** - **start**. - :ivar transaction: UUID identifying this transaction. - :ivar environment: The environment assigned to the node that submitted\ - this report. - :ivar status: The status associated to this report's node. - :ivar metrics: :obj:`list` containing :obj:`dict` of all metrics\ - associated with this report. - :ivar logs: :obj:`list` containing :obj:`dict` of all logs\ - associated with this report. - :ivar code_id: :obj:`string` used to tie a catalog to the Puppet Code\ - which generated the catalog. - :ivar catalog_uuid: :obj:`string` used to tie this report to the catalog\ - used on this Puppet run. - :ivar cached_catalog_status: :obj:`string` identifying if this Puppet run\ - used a cached catalog, if so weather it was a result of an error or\ - otherwise. - :ivar producer: :obj:`string` representing the certname of the Puppet\ - Master that sent the report to PuppetDB + """This object represents a Report (of a Puppet run). """ - def __init__(self, api, node, hash_, start, end, received, version, - format_, agent_version, transaction, status=None, - metrics={}, logs={}, environment=None, - noop=False, noop_pending=False, code_id=None, - catalog_uuid=None, cached_catalog_status=None, - producer=None): + # no type hint for "api" to prevent circular import + def __init__(self, api, node: str, hash_: str, start: str, end: str, received: str, + version: str, format_: int, agent_version: str, transaction: str, + environment: str = None, status: str = None, noop: bool = False, + noop_pending: bool = False, metrics: List[dict] = [{}], logs: List[dict] = [{}], + code_id: str = None, catalog_uuid: str = None, cached_catalog_status: str = None, + producer: str = None): + """Creates a Report object. + + :param api: API object (for subqueries) + :param node: The hostname of the node this report originated on. + :param hash\_: A string uniquely identifying this report. + :param start: The start time of the agent run. Formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` + :param end: The time the agent finished its run. Formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` + :param received: The time PuppetDB received the report. + Formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` + :param version: The catalog / configuration version. + :param format\_: The catalog format version. + :param agent_version: The Puppet agent version. + :param transaction: The UUID of this transaction. + :param environment: (Optional) The environment assigned to the node that\ + submitted this report. + :param status: (Optional) The status associated to this report's node. + :param noop: (Default `False`) A flag indicating weather the report was\ + produced by a noop run. + :param noop_pending: (Default `False`) A flag indicating weather the \ + report had pending changes produced by a noop run. + :param metrics: (Optional) All metrics associated with this report. + :param logs: (Optional) All logs associated with this report. + :param code_id: (Optional) Ties the catalog to the Puppet Code that\ + generated the catalog. + :param catalog_uuid: (Optional) Ties the report to the catalog used\ + from that Puppet run. + :param cached_catalog_status: (Optional) Identifies if the Puppet run\ + used a cached catalog and weather or not it was used due to an\ + error. Can be one of 'explicitly_requested', 'on_failure',\ + 'not_used' not 'null'. + :param producer: (Optional) The certname of the Puppet Master that\ + sent the report to PuppetDB + """ + self.node = node self.hash_ = hash_ self.start = json_to_datetime(start) @@ -218,16 +159,24 @@ def __str__(self): return str('{0}').format(self.__string) def events(self, **kwargs): - """Get all events for this report. Additional arguments may also be + """Get all :class:`pypuppetdb.types.Event` for this report. Additional arguments may also be specified that will be passed to the query function. """ return self.__api.events(query=EqualsOperator("report", self.hash_), **kwargs) @staticmethod - def create_from_dict(query_api, report): + # no type hint for "api" to prevent circular import + def create_from_dict(api, report: dict) -> 'Report': + """Create a Report from a JSON object (dict) representing a Report + returned by PuppetDB API reports endpoint: + https://puppet.com/docs/puppetdb/5.2/api/query/v4/reports.html#response-format . + + :param api: API object (for subqueries) + :param report: JSON object (dict) representing a Report + """ return Report( - api=query_api, + api=api, node=report['certname'], hash_=report['hash'], start=report['start_time'], @@ -239,13 +188,14 @@ def create_from_dict(query_api, report): transaction=report['transaction_uuid'], environment=report['environment'], status=report['status'], - noop=report.get('noop'), - noop_pending=report.get('noop_pending'), + noop=report.get('noop', False), + noop_pending=report.get('noop_pending', False), metrics=report['metrics']['data'], logs=report['logs']['data'], - code_id=report.get('code_id'), - catalog_uuid=report.get('catalog_uuid'), - cached_catalog_status=report.get('cached_catalog_status') + code_id=report.get('code_id', None), + catalog_uuid=report.get('catalog_uuid', None), + cached_catalog_status=report.get('cached_catalog_status', None), + # producer=report.get('producer', None) # TODO: consider adding this missing param ) From 6e5f14bc2b21cd3d4cef2a778e968d1bf71471ba Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sat, 17 Apr 2021 12:19:34 +0200 Subject: [PATCH 06/16] Use type hints for utils --- pypuppetdb/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pypuppetdb/utils.py b/pypuppetdb/utils.py index 268a33ba..2fed4bab 100644 --- a/pypuppetdb/utils.py +++ b/pypuppetdb/utils.py @@ -29,21 +29,19 @@ def __unicode__(self): return 'UTC' -def json_to_datetime(date): +def json_to_datetime(date: str) -> datetime.datetime: """Tranforms a JSON datetime string into a timezone aware datetime object with a UTC tzinfo object. :param date: The datetime representation. - :type date: :obj:`string` :returns: A timezone aware datetime object. - :rtype: :class:`datetime.datetime` """ return datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%fZ').replace( tzinfo=UTC()) -def versioncmp(v1, v2): +def versioncmp(v1, v2) -> int: """Compares two objects, x and y, and returns an integer according to the outcome. The return value is negative if x < y, zero if x == y and positive if x > y. @@ -52,7 +50,6 @@ def versioncmp(v1, v2): :param v2: The second object to compare. :returns: -1, 0 or 1. - :rtype: :obj:`int` """ def normalize(v): From 1d7529a0d300c182fd10e5d57eaa8abc57b2645e Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sat, 17 Apr 2021 12:50:01 +0200 Subject: [PATCH 07/16] Ensure modern version of Sphinx & deps --- docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f852e84d..a66e90ec 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ -sphinx -sphinx-rtd-theme -sphinx-autodoc-typehints +Sphinx>=3.5.4,<4 +sphinx-rtd-theme>=0.5.1,<1 +sphinx-autodoc-typehints>=1.12.0,<2 From eed13d4ab81d489546efbc17a130a6ae7dfbd815 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Sat, 17 Apr 2021 13:03:11 +0200 Subject: [PATCH 08/16] Uptick Sphinx to a version that fixes warning "WARNING: more than one target found for cross-reference 'facts': pypuppetdb.api.QueryAPI.facts, pypuppetdb.types.Node.facts" See https://stackoverflow.com/a/67137409/2693875. --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index a66e90ec..2390b709 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,3 @@ -Sphinx>=3.5.4,<4 +Sphinx>=4.0.0.b1,<5 sphinx-rtd-theme>=0.5.1,<1 sphinx-autodoc-typehints>=1.12.0,<2 From cfb84dd8b0fa28275a56267a05900fedb2160186 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 22 Nov 2021 21:17:42 +0100 Subject: [PATCH 09/16] Use latest stable as main, test on 3.10 too even though it's a RC1 as of now --- .github/workflows/docs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e9d74f48..e3ed69d6 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | python setup.py develop From d6dc1cf6f531e0529693b3223758d62e948258f6 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 22 Nov 2021 21:18:10 +0100 Subject: [PATCH 10/16] We want to see results on all versions, even if one fail --- .github/workflows/tests.yml | 1 + mypy.ini | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 mypy.ini diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1c3a8e5e..37181eb1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,6 +11,7 @@ jobs: max-parallel: 5 matrix: python-version: [3.6, 3.7, 3.8, 3.9, '3.10'] + fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..1215375e --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True \ No newline at end of file From a2a8a00d4d11b192218fc4f38ff9be10a202dc99 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Thu, 19 Aug 2021 10:53:52 +0200 Subject: [PATCH 11/16] It seems that we need to do this for mypy --- mypy.ini | 2 -- requirements-test.txt | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 1215375e..00000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -ignore_missing_imports = True \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt index 300b84b8..384f63d7 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -10,4 +10,5 @@ pytest-mock pytest-mypy cov-core mypy +types-requests httpretty From 8c22680a21a40aca1d5df9c4b665bc64edb817a0 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Thu, 19 Aug 2021 11:04:33 +0200 Subject: [PATCH 12/16] ...and this --- requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-test.txt b/requirements-test.txt index 384f63d7..6ce6142c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -11,4 +11,5 @@ pytest-mypy cov-core mypy types-requests +types-mock httpretty From 23905f8c7a1163a2768b3c13068b37607651c4bc Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 22 Nov 2021 21:23:28 +0100 Subject: [PATCH 13/16] Fix warnings and deprecations --- .github/workflows/docs.yml | 4 ++-- pypuppetdb/types.py | 4 ++-- pytest.ini | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e3ed69d6..9b33c225 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: '3.10' - name: Install dependencies run: | python setup.py develop diff --git a/pypuppetdb/types.py b/pypuppetdb/types.py index 8f5b1542..91123af1 100644 --- a/pypuppetdb/types.py +++ b/pypuppetdb/types.py @@ -100,13 +100,13 @@ def __init__(self, api, node: str, hash_: str, start: str, end: str, received: s :param api: API object (for subqueries) :param node: The hostname of the node this report originated on. - :param hash\_: A string uniquely identifying this report. + :param hash_: A string uniquely identifying this report. :param start: The start time of the agent run. Formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` :param end: The time the agent finished its run. Formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` :param received: The time PuppetDB received the report. Formatted as ``%Y-%m-%dT%H:%M:%S.%fZ`` :param version: The catalog / configuration version. - :param format\_: The catalog format version. + :param format_: The catalog format version. :param agent_version: The Puppet agent version. :param transaction: The UUID of this transaction. :param environment: (Optional) The environment assigned to the node that\ diff --git a/pytest.ini b/pytest.ini index 5fbdb4d8..d2f52a03 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -pep8maxlinelength = 100 +flake8-max-line-length = 100 norecursedirs = docs .tox venv .eggs lib python_files = tests/*.py From 853b375cf0670b8a21d9a51ecc19944c50b0f074 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 22 Nov 2021 21:32:27 +0100 Subject: [PATCH 14/16] Fix mutable default arguments --- pypuppetdb/types.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pypuppetdb/types.py b/pypuppetdb/types.py index 91123af1..9c765e1a 100644 --- a/pypuppetdb/types.py +++ b/pypuppetdb/types.py @@ -93,7 +93,7 @@ class Report(object): def __init__(self, api, node: str, hash_: str, start: str, end: str, received: str, version: str, format_: int, agent_version: str, transaction: str, environment: str = None, status: str = None, noop: bool = False, - noop_pending: bool = False, metrics: List[dict] = [{}], logs: List[dict] = [{}], + noop_pending: bool = False, metrics: List[dict] = None, logs: List[dict] = None, code_id: str = None, catalog_uuid: str = None, cached_catalog_status: str = None, producer: str = None): """Creates a Report object. @@ -130,6 +130,11 @@ def __init__(self, api, node: str, hash_: str, start: str, end: str, received: s sent the report to PuppetDB """ + if logs is None: + logs = [{}] + if metrics is None: + metrics = [{}] + self.node = node self.hash_ = hash_ self.start = json_to_datetime(start) From e78d5fa5750f4552cc3f2c18bd1f9a9858d83805 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 22 Nov 2021 21:44:48 +0100 Subject: [PATCH 15/16] Add types to Fact --- pypuppetdb/api/query.py | 7 +++--- pypuppetdb/types.py | 50 ++++++++++++++++++----------------------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/pypuppetdb/api/query.py b/pypuppetdb/api/query.py index 9c20a323..ea8d47e8 100644 --- a/pypuppetdb/api/query.py +++ b/pypuppetdb/api/query.py @@ -3,6 +3,7 @@ import logging from datetime import datetime +from typing import Iterator from pypuppetdb.QueryBuilder import (EqualsOperator) from pypuppetdb.api.base import BaseAPI @@ -18,7 +19,7 @@ class QueryAPI(BaseAPI): """ def nodes(self, unreported=2, with_status=False, with_event_numbers=True, - **kwargs): + **kwargs) -> Iterator[Node]: r"""Query for nodes by either name or query. If both aren't provided this will return a list of all nodes. This method also (optionally) fetches the nodes status and (optionally) @@ -211,7 +212,7 @@ def catalog(self, node): catalogs = self.catalogs(path=node) return next(catalog for catalog in catalogs) - def catalogs(self, **kwargs): + def catalogs(self, **kwargs) -> Iterator[Catalog]: r"""Get the catalog information from the infrastructure based on path and/or query results. It is strongly recommended to include query and/or paging parameters for this endpoint to prevent large result @@ -232,7 +233,7 @@ def catalogs(self, **kwargs): for catalog in catalogs: yield Catalog.create_from_dict(catalog) - def events(self, **kwargs): + def events(self, **kwargs) -> Iterator[Event]: r"""A report is made up of events which can be queried either individually or based on their associated report hash. It is strongly recommended to include query and/or paging parameters for this diff --git a/pypuppetdb/types.py b/pypuppetdb/types.py index 9c765e1a..98fceca1 100644 --- a/pypuppetdb/types.py +++ b/pypuppetdb/types.py @@ -4,7 +4,7 @@ import logging from datetime import timedelta from pypuppetdb.QueryBuilder import (EqualsOperator, AndOperator) -from typing import List +from typing import List, Iterator, Union, Dict from pypuppetdb.utils import json_to_datetime log = logging.getLogger(__name__) @@ -163,7 +163,7 @@ def __repr__(self): def __str__(self): return str('{0}').format(self.__string) - def events(self, **kwargs): + def events(self, **kwargs) -> Iterator[Event]: """Get all :class:`pypuppetdb.types.Event` for this report. Additional arguments may also be specified that will be passed to the query function. """ @@ -204,37 +204,22 @@ def create_from_dict(api, report: dict) -> 'Report': ) -class Fact(object): - """This object represents a fact. Unless otherwise specified all - parameters are required. +FactValue = Union[str, int, bool, float, List, Dict] - :param node: The hostname this fact was collected from. - :type node: :obj:`string` - :param name: The fact's name, such as 'osfamily' - :type name: :obj:`string` - :param value: The fact's value, such as 'Debian' - :type value: :obj:`string` or :obj:`int` or :obj:`dict` - :param environment: (Optional) The fact's environment, such as\ - 'production' - :type environment: :obj:`string` - :ivar node: :obj:`string` holding the hostname. - :ivar name: :obj:`string` holding the fact's name. - :ivar value: :obj:`string` or :obj:`int` or :obj:`dict` holding the\ - fact's value. - :ivar environment: :obj:`string` holding the fact's environment +class Fact(object): + """This object represents a Fact. """ - @staticmethod - def create_from_dict(fact): - return Fact( - node=fact['certname'], - name=fact['name'], - value=fact['value'], - environment=fact['environment'] - ) + def __init__(self, node: str, name: str, value: FactValue, environment: str = None): + """Creates a Fact object. + + :param node: The hostname this fact was collected from. + :param name: The fact's name, such as 'osfamily' + :param value: The fact's value, such as 'Debian' + :param environment: (Optional) The fact's environment, such as 'production' + """ - def __init__(self, node, name, value, environment=None): self.node = node self.name = name self.value = value @@ -247,6 +232,15 @@ def __repr__(self): def __str__(self): return str('{0}').format(self.__string) + @staticmethod + def create_from_dict(fact: dict) -> 'Fact': + return Fact( + node=fact['certname'], + name=fact['name'], + value=fact['value'], + environment=fact['environment'] + ) + class Resource(object): """This object represents a resource. Unless otherwise specified all From 2eee541965ba61ec8b4ffbcb738ac1bc8fd1cf55 Mon Sep 17 00:00:00 2001 From: Greg Dubicki Date: Mon, 22 Nov 2021 22:02:46 +0100 Subject: [PATCH 16/16] Add more type hints --- pypuppetdb/types.py | 81 ++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/pypuppetdb/types.py b/pypuppetdb/types.py index 98fceca1..8568f1cd 100644 --- a/pypuppetdb/types.py +++ b/pypuppetdb/types.py @@ -234,6 +234,12 @@ def __str__(self): @staticmethod def create_from_dict(fact: dict) -> 'Fact': + """Create a Fact object from a JSON object (dict) representing a Fact + returned by PuppetDB API facts endpoint: + https://puppet.com/docs/puppetdb/5.2/api/query/v4/facts.html#response-format . + + :param fact: JSON object (dict) representing a Fact + """ return Fact( node=fact['certname'], name=fact['name'], @@ -243,45 +249,31 @@ def create_from_dict(fact: dict) -> 'Fact': class Resource(object): - """This object represents a resource. Unless otherwise specified all - parameters are required. - - :param node: The hostname this resource is located on. - :type node: :obj:`string` - :param name: The name of the resource in the Puppet manifest. - :type name: :obj:`string` - :param type_: Type of the Puppet resource. - :type type_: :obj:`string` - :param tags: Tags associated with this resource. - :type tags: :obj:`list` - :param exported: If it's an exported resource. - :type exported: :obj:`bool` - :param sourcefile: The Puppet manifest this resource is declared in. - :type sourcefile: :obj:`string` - :param sourceline: The line this resource is declared at. - :type sourceline: :obj:`int` - :param parameters: (Optional) The parameters this resource has been\ - declared with. - :type parameters: :obj:`dict` - :param environment: (Optional) The environment of the node associated\ - with this resource. - :type environment: :obj:`string` - - :ivar node: The hostname this resources is located on. - :ivar name: The name of the resource in the Puppet manifest. - :ivar type_: The type of Puppet resource. - :ivar exported: :obj:`bool` if the resource is exported. - :ivar sourcefile: The Puppet manifest this resource is declared in. - :ivar sourceline: The line this resource is declared at. - :ivar parameters: :obj:`dict` with key:value pairs of parameters. - :ivar relationships: :obj:`list` Contains all relationships to other\ - resources - :ivar environment: :obj:`string` The environment of the node associated\ - with this resource. + """This object represents a Resource. """ - def __init__(self, node, name, type_, tags, exported, sourcefile, - sourceline, environment=None, parameters={}): + def __init__(self, node: str, name: str, type_: str, tags: list[str], exported: bool, + sourcefile: str, sourceline: int, environment: str = None, + parameters: dict = None, relationships: list['Resource'] = None): + """ + Creates a Resource object. + + :param node: The hostname this resource is located on. + :param name: The name of the resource in the Puppet manifest. + :param type_: Type of the Puppet resource. + :param tags: Tags associated with this resource. + :param exported: If it's an exported resource. + :param sourcefile: The Puppet manifest this resource is declared in. + :param sourceline: The line this resource is declared at. + :param environment: (Optional) The environment of the node associated with this resource. + :param parameters: (Optional) The parameters this resource has been declared with. + :param relationships: (Optional) The relationships of this resource to other resources. + """ + if parameters is None: + parameters = {} + if relationships is None: + relationships = [] + self.node = node self.name = name self.type_ = type_ @@ -289,9 +281,9 @@ def __init__(self, node, name, type_, tags, exported, sourcefile, self.exported = exported self.sourcefile = sourcefile self.sourceline = sourceline - self.parameters = parameters - self.relationships = [] self.environment = environment + self.parameters = parameters + self.relationships = relationships self.__string = '{0}[{1}]'.format(self.type_, self.name) def __repr__(self): @@ -301,7 +293,13 @@ def __str__(self): return str('{0}').format(self.__string) @staticmethod - def create_from_dict(resource): + def create_from_dict(resource: dict) -> 'Resource': + """Create a Resource object from a JSON object (dict) representing a Resource + returned by PuppetDB API resources endpoint: + https://puppet.com/docs/puppetdb/5.2/api/query/v4/resources.html#response-format . + + :param resource: JSON object (dict) representing a Resource + """ return Resource( node=resource['certname'], name=resource['title'], @@ -310,8 +308,9 @@ def create_from_dict(resource): exported=resource['exported'], sourcefile=resource['file'], sourceline=resource['line'], - parameters=resource['parameters'], environment=resource['environment'], + parameters=resource['parameters'], + # resource=resource['resource'], # what about this - "the resource's unique hash" ? )