Package updates & linter/typing fixes

README.md

@@ -5,7 +5,7 @@
 
 *<p style="text-align: center;">DataOps Observability is part of DataKitchen's Open Source Data Observability. DataOps Observability monitors every data journey from data source to customer value, from any team development environment into production, across every tool, team, environment, and customer so that problems are detected, localized, and understood immediately.</p>*
 
-[![DatKitchen Open Source Data Observability](https://datakitchen.io/wp-content/uploads/2024/04/both-products.png)](https://datakitchen.storylane.io/share/g01ss0plyamz)
+[![DataKitchen Open Source Data Observability](https://datakitchen.io/wp-content/uploads/2024/04/both-products.png)](https://datakitchen.storylane.io/share/g01ss0plyamz)
 [Interactive Product Tour](https://datakitchen.storylane.io/share/g01ss0plyamz)
 
 ## Developer Setup
@@ -100,9 +100,7 @@ We enforce the use of certain linting tools. To not get caught by the build-syst
 
 The following hooks are enabled in pre-commit:
 
-- `black`: The black formatter is enforced on the project. We use a basic configuration. Ideally this should solve any and all
-formatting questions we might encounter.
-- `isort`: the isort import-sorter is enforced on the project. We use it with the `black` profile.
+- `ruff`: Handles code formatting, import sorting, and linting
 
 To enable pre-commit from within your virtual environment, simply run:
 

agent_api/config/defaults.py

@@ -5,13 +5,12 @@
 """
 
 import os
-from typing import Optional
 
 # Flask specific settings: https://flask.palletsprojects.com/en/latest/config/#builtin-configuration-values
 from common.entities import Service
 
-PROPAGATE_EXCEPTIONS: Optional[bool] = None
-SERVER_NAME: Optional[str] = os.environ.get("AGENT_API_HOSTNAME")  # Use flask defaults if none set
+PROPAGATE_EXCEPTIONS: bool | None = None
+SERVER_NAME: str | None = os.environ.get("AGENT_API_HOSTNAME")  # Use flask defaults if none set
 USE_X_SENDFILE: bool = False  # If we serve files enable this in production settings when webserver support configured
 
 # Application settings

agent_api/config/local.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 # Flask specific settings: https://flask.palletsprojects.com/en/latest/config/#builtin-configuration-values
-PROPAGATE_EXCEPTIONS: Optional[bool] = True
+PROPAGATE_EXCEPTIONS: bool | None = True
 SECRET_KEY: str = "NOT_VERY_SECRET"

agent_api/config/minikube.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 # Flask specific settings: https://flask.palletsprojects.com/en/latest/config/#builtin-configuration-values
-TESTING: Optional[bool] = True
+TESTING: bool | None = True
 SECRET_KEY: str = "NOT_VERY_SECRET"

agent_api/endpoints/v1/heartbeat.py

@@ -1,7 +1,7 @@
 import logging
-from datetime import datetime, timezone
+from datetime import datetime, UTC
 from http import HTTPStatus
-from typing import Optional, Union, cast
+from typing import Union, cast
 from uuid import UUID
 
 from flask import Response, g, make_response
@@ -23,7 +23,7 @@ def _update_or_create(
     version: str,
     project_id: Union[str, UUID],
     latest_heartbeat: datetime,
-    latest_event_timestamp: Optional[datetime],
+    latest_event_timestamp: datetime | None,
 ) -> None:
     try:
         agent = Agent.select().where(Agent.key == key, Agent.tool == tool, Agent.project_id == project_id).get()
@@ -57,7 +57,7 @@ class Heartbeat(BaseView):
 
     def post(self) -> Response:
         data = self.parse_body(schema=HeartbeatSchema())
-        data["latest_heartbeat"] = datetime.now(tz=timezone.utc)
+        data["latest_heartbeat"] = datetime.now(tz=UTC)
         data["project_id"] = g.project.id
         _update_or_create(**data)
         return make_response("", HTTPStatus.NO_CONTENT)

agent_api/tests/integration/v1_endpoints/test_heartbeat.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timezone
+from datetime import datetime, timezone, UTC
 from http import HTTPStatus
 
 import pytest
@@ -9,7 +9,7 @@
 
 @pytest.mark.integration
 def test_agent_heartbeat(client, database_ctx, headers):
-    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=timezone.utc)
+    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=UTC)
     data = {
         "key": "test-key",
         "tool": "test-tool",
@@ -35,7 +35,7 @@ def test_agent_heartbeat_no_event_timestamp(client, database_ctx, headers):
 
 @pytest.mark.integration
 def test_agent_heartbeat_update(client, database_ctx, headers):
-    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=timezone.utc)
+    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=UTC)
     data = {
         "key": "test-key",
         "tool": "test-tool",
@@ -47,7 +47,7 @@ def test_agent_heartbeat_update(client, database_ctx, headers):
     assert HTTPStatus.NO_CONTENT == response_1.status_code, response_1.json
 
     # The latest_event_timestamp should be older than "now"
-    now = datetime.now(timezone.utc)
+    now = datetime.now(UTC)
     agent_1 = Agent.select().get()
     assert agent_1.latest_heartbeat < now
     assert agent_1.status == AgentStatus.ONLINE
@@ -62,7 +62,7 @@ def test_agent_heartbeat_update(client, database_ctx, headers):
 
 @pytest.mark.integration
 def test_agent_heartbeat_existing_update(client, database_ctx, headers):
-    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=timezone.utc)
+    last_event_timestamp = datetime(2023, 10, 20, 4, 42, 42, tzinfo=UTC)
     data_1 = {
         "key": "test-key",
         "tool": "test-tool",
@@ -79,7 +79,7 @@ def test_agent_heartbeat_existing_update(client, database_ctx, headers):
 
     data_2 = data_1.copy()
     data_2["version"] = "12.0.3"
-    data_2["latest_event_timestamp"] = datetime(2023, 10, 20, 4, 44, 44, tzinfo=timezone.utc).isoformat()
+    data_2["latest_event_timestamp"] = datetime(2023, 10, 20, 4, 44, 44, tzinfo=UTC).isoformat()
 
     response_2 = client.post("/agent/v1/heartbeat", json=data_2, headers=headers)
     assert HTTPStatus.NO_CONTENT == response_2.status_code, response_2.json

cli/base.py

@@ -4,7 +4,7 @@
 from argparse import ArgumentParser
 from logging.config import dictConfig
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any
 from collections.abc import Callable
 
 from log_color import ColorFormatter, ColorStripper
@@ -80,7 +80,7 @@ def __init__(self, **kwargs: Any) -> None:
         LOG.info("#g<\u2714> Established #c<%s> connection to #c<%s>", DB.obj.__class__.__name__, DB.obj.database)
 
 
-def logging_init(*, level: str, logfile: Optional[str] = None) -> None:
+def logging_init(*, level: str, logfile: str | None = None) -> None:
     """Given the log level and an optional logging file location, configure all logging."""
     # Don't bother with a file handler if we're not logging to a file
     handlers = ["console", "filehandler"] if logfile else ["console"]

cli/entry_points/database_schema.py

@@ -1,6 +1,6 @@
 import re
 from argparse import ArgumentParser
-from typing import Any, Optional
+from typing import Any
 from re import Pattern
 from collections.abc import Iterable
 
@@ -19,7 +19,7 @@ class MysqlPrintDatabase(MySQLDatabase):
     def __init__(self) -> None:
         super().__init__("")
 
-    def execute_sql(self, sql: str, params: Optional[Iterable[Any]] = None, commit: Optional[bool] = None) -> None:
+    def execute_sql(self, sql: str, params: Iterable[Any] | None = None, commit: bool | None = None) -> None:
         if params:
             raise Exception(f"Params are not expected to be needed to run DDL SQL, but found {params}")
         if match := self._create_table_re.match(sql):

cli/entry_points/gen_events.py

@@ -7,7 +7,7 @@
 import time
 from argparse import Action, ArgumentParser, Namespace
 from datetime import datetime
-from typing import Any, Optional, Union
+from typing import Any, Union
 from collections.abc import Sequence
 
 from requests_extensions import get_session
@@ -90,7 +90,7 @@ def __call__(
             parser: ArgumentParser,
             namespace: Namespace,
             values: Union[str, Sequence[Any], None],
-            option_string: Optional[str] = None,
+            option_string: str | None = None,
         ) -> None:
             event_data = {}
             remove_fields = []

cli/entry_points/graph_schema.py

@@ -2,6 +2,7 @@
 import sys
 from argparse import ArgumentParser
 from pathlib import Path
+from typing import Any
 
 from jinja2 import Environment, FileSystemLoader
 from peewee import Field, ForeignKeyField, ManyToManyField, Model
@@ -54,7 +55,7 @@ def subcmd_entry_point(self) -> None:
         dot_parts = [head.render({})]
 
         # Initial context/config
-        model_context = []
+        model_context: list[dict[str, Any]] = []
 
         LOG.info("#m<Graphing models...>")
         for name, model in model_map.items():

cli/lib.py

@@ -2,7 +2,6 @@
 import re
 import textwrap
 from argparse import ArgumentParser, ArgumentTypeError
-from typing import Optional
 from uuid import UUID
 
 from log_color.colors import ColorStr
@@ -21,7 +20,7 @@ def uuid_type(arg: str) -> UUID:
 def slice_type(arg: str) -> slice:
     """Convert an argument to a slice; for simplicity, disallow negative slice values and steps."""
 
-    def _int_or_none(val: str) -> Optional[int]:
+    def _int_or_none(val: str) -> int | None:
         if not val:
             return None
         else:

common/actions/action.py

@@ -8,7 +8,7 @@
 ]
 
 import logging
-from typing import Any, NamedTuple, Optional
+from typing import Any, NamedTuple
 from uuid import UUID
 
 from common.entities import Action, Rule
@@ -35,15 +35,15 @@ class InvalidActionTemplate(ActionException):
 
 class ActionResult(NamedTuple):
     result: bool
-    response: Optional[dict]
-    exception: Optional[Exception]
+    response: dict | None
+    exception: Exception | None
 
 
 class BaseAction:
     required_arguments: set = set()
     requires_action_template: bool = False
 
-    def __init__(self, action_template: Optional[Action], override_arguments: dict) -> None:
+    def __init__(self, action_template: Action | None, override_arguments: dict) -> None:
         if self.requires_action_template and not action_template:
             raise ActionTemplateRequired(f"'{self.__class__.__name__}' requires an action template to be set")
 
@@ -70,7 +70,7 @@ def _validate_args(self) -> None:
         if missing_args:
             raise ValueError(f"Required arguments {missing_args} missing for {self.__class__.__name__}")
 
-    def _run(self, event: EVENT_TYPE, rule: Rule, journey_id: Optional[UUID]) -> ActionResult:
+    def _run(self, event: EVENT_TYPE, rule: Rule, journey_id: UUID | None) -> ActionResult:
         raise NotImplementedError("Base Action cannot be executed")
 
     def _store_action_result(self, action_result: ActionResult) -> None:
@@ -88,7 +88,7 @@ def _store_action_result(self, action_result: ActionResult) -> None:
                 exc_info=action_result.exception,
             )
 
-    def execute(self, event: EVENT_TYPE, rule: Rule, journey_id: Optional[UUID]) -> bool:
+    def execute(self, event: EVENT_TYPE, rule: Rule, journey_id: UUID | None) -> bool:
         action_result = self._run(event, rule, journey_id)
         self._store_action_result(action_result)
         return action_result.result

common/actions/action_factory.py

@@ -1,6 +1,5 @@
 __all__ = ["ACTION_CLASS_MAP", "action_factory"]
 
-from typing import Optional
 
 from common.entities import Action
 
@@ -11,7 +10,7 @@
 ACTION_CLASS_MAP: dict[str, type[BaseAction]] = {"CALL_WEBHOOK": WebhookAction, "SEND_EMAIL": SendEmailAction}
 
 
-def action_factory(implementation: str, action_args: dict, template: Optional[Action]) -> BaseAction:
+def action_factory(implementation: str, action_args: dict, template: Action | None) -> BaseAction:
     try:
         action_class = ACTION_CLASS_MAP[implementation]
     except KeyError as ke: