fix: clean up

Author: joscha

dlt_source_notion/__init__.py

@@ -9,44 +9,6 @@
 from notion_client.helpers import iterate_paginated_api
 
 
-# logging.basicConfig(level=logging.DEBUG)
-
-# if is_logging():
-#     logger = logging.getLogger("dlt")
-
-#     class HideSinglePagingNonsense(logging.Filter):
-#         def filter(self, record):
-#             msg = record.getMessage()
-#             if (
-#                 "Extracted data of type list from path _data with length 1" in msg
-#                 or re.match(
-#                     r"Paginator JSONLinkPaginator at [a-fA-F0-9]+: next_url_path: _meta\.links\.next\.href does not have more pages",
-#                     msg,
-#                 )
-#             ):
-#                 return False
-#             return True
-
-#     logger.addFilter(HideSinglePagingNonsense())
-
-
-# def anyurl_encoder(obj: Any) -> JsonSerializable:
-#     if isinstance(obj, AnyUrl):
-#         return obj.unicode_string()
-#     raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
-
-
-# json.set_custom_encoder(anyurl_encoder)
-
-
-# def pydantic_model_dump(model: BaseModel, **kwargs):
-#     """
-#     Dumps a Pydantic model to a dictionary, using the model's field names as keys and NOT observing the field aliases,
-#     which is important for DLT to correctly map the data to the destination.
-#     """
-#     return model.model_dump(by_alias=True, **kwargs)
-
-
 class Table(StrEnum):
     PERSONS = "persons"
     BOTS = "bots"
@@ -56,72 +18,12 @@ def use_id(entity: Dict, **kwargs) -> Dict:
     return filter_dict(entity, **kwargs) | {"_dlt_id": __get_id(entity)}
 
 
-# @dlt.resource(
-#     selected=True,
-#     parallelized=True,
-#     primary_key="id",
-# )
-# def persons(rest_client: RESTClient) -> Iterable[TDataItem]:
-#     for persons_raw in rest_client.paginate(
-#         V2_PERSONS, params={"limit": V2_MAX_PAGE_LIMIT}, hooks=hooks
-#     ):
-#         yield persons_adapter.validate_python(persons_raw)
-
-
-# async def person_employments(
-#     person: Person,
-#     rest_client: RESTClient,
-# ):
-#     href = jmespath.search("links.employments.href", person.field_meta)
-#     if not href:
-#         return
-#     for employments_raw in rest_client.paginate(
-#         href, params={"limit": V2_MAX_PAGE_LIMIT}, hooks=hooks
-#     ):
-#         employments = employments_adapter.validate_python(employments_raw)
-#         for employment in employments:
-#             yield dlt.mark.with_hints(
-#                 item=use_id(employment, exclude=["field_meta", "org_units"]),
-#                 hints=dlt.mark.make_hints(
-#                     table_name=Table.EMPLOYMENTS.value,
-#                 ),
-#                 # needs to be a variant due to https://github.com/dlt-hub/dlt/pull/2109
-#                 create_table_variant=True,
-#             )
-
-
 def __get_id(obj):
     if isinstance(obj, dict):
         return obj.get("id")
     return getattr(obj, "id", None)
 
 
-# @dlt.transformer(
-#     max_table_nesting=1,
-#     parallelized=True,
-#     table_name=Table.PERSONS.value,
-# )
-# async def person_details(persons: List[Person], rest_client: RESTClient):
-#     yield [
-#         use_id(person, exclude=["field_meta", "custom_attributes", "employments"])
-#         for person in persons
-#     ]
-#     for person in persons:
-#         yield person_employments(person, rest_client)
-#         yield dlt.mark.with_hints(
-#             item={"person_id": person.id}
-#             | {cas.root.id: cas.root.value for cas in person.custom_attributes},
-#             hints=dlt.mark.make_hints(
-#                 table_name=Table.CUSTOM_ATTRIBUTES.value,
-#                 primary_key="person_id",
-#                 merge_key="person_id",
-#                 write_disposition="merge",
-#             ),
-#             # needs to be a variant due to https://github.com/dlt-hub/dlt/pull/2109
-#             create_table_variant=True,
-#         )
-
-
 @dlt.resource(
     selected=True,
     parallelized=True,

dlt_source_notion/client.py

@@ -4,60 +4,3 @@
 
 def get_notion_client():
     return Client(auth=dlt.secrets["notion_token"])
-
-
-# Share a session (and thus pool) between all rest clients
-# session: Session = Session(raise_for_status=False)
-
-# auth: notionOAuth2ClientCredentials = None
-
-
-# def get_rest_client(
-#     api_base: str = API_BASE,
-# ):
-#     global session
-#     global auth
-
-#     if auth is None:
-#         auth = notionOAuth2ClientCredentials(
-#             api_base=api_base,
-#             client_id=dlt.secrets["notion_client_id"],
-#             client_secret=dlt.secrets["notion_client_secret"],
-#             default_token_expiration=86400,
-#             session=session,
-#         )
-
-#     client = RESTClient(
-#         base_url=api_base,
-#         headers={
-#             "Accept": "application/json",
-#             "X-notion-App-ID": X_notion_APP_ID,
-#         },
-#         auth=auth,
-#         data_selector="_data",
-#         paginator=JSONLinkPaginator(next_url_path="_meta.links.next.href"),
-#         session=session,
-#     )
-#     return client, auth
-
-
-# def debug_response(response: Response, *args: Any, **kwargs: Any) -> None:
-#     if logging.getLogger().isEnabledFor(logging.DEBUG):
-#         logging.debug(
-#             f"Response: {response.status_code} {response.reason} {response.url} {response.text}"
-#         )
-
-
-# def raise_for_status(response: Response, *args: Any, **kwargs: Any) -> None:
-#     # TODO: Add more detailed error handling, using the notion error response model
-#     response.raise_for_status()
-
-
-# hooks = {
-#     "response": [
-#         debug_response,
-#         raise_for_status,
-#     ]
-# }
-# V2_MAX_PAGE_LIMIT = 50
-# V1_BASE_PAGE = 0

dlt_source_notion/tests/__init__.py

@@ -1,2 +0,0 @@
-def test_anything():
-    pass

dlt_source_notion/tests/test_noop.py

@@ -0,0 +1,2 @@
+def test_noop():
+    pass