♻️ split off source fields in NeedItem internal data (#1491)

Author: chrisjsewell

docs/api.rst

@@ -38,10 +38,10 @@ Data
 ----
 
 .. automodule:: sphinx_needs.need_item
-   :members: NeedItem, NeedPartItem
+   :members: NeedItem, NeedPartItem, NeedItemSourceProtocol
 
 .. automodule:: sphinx_needs.data
-   :members: NeedsInfoType, NeedsMutable, NeedsPartType
+   :members: NeedsInfoType, NeedsSourceInfoType, NeedsMutable, NeedsPartType
 
 Views
 -----

sphinx_needs/api/need.py

@@ -25,7 +25,11 @@
     apply_default_predicate,
 )
 from sphinx_needs.logging import get_logger, log_warning
-from sphinx_needs.need_item import NeedItem
+from sphinx_needs.need_item import (
+    NeedItem,
+    NeedItemSourceProtocol,
+    NeedItemSourceUnknown,
+)
 from sphinx_needs.nodes import Need
 from sphinx_needs.roles.need_part import find_parts, update_need_with_parts
 from sphinx_needs.utils import jinja_parse
@@ -47,6 +51,7 @@ def generate_need(
     need_type: str,
     title: str,
     *,
+    need_source: NeedItemSourceProtocol | None = None,
     docname: None | str = None,
     lineno: None | int = None,
     id: str | None = None,
@@ -128,8 +133,25 @@ def generate_need(
     :param pre_template: Template name to use for content added before need
     :param post_template: Template name to use for the content added after need
     """
+    source = (
+        NeedItemSourceUnknown(
+            docname=docname,
+            lineno=lineno,
+            lineno_content=lineno_content,
+            is_import=is_import,
+            is_external=is_external,
+            external_url=external_url if is_external else None,
+        )
+        if need_source is None
+        else need_source
+    )
+
     # location is used to provide source mapped warnings
-    location = (docname, lineno) if docname else None
+    location = (
+        (source.dict_repr["docname"], source.dict_repr["lineno"])
+        if source.dict_repr["docname"]
+        else None
+    )
 
     # validate kwargs
     allowed_kwargs = {x["option"] for x in needs_config.extra_links} | set(
@@ -230,9 +252,9 @@ def generate_need(
         "title": title,
         "tags": tuple(tags or []),
         "status": status,
-        "docname": docname,
-        "is_import": is_import,
-        "is_external": is_external,
+        "docname": source.dict_repr["docname"],
+        "is_import": source.dict_repr["is_import"],
+        "is_external": source.dict_repr["is_external"],
     }
     defaults_extras = extras_no_defaults.copy()
     defaults_links = {k: tuple(v or []) for k, v in links_no_defaults.items()}
@@ -306,9 +328,6 @@ def generate_need(
 
     # Add the need and all needed information
     needs_data: NeedsInfoType = {
-        "docname": docname,
-        "lineno": lineno,
-        "lineno_content": lineno_content,
         "doctype": doctype,
         "content": content,
         "pre_content": None,
@@ -340,9 +359,6 @@ def generate_need(
         "is_need": True,
         "id_parent": need_id,
         "id_complete": need_id,
-        "is_import": is_import,
-        "is_external": is_external,
-        "external_url": external_url if is_external else None,
         "external_css": external_css or "external_link",
         "is_modified": False,
         "modifications": 0,
@@ -354,7 +370,9 @@ def generate_need(
         "parent_need": parent_need,
     }
 
-    needs_info = NeedItem(core=needs_data, extras=extras, links=links, _validate=False)
+    needs_info = NeedItem(
+        core=needs_data, extras=extras, links=links, source=source, _validate=False
+    )
 
     if jinja_content:
         need_content_context: dict[str, Any] = {**needs_info}
@@ -390,12 +408,13 @@ def generate_need(
 
 def add_need(
     app: Sphinx,
-    state: None | RSTState,
-    docname: None | str,
-    lineno: None | int,
-    need_type: str,
-    title: str,
+    state: None | RSTState = None,
+    docname: None | str = None,
+    lineno: None | int = None,
+    need_type: str = "",
+    title: str = "",
     *,
+    need_source: NeedItemSourceProtocol | None = None,
     id: str | None = None,
     content: str | StringList = "",
     lineno_content: None | int = None,
@@ -485,11 +504,16 @@ def add_need(
         )
         kwargs = {k: v for k, v in kwargs.items() if k not in _deprecated_kwargs}
 
-    if doctype is None and not is_external and docname:
-        doctype = os.path.splitext(app.env.doc2path(docname))[1]
+    if (
+        doctype is None
+        and not (need_source.dict_repr["is_external"] if need_source else is_external)
+        and (_docname := need_source.dict_repr["docname"] if need_source else docname)
+    ):
+        doctype = os.path.splitext(app.env.doc2path(_docname))[1]
 
     needs_info = generate_need(
         needs_config=NeedsSphinxConfig(app.config),
+        need_source=need_source,
         need_type=need_type,
         title=title,
         full_title=full_title,
@@ -697,6 +721,7 @@ def add_external_need(
     need_type: str,
     title: str | None = None,
     id: str | None = None,
+    need_source: NeedItemSourceProtocol | None = None,
     external_url: str | None = None,
     external_css: str = "external_link",
     content: str = "",
@@ -738,9 +763,7 @@ def add_external_need(
 
     return add_need(
         app=app,
-        state=None,
-        docname=None,
-        lineno=None,
+        need_source=need_source,
         need_type=need_type,
         id=id,
         content=content,

sphinx_needs/data.py

@@ -385,18 +385,28 @@ class CoreFieldParameters(TypedDict):
 }
 
 
-class NeedsInfoType(TypedDict):
-    """Data for a single need."""
-
-    id: str
-    """ID of the data."""
+class NeedsSourceInfoType(TypedDict):
+    """Data for the source of a single need."""
 
     docname: str | None
     """Name of the document where the need is defined (None if external)."""
     lineno: int | None
     """Line number where the need is defined (None if external)."""
     lineno_content: int | None
     """Line number on which the need content starts (None if external)."""
+    external_url: None | str
+    """URL of the need, if it is an external need."""
+    is_import: bool
+    """If true, the need was derived from an import."""
+    is_external: bool
+    """If true, no node is created and need is referencing external url."""
+
+
+class NeedsInfoType(TypedDict):
+    """Data for a single need."""
+
+    id: str
+    """ID of the data."""
 
     # meta information
     title: str
@@ -418,14 +428,6 @@ class NeedsInfoType(TypedDict):
     arch: dict[str, str]
     """Mapping of uml key to uml content."""
 
-    is_import: bool
-    """If true, the need was derived from an import."""
-
-    # external reference information
-    is_external: bool
-    """If true, no node is created and need is referencing external url."""
-    external_url: None | str
-    """URL of the need, if it is an external need."""
     external_css: str
     """CSS class name, added to the external reference."""
 

sphinx_needs/directives/need.py

@@ -24,7 +24,7 @@
 from sphinx_needs.layout import build_need_repr
 from sphinx_needs.logging import WarningSubTypes, get_logger, log_warning
 from sphinx_needs.need_constraints import process_constraints
-from sphinx_needs.need_item import NeedItem
+from sphinx_needs.need_item import NeedItem, NeedItemSourceDirective
 from sphinx_needs.nodes import Need
 from sphinx_needs.utils import (
     DummyOptionSpec,
@@ -166,18 +166,22 @@ def run(self) -> Sequence[nodes.Node]:
                 self._log_warning(f"Invalid value for '{key}' option: {err}")
                 return []
 
+        source = NeedItemSourceDirective(
+            docname=self.env.docname,
+            lineno=self.lineno,
+            lineno_content=self.content_offset + 1,
+        )
+
         try:
             need_nodes = add_need(
-                self.env.app,
-                self.state,
-                self.env.docname,
-                self.lineno,
+                app=self.env.app,
+                state=self.state,
+                need_source=source,
                 need_type=self.name,
                 title=title,
                 full_title=full_title,
                 id=id,
                 content=self.content,
-                lineno_content=self.content_offset + 1,
                 status=status,
                 tags=tags,
                 template=template,

sphinx_needs/directives/needimport.py

@@ -19,6 +19,7 @@
 from sphinx_needs.debug import measure_time
 from sphinx_needs.filter_common import filter_import_item
 from sphinx_needs.logging import log_warning
+from sphinx_needs.need_item import NeedItemSourceImport
 from sphinx_needs.needsfile import SphinxNeedsFileException, check_needs_data
 from sphinx_needs.utils import (
     add_doc,
@@ -241,14 +242,20 @@ def run(self) -> Sequence[nodes.Node]:
             # Replace id, to get unique ids
             need_id = need_params["id"] = id_prefix + need_params["id"]
 
-            # override location
-            need_params["docname"] = self.docname
-            need_params["lineno"] = self.lineno
-
-            need_params["is_import"] = True
+            # set location
+            need_source = NeedItemSourceImport(
+                docname=self.env.docname,
+                lineno=self.lineno,
+                path=need_import_path,
+            )
 
             try:
-                nodes = add_need(self.env.app, self.state, **need_params)
+                nodes = add_need(
+                    app=self.env.app,
+                    state=self.state,
+                    need_source=need_source,
+                    **need_params,
+                )
             except InvalidNeedException as err:
                 log_warning(
                     logger,

sphinx_needs/directives/needservice.py

@@ -14,6 +14,7 @@
 from sphinx_needs.config import NeedsSphinxConfig
 from sphinx_needs.data import SphinxNeedsData
 from sphinx_needs.logging import get_logger, log_warning
+from sphinx_needs.need_item import NeedItemSourceService
 from sphinx_needs.utils import DummyOptionSpec, add_doc, coerce_to_boolean
 
 
@@ -135,14 +136,16 @@ def run(self) -> Sequence[nodes.Node]:
                 datum.update(options)
 
                 # ToDo: Tags and Status are not set (but exist in data)
+                source = NeedItemSourceService(
+                    docname=self.env.docname, lineno=self.lineno
+                )
                 try:
                     section += add_need(
-                        self.env.app,
-                        self.state,
-                        self.env.docname,
-                        self.lineno,
-                        need_type,
-                        need_title,
+                        app=self.env.app,
+                        state=self.state,
+                        need_source=source,
+                        need_type=need_type,
+                        title=need_title,
                         **datum,
                     )
                 except InvalidNeedException as err:

sphinx_needs/external_needs.py

@@ -14,6 +14,7 @@
 from sphinx_needs.config import NeedsSphinxConfig
 from sphinx_needs.data import NeedsCoreFields, SphinxNeedsData
 from sphinx_needs.logging import get_logger, log_warning
+from sphinx_needs.need_item import NeedItemSourceExternal
 from sphinx_needs.utils import clean_log, import_prefix_link_edit
 
 log = get_logger(__name__)
@@ -170,11 +171,9 @@ def load_external_needs(
                 # render jinja content
                 mem_template = get_target_template(target_url)
                 cal_target_url = mem_template.render(**{"need": need})
-                need_params["external_url"] = f"{source['base_url']}/{cal_target_url}"
+                external_url = f"{source['base_url']}/{cal_target_url}"
             else:
-                need_params["external_url"] = (
-                    f"{source['base_url']}/{need.get('docname', '__error__')}.html#{need['id']}"
-                )
+                external_url = f"{source['base_url']}/{need.get('docname', '__error__')}.html#{need['id']}"
 
             # check if external needs already exist
             ext_need_id = need_params["id"]
@@ -190,7 +189,11 @@ def load_external_needs(
                 del_need(app, ext_need_id)
 
             try:
-                add_external_need(app, **need_params)
+                add_external_need(
+                    app,
+                    need_source=NeedItemSourceExternal(url=external_url),
+                    **need_params,
+                )
             except InvalidNeedException as err:
                 location = source.get("json_url", "") or source.get("json_path", "")
                 log_warning(