Further docs improvements; closes #13

lmmx · lmmx · commit 8bb99f9dc4b0 · 2021-07-28T23:24:45.000+01:00
diff --git a/src/range_streams/__init__.py b/src/range_streams/__init__.py
@@ -1,8 +1,8 @@
 r"""
 :mod:`range_streams` provides file-like object handling through
 an API familiar to users of the standard library
-:mod:`io` module.  It uses :class:`ranges.Range`, :class:`ranges.RangeSet`,
-and :class:`ranges.RangeDict` classes (from the externally maintained
+:mod:`io` module.  It uses :class:`~ranges.Range`, :class:`~ranges.RangeSet`,
+and :class:`~ranges.RangeDict` classes (from the externally maintained
 `python-ranges <https://python-ranges.readthedocs.io/en/latest/>`_ library)
 to represent and look up range operations in an efficient linked
 list data structure.
@@ -50,7 +50,7 @@
 or removed due to overlap with another range. See the docs for further details.
 
 Further ranges are requested by simply calling the :meth:`~range_streams.range_stream.RangeStream.add`
-method with another :class:`ranges.Range` object. To create this implicitly, you can
+method with another :class:`~ranges.Range` object. To create this implicitly, you can
 simply provide a byte range to the `add` method as a tuple of two integers,
 which will be interpreted per the usual convention for ranges in Python,
 as an ``[a,b)`` half-open interval.
diff --git a/src/range_streams/codecs/png/stream.py b/src/range_streams/codecs/png/stream.py
@@ -42,10 +42,30 @@ def __init__(
             self.scan_ihdr()
 
     def populate_chunks(self):
+        """
+        Call :meth:`~range_streams.codecs.png.PngStream.enumerate_chunks`
+        and store in the internal
+        :attr:`~range_streams.codecs.png.PngStream._chunks` attribute,
+        accessible through the :attr:`~range_streams.codecs.png.PngStream.chunks`
+        property.
+
+        If the :attr:`~range_streams.codecs.png.PngStream.chunks` property is
+        called 'prematurely', to avoid an access error it will 'proactively'
+        call this method before returning the gated internal attribute.
+        """
         self._chunks: dict[str, list[PngChunkInfo]] = self.enumerate_chunks()
 
     @property
     def chunks(self):
+        """
+        'Gate' to the internal :attr:`~range_streams.codecs.png.PngStream._chunks`
+        attribute.
+
+        If this property is called before the internal attribute is set,
+        ('prematurely'), to avoid an access error it will 'proactively'
+        call :meth:`~range_streams.codecs.png.PngStream.populate_chunks`
+        before returning the gated internal attribute.
+        """
         if not hasattr(self, "_chunks"):
             self.populate_chunks()
         return self._chunks
@@ -77,6 +97,13 @@ def enumerate_chunks(self):
         possible). Build a dictionary of all chunks with keys of the chunk type (four
         letter strings) and values of lists (since some chunks e.g. IDAT can appear
         multiple times in the PNG).
+
+        See `the official specification
+        <http://www.libpng.org/pub/png/spec/1.2/PNG-Chunks.html>`_ for full details
+        (or `Wikipedia
+        <https://en.wikipedia.org/wiki/
+        Portable_Network_Graphics#%22Chunks%22_within_the_file>`_,
+        or `the W3C <https://www.w3.org/TR/PNG/#5Chunk-layout>`_).
         """
         png_signature = 8  # PNG files start with an 8-byte signature
         chunk_preamble_size = 8  # 4-byte length chunk + 4-byte type chunk
diff --git a/src/range_streams/overlaps.py b/src/range_streams/overlaps.py
@@ -13,7 +13,7 @@
 # This could be written more clearly by using a range_utils helper function shared with
 # most_recent_range
 def get_range_containing(rng_dict: RangeDict, position: int) -> Range:
-    """Get a :class:`ranges.Range` from ``rng_dict`` by looking up the ``position`` it
+    """Get a :class:`~ranges.Range` from ``rng_dict`` by looking up the ``position`` it
     contains, where ``rng_dict`` is either the internal
     :obj:`RangeStream._ranges` attribute
     or the external :obj:`~range_streams.range_stream.RangeStream.ranges` property.
@@ -41,11 +41,12 @@ def overlap_whence(
 ) -> int | None:
     """
     Determine if any overlap exists, whence (i.e. from where) on the pre-existing
-    range it overlapped. 0 if the new range overlapped at the start ('head') of
-    the existing range, 1 if fully contained (in the 'body'), 2 if at the end
-    ('tail'), or None if the range is non-overlapping with any pre-existing range.
+    range it overlapped. ``0`` if the new range overlapped at the start ('head') of
+    the existing range, ``1`` if fully contained (in the 'body'), ``2`` if at the end
+    ('tail'), or ``None`` if the range is non-overlapping with any pre-existing range.
 
-    Note: same convention as Python io module's SEEK_SET, SEEK_CUR, and SEEK_END.
+    Note: same convention as Python io module's
+    :obj:`~io.SEEK_SET`, :obj:`~io.SEEK_CUR`, and :obj:`~io.SEEK_END`.
     """
     if rng in rng_dict:
         # Full overlap (i.e. in middle of pre-existing range)
diff --git a/src/range_streams/range_request.py b/src/range_streams/range_request.py
@@ -16,9 +16,13 @@
 class RangeRequest:
     """
     Store a GET request and the response stream while keeping a reference to
-    the client that spawned it, providing an overridable `_iterator` attribute
-    [by default giving access to `iter_raw()`] on the underlying response,
-    suitable for `RangeResponse` to wrap in a `io.BytesIO` buffered stream.
+    the client that spawned it, providing an overridable
+    :attr:`~range_streams.range_response.RangeResponse._iterator` attribute
+    [by default giving access to
+    :meth:`~range_streams.range_response.RangeResponse.iter_raw`] on the
+    underlying ``httpx.Response``, suitable for
+    :class:`~range_streams.range_response.RangeResponse`
+    to wrap in a :class:`io.BytesIO` buffered stream.
     """
 
     def __init__(self, byte_range: Range, url: str, client):
@@ -36,7 +40,7 @@ def range_header(self):
 
     def setup_stream(self) -> None:
         """
-        `client.stream("GET", url)` but leave the stream to be manually closed
+        ``client.stream("GET", url)`` but leave the stream to be manually closed
         rather than using a context manager
         """
         self.request = self.client.build_request(
@@ -57,24 +61,42 @@ def raise_for_non_partial_content(self):
 
     def content_range_header(self) -> str:
         """
-        Validate request was range request by presence of `content-range` header
+        Validate request was range request by presence of ``content-range`` header
         """
         return detect_header_value(headers=self.response.headers, key="content-range")
 
     @property
     def total_content_length(self) -> int:
+        """
+        Obtain the total content length from the ``content-range`` header of a
+        partial content HTTP GET request. This method is not used for the HTTP HEAD
+        request sent when a :class:`~range_streams.range_stream.RangeStream` is
+        initialised with an empty :class:`~ranges.Range` (since that is not a partial
+        content request it returns a ``content-length`` header which can be read
+        as an integer directly).
+        """
         return int(self.content_range.split("/")[-1])
 
     def iter_raw(self) -> Iterator[bytes]:
+        """
+        Wrap the :meth:`iter_raw` method of the underlying :class:`httpx.Response`
+        object within the :class:`~range_streams.range_response.RangeResponse` in
+        :attr:`~range_streams.range_request.RangeRequest.response`.
+        """
         return self.response.iter_raw()
 
     def close(self) -> None:
+        """
+        Close the :attr:`~range_streams.range_request.RangeRequest.response`
+        :class:`~range_streams.range_response.RangeResponse`.
+        """
         if not self.response.is_closed:
             self.response.close()
 
     def check_client(self):
         """
-        Typing workaround (Sphinx type hint extension does not like httpx)
+        Type checking workaround (Sphinx type hint extension does not like httpx
+        so check the type manually with a method called at initialisation).
         """
         if not isinstance(self.client, httpx.Client):  # pragma: no cover
             raise NotImplementedError("Only HTTPX clients currently supported")
diff --git a/src/range_streams/range_response.py b/src/range_streams/range_response.py
@@ -13,7 +13,25 @@
 
 
 class RangeResponse:
-    tail_mark = 0
+    """
+    Adapted from `obskyr's ResponseStream demo code
+    <https://gist.github.com/obskyr/b9d4b4223e7eaf4eedcd9defabb34f13>`_,
+    this class handles the streamed partial request as a file-like object.
+    """
+
+    tail_mark: int = 0
+    """
+    The amount by which to shorten the 'tail' (i.e. the upper end) of the
+    range when deciding if it is 'consumed'. Incremented within the
+    :meth:`~range_streams.range_stream.RangeStream.handle_overlap` method
+    when the ``pruning_level`` is set to ``1`` (indicating a "replant" policy).
+
+    Under a 'replant' policy, when a new range is to be added and would overlap
+    at the tail of an existing range, the pre-existing range should be effectively
+    truncated by 'marking their tails'
+    (where `an existing range` is assumed here to only be considered a range
+    if it is not 'consumed' yet).
+    """
 
     def __init__(
         self,
@@ -43,10 +61,18 @@ def client(self):
 
     @property
     def url(self) -> str:
+        """
+        A wrapper to access the :attr:`~range_streams.range_stream.RangeStream.url`
+        of the 'parent' :class:`~range_streams.range_stream.RangeStream`.
+        """
         return self.parent_stream.url
 
     @property
     def name(self) -> str:
+        """
+        A wrapper to access the :attr:`~range_streams.range_stream.RangeStream.name`
+        of the 'parent' :class:`~range_streams.range_stream.RangeStream`.
+        """
         return self.parent_stream.name
 
     def _load_all(self):
@@ -63,9 +89,15 @@ def _load_until(self, goal_position):
                 break
 
     def tell(self):
+        """
+        File-like tell (position indicator) within the range request stream.
+        """
         return self._bytes.tell()
 
     def read(self, size=None):
+        """
+        File-like reading within the range request stream.
+        """
         left_off_at = self._bytes.tell()
         if size is None:
             self._load_all()
@@ -77,11 +109,39 @@ def read(self, size=None):
         return self._bytes.read(size)
 
     def seek(self, position, whence=SEEK_SET):
+        """
+        File-like seeking within the range request stream.
+        """
         if whence == SEEK_END:
             self._load_all()
         self._bytes.seek(position, whence)
 
     def is_consumed(self) -> bool:
+        """
+        Whether the :meth:`~range_streams.range_response.RangeResponse.tell`
+        position (indicating 'consumed' or 'read so far') along with the
+        :attr:`~range_streams.range_response.RangeResponse.tail_mark` indicates
+        whether the stream should be considered consumed.
+
+        The :attr:`~range_streams.range_response.RangeResponse.tail_mark`
+        is part of a mechanism to 'shorten' ranges when an overlap is detected,
+        to preserve the one-to-one integrity of the :class:`~ranges.RangeDict`
+        (see notes on the "replant" policy of
+        :meth:`~range_streams.range_stream.RangeStream.handle_overlap`, set
+        by the ``pruning_level`` passed into
+        :class:`~range_streams.range_stream.RangeStream` on initialisation).
+
+        Note that there is (absolutely!) nothing stopping a stream from being
+        re-consumed, but this library works on the assumption that all streams
+        will be handled in an efficient manner (with any data read out from them
+        either used once only or else will be reused from the first output rather
+        than re-accessed directly from the stream itself).
+
+        To this end, :class:`~range_streams.range_stream.RangeStream` has measures
+        in place to "decommission" ranges once they are consumed (see in particular
+        :meth:`~range_streams.range_stream.RangeStream.burn_range` and
+        :meth:`~range_streams.range_stream.RangeStream.handle_overlap`).
+        """
         read_so_far = self.tell()
         len_to_read = range_len(self.request.range) - self.tail_mark
         return read_so_far - len_to_read > 0
diff --git a/src/range_streams/range_stream.py b/src/range_streams/range_stream.py
@@ -1,7 +1,7 @@
 r""":mod:`range_streams.range_stream` exposes a class
 :py:func:`RangeStream`, whose key property (once initialised) is
 :attr:`~range_streams.range_stream.RangeStream.ranges`,
-which provides a :class:`ranges.RangeDict` comprising the ranges of
+which provides a :class:`~ranges.RangeDict` comprising the ranges of
 the file being streamed.
 
 The method :py:func:`RangeStream.add` will request further ranges,
@@ -66,7 +66,7 @@ class RangeStream:
     """
     `'Internal'` ranges attribute. Start position is not affected by
     reading in bytes from the :class:`RangeResponse` (unlike the
-    'externa' :attr:`ranges` property)
+    'external' :attr:`~range_streams.range_stream.RangeStream.ranges` property)
     """
 
     def __init__(
@@ -199,18 +199,22 @@ def register_range(
 
     def set_active_range(self, rng: Range):
         """
-        Setter for the active range (through which active_range_response is also set).
+        Setter for the active range (through which
+        :attr:`~range_streams.range_stream.RangeStream.active_range_response`
+        is also set).
         """
         if self._active_range != rng:
             self._active_range = rng
 
     @property
     def active_range_response(self) -> RangeResponse:
-        """Look up the :class:`RangeResponse` object associated with the
-        currently active range by using
+        """
+        Look up the :class:`~range_streams.range_response.RangeResponse`
+        object associated with the currently active range by using
         :attr:`~range_streams.range_stream.RangeStream._active_range` as the
-        :class:`Range` key for the internal
-        :attr:`~range_streams.range_stream.RangeStream._ranges` :class:`RangeDict`.
+        :class:`~ranges.Range` key for the internal
+        :attr:`~range_streams.range_stream.RangeStream._ranges`
+        :class:`RangeDict`.
         """
         try:
             return self._ranges[self._active_range]
@@ -221,7 +225,8 @@ def active_range_response(self) -> RangeResponse:
             raise ValueError(f"{e_pre}({self._active_range=}")
 
     def ext2int(self, ext_rng: Range) -> RangeResponse:
-        """Given the external range `ext_rng` and the :class:`RangeStream`
+        """
+        Given the external range `ext_rng` and the :class:`RangeStream`
         on which it is 'stored' (or rather, computed, in the
         :attr:`~range_streams.range_stream.RangeStream.ranges` property),
         return the internal :class:`~ranges.Range` stored on the
@@ -230,7 +235,7 @@ def ext2int(self, ext_rng: Range) -> RangeResponse:
         shared :class:`~range_streams.range_response.RangeResponse` value.
 
         Args:
-          ext_rng : A :class:`ranges.Range` from the 'external'
+          ext_rng : A :class:`~ranges.Range` from the 'external'
                     :attr:`~range_streams.range_stream.RangeStream.ranges`
                     with which to cross-reference in
                     :attr:`~range_streams.range_stream.RangeStream._ranges`
diff --git a/src/range_streams/range_utils.py b/src/range_streams/range_utils.py
@@ -36,7 +36,7 @@ def ranges_in_reg_order(ranges: RangeDict) -> list[Range]:
 
 def response_ranges_in_reg_order(ranges: RangeDict) -> list[Range]:
     """For all of the :class:~range_streams.range_response.RangeResponse`
-    values in the :class:`ranges.RangeDict`, list the ranges from their
+    values in the :class:`~ranges.RangeDict`, list the ranges from their
     original :attribute:~range_streams.range_response.RangeResponse.request`
     in order of registration.
 
@@ -50,14 +50,15 @@ def response_ranges_in_reg_order(ranges: RangeDict) -> list[Range]:
 def most_recent_range(
     stream: range_streams.range_stream.RangeStream, internal: bool = True
 ) -> Range | None:
-    """For all of the :class:`~range_streams.range_response.RangeResponse`
-    values in the :class:`ranges.RangeDict`, list the ranges from their
+    """
+    For all of the :class:`~range_streams.range_response.RangeResponse`
+    values in the :class:`~ranges.RangeDict`, list the ranges from their
     original :attr:`~range_streams.range_response.RangeResponse.request`
     in order of registration.
 
     If ``internal`` is ``True``, use
     :attr:`~range_streams.range_stream.RangeStream._ranges` as the
-    :class:`ranges.RangeDict`, else use the 'external' (computed) property
+    :class:`~ranges.RangeDict`, else use the 'external' (computed) property
     :attr:`~range_streams.range_stream.RangeStream.ranges`. The external
     ones take into account the position the file has been read/seeked to.
 
@@ -76,7 +77,7 @@ def most_recent_range(
 
 def range_termini(rng: Range) -> tuple[int, int]:
     """Get the inclusive start and end positions ``[start,end]``
-    from a :class`ranges.Range`. These are referred to as the
+    from a :class:`ranges.Range`. These are referred to as the
     'termini'. Ranges are always ascending.
 
     Args:
@@ -130,7 +131,7 @@ def validate_range(
     byte_range: Range | tuple[int, int], allow_empty: bool = True
 ) -> Range:
     """Validate ``byte_range`` and convert to a half-closed (i.e.
-    not inclusive of the end position) ``[start,end)`` :class:`ranges.Range`
+    not inclusive of the end position) ``[start,end)`` :class:`~ranges.Range`
     if given as integer tuple.
 
     Args:
@@ -163,7 +164,7 @@ def range_span(ranges: list[Range]) -> Range:
     (i.e. the range spanned from their minimum to maximum). This span
     may of course not be completely 'covered' by the ranges in the list.
 
-    Assumes input list of :class:`RangeSets` are in ascending order,
+    Assumes input list of :class:`~ranges.RangeSet` are in ascending order,
     switches if not.
 
     Args:
