improved type support

mypy.ini

@@ -0,0 +1,33 @@
+[mypy]
+python_version = 3.9
+warn_return_any = True
+warn_unused_configs = True
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+check_untyped_defs = True
+disallow_untyped_decorators = True
+no_implicit_optional = True
+warn_redundant_casts = True
+warn_unused_ignores = True
+warn_no_return = True
+warn_unreachable = True
+strict_equality = True
+
+# Allow untyped calls for external libraries
+[mypy-make_it_sync]
+ignore_missing_imports = True
+
+[mypy-aioboto3.*]
+ignore_missing_imports = True
+
+[mypy-boto3.*]
+ignore_missing_imports = True
+
+[mypy-qastle]
+ignore_missing_imports = True
+
+[mypy-func_adl.*]
+ignore_missing_imports = True
+
+[mypy-ruamel.*]
+ignore_missing_imports = True

servicex/_version.py

@@ -1,3 +1,3 @@
-import importlib.metadata  # type: ignore
+import importlib.metadata
 
 __version__ = importlib.metadata.version("servicex")

servicex/app/__init__.py

@@ -25,15 +25,16 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import rich
 
 
-def is_terminal_output():
+def is_terminal_output() -> bool:
     import sys
 
     return sys.stdout.isatty()
 
 
-def pipeable_table(title: str):
+def pipeable_table(title: str) -> rich.table.Table:
     """
     Create a table that can be used in a pipeable command. Make it pretty if we
     are outputting to a terminal, otherwise just make it as simple as possible.

servicex/app/cache.py

@@ -40,15 +40,15 @@
 
 
 @cache_app.callback()
-def cache():
+def cache() -> None:
     """
     Sub-commands for creating and manipulating the local query cache
     """
     pass
 
 
 @cache_app.command()
-def list():
+def list() -> None:
     """
     List the cached queries
     """
@@ -75,19 +75,20 @@ def list():
 
 
 @cache_app.command()
-def clear(force: bool = force_opt):
+def clear(force: bool = force_opt) -> None:
     """
     Clear the local query cache
     """
     if force or Confirm.ask("Really clear cache and delete downloaded files?"):
         sx = ServiceXClient()
         sx.query_cache.close()
-        shutil.rmtree(sx.config.cache_path)
+        if sx.config.cache_path:
+            shutil.rmtree(sx.config.cache_path)
         rich.print("Cache cleared")
 
 
 @cache_app.command(no_args_is_help=True)
-def delete(transform_id: str = transform_id_arg):
+def delete(transform_id: str = transform_id_arg) -> None:
     """
     Delete a cached query. Use -t to specify the transform ID
     """

servicex/app/codegen.py

@@ -40,7 +40,7 @@
 def list(
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
-):
+) -> None:
     """
     List the available code generators
     """

servicex/app/datasets.py

@@ -58,7 +58,7 @@ def list(
     config_path: Optional[str] = config_file_option,
     did_finder: Optional[str] = did_finder_opt,
     show_deleted: Optional[bool] = show_deleted_opt,
-):
+) -> None:
     """
     List the datasets. Use fancy formatting if printing to a terminal.
     Output as plain text if redirected.
@@ -102,7 +102,7 @@ def get(
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
     dataset_id: int = dataset_id_get_arg,
-):
+) -> None:
     """
     Get the details of a dataset. Output as a pretty, nested table if printing to a terminal.
     Output as json if redirected.
@@ -116,7 +116,7 @@ def get(
 
     dataset = sx.get_dataset(dataset_id)
 
-    if table:
+    if table and dataset.files:
         for file in dataset.files:
             sub_table = Table(title="")
             sub_table.add_column(f"File ID: {file.id}")
@@ -134,7 +134,7 @@ def get(
                 "name": dataset.name,
                 "files": [
                     {"id": file.id, "paths": file.paths.split(",")}
-                    for file in dataset.files
+                    for file in (dataset.files or [])
                 ],
             }
         }
@@ -146,7 +146,7 @@ def delete(
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
     dataset_id: int = dataset_id_delete_arg,
-):
+) -> None:
     sx = ServiceXClient(backend=backend, config_path=config_path)
     result = sx.delete_dataset(dataset_id)
     if result:

servicex/app/main.py

@@ -51,7 +51,7 @@
 )
 
 
-def show_version(show: bool):
+def show_version(show: bool) -> None:
     """Display the installed version and quit."""
     if show:
         rich.print(f"ServiceX {__version__}")
@@ -62,7 +62,7 @@ def show_version(show: bool):
 
 
 @app.callback()
-def main_info(version: Optional[bool] = version_opt):
+def main_info(version: Optional[bool] = version_opt) -> None:
     """
     ServiceX Client
     """
@@ -75,7 +75,7 @@ def deliver(
     config_path: Optional[str] = config_file_option,
     spec_file: str = spec_file_arg,
     ignore_cache: Optional[bool] = ignore_cache_opt,
-):
+) -> None:
     """
     Deliver a file to the ServiceX cache.
     """

servicex/app/transforms.py

@@ -64,7 +64,7 @@
 
 
 @transforms_app.callback()
-def transforms():
+def transforms() -> None:
     """
     Sub-commands for interacting with transforms.
     """
@@ -77,7 +77,7 @@ def list(
     config_path: Optional[str] = config_file_option,
     complete: Optional[bool] = complete_opt,
     running: Optional[bool] = running_opt,
-):
+) -> None:
     """
     List the transforms that have been run.
     """
@@ -111,7 +111,7 @@ def files(
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
     transform_id: str = transform_id_arg,
-):
+) -> None:
     """
     List the files that were produced by a transform.
     """
@@ -139,20 +139,22 @@ def download(
     transform_id: str = transform_id_arg,
     local_dir: str = local_dir_opt,
     concurrency: int = concurrency_opt,
-):
+) -> None:
     """
     Download the files that were produced by a transform.
     """
 
-    async def download_files(sx: ServiceXClient, transform_id: str, local_dir):
+    async def download_files(
+        sx: ServiceXClient, transform_id: str, local_dir: str
+    ) -> None:
         s3_semaphore = asyncio.Semaphore(concurrency)
 
-        async def download_with_progress(filename) -> Path:
+        async def download_with_progress(filename: str) -> Path:
             async with s3_semaphore:
                 p = await minio.download_file(
                     filename,
                     local_dir,
-                    shorten_filename=sx.config.shortened_downloaded_filename,
+                    shorten_filename=sx.config.shortened_downloaded_filename or False,
                 )
             progress.advance(download_progress)
             return p
@@ -180,7 +182,7 @@ def delete(
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
     transform_id_list: List[str] = transform_id_arg,
-):
+) -> None:
     """
     Delete a completed transform along with the result files.
     """
@@ -197,7 +199,7 @@ def cancel(
     backend: Optional[str] = backend_cli_option,
     config_path: Optional[str] = config_file_option,
     transform_id_list: List[str] = transform_id_arg,
-):
+) -> None:
     """
     Cancel a running transform request.
     """
@@ -226,15 +228,15 @@ class LogLevel(str, Enum):
     error = ("ERROR",)
 
 
-def add_query(key, value):
+def add_query(key: str, value: str) -> str:
     """
     Creates query string from the key and value pairs
     """
     query_string = "(query:(match_phrase:({0}:'{1}')))".format(key, value)
     return query_string
 
 
-def select_time(time_frame=TimeFrame.day):
+def select_time(time_frame: TimeFrame = TimeFrame.day) -> str:
     """
     Takes input as 'day','week','month' and returns the time filter
     """
@@ -251,8 +253,11 @@ def select_time(time_frame=TimeFrame.day):
 
 
 def create_kibana_link_parameters(
-    log_url, transform_id=None, log_level=None, time_frame=None
-):
+    log_url: str,
+    transform_id: Optional[str] = None,
+    log_level: Optional["LogLevel"] = None,
+    time_frame: Optional[TimeFrame] = None,
+) -> str:
     """
     Create the _a and _g parameters for the kibana dashboard link
     """
@@ -274,7 +279,7 @@ def logs(
     transform_id: str = transform_id_arg,
     log_level: Optional[LogLevel] = log_level_opt,
     time_frame: Optional[TimeFrame] = time_frame_opt,
-):
+) -> None:
     """
     Open the URL to the Kibana dashboard of the logs of a tranformer
     """

servicex/configuration.py

@@ -55,7 +55,7 @@ class Configuration(BaseModel):
     config_file: Optional[str] = Field(default=None, exclude=True)
 
     @model_validator(mode="after")
-    def expand_cache_path(self):
+    def expand_cache_path(self) -> "Configuration":
         """
         Expand the cache path to a full path, and create it if it doesn't exist.
         Expand ${USER} to be the user name on the system. Works for windows, too.
@@ -91,7 +91,7 @@ def endpoint_dict(self) -> Dict[str, Endpoint]:
         return {endpoint.name: endpoint for endpoint in self.api_endpoints}
 
     @classmethod
-    def read(cls, config_path: Optional[str] = None):
+    def read(cls, config_path: Optional[str] = None) -> "Configuration":
         r"""
         Read configuration from .servicex or servicex.yaml file.
         :param config_path: If provided, use this as the path to the .servicex file.
@@ -118,7 +118,9 @@ def read(cls, config_path: Optional[str] = None):
             )
 
     @classmethod
-    def _add_from_path(cls, path: Optional[Path] = None, walk_up_tree: bool = False):
+    def _add_from_path(
+        cls, path: Optional[Path] = None, walk_up_tree: bool = False
+    ) -> tuple[Optional[dict], Optional[Path]]:
         config = None
         found_file: Optional[Path] = None
         if path:

servicex/dataset_group.py

@@ -32,7 +32,7 @@
 from servicex.query_core import Query
 from servicex.expandable_progress import ExpandableProgress
 from servicex.models import TransformedResults, ResultFormat
-from make_it_sync import make_sync
+from make_it_sync import make_sync  # type: ignore
 
 
 DatasetGroupMember = Query
@@ -47,7 +47,7 @@ class to allow you to submit multiple datasets to a ServiceX instance and
 
         :param datasets: List of transform request as dataset instances
         """
-        self.tasks = []
+        self.tasks: List = []
         self.datasets = datasets
 
     def set_result_format(self, result_format: ResultFormat):