feat: add spark support

Author: vorel99

src/ydata_profiling/model/pandas/__init__.py

@@ -3,10 +3,8 @@
     dataframe_pandas,
     describe_boolean_pandas,
     describe_categorical_pandas,
-    describe_counts_pandas,
     describe_date_pandas,
     describe_file_pandas,
-    describe_generic_pandas,
     describe_image_pandas,
     describe_numeric_pandas,
     describe_path_pandas,
@@ -27,10 +25,8 @@
     "dataframe_pandas",
     "describe_boolean_pandas",
     "describe_categorical_pandas",
-    "describe_counts_pandas",
     "describe_date_pandas",
     "describe_file_pandas",
-    "describe_generic_pandas",
     "describe_image_pandas",
     "describe_numeric_pandas",
     "describe_path_pandas",

src/ydata_profiling/model/pandas/describe_counts_pandas.py

@@ -3,9 +3,7 @@
     dataframe_spark,
     describe_boolean_spark,
     describe_categorical_spark,
-    describe_counts_spark,
     describe_date_spark,
-    describe_generic_spark,
     describe_numeric_spark,
     describe_supported_spark,
     duplicates_spark,
@@ -21,9 +19,7 @@
     "dataframe_spark",
     "describe_boolean_spark",
     "describe_categorical_spark",
-    "describe_counts_spark",
     "describe_date_spark",
-    "describe_generic_spark",
     "describe_numeric_spark",
     "describe_supported_spark",
     "duplicates_spark",

src/ydata_profiling/model/pandas/describe_generic_pandas.py

@@ -1,4 +1,5 @@
 """Correlations between variables."""
+
 from typing import Optional
 
 import pandas as pd

src/ydata_profiling/model/spark/__init__.py

@@ -4,12 +4,13 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.summary_algorithms import describe_boolean_1d
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 @describe_boolean_1d.register
 def describe_boolean_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a boolean series.
 
     Args:
@@ -20,7 +21,7 @@ def describe_boolean_1d_spark(
         A dict containing calculated series description values.
     """
 
-    value_counts = summary["value_counts"]
+    value_counts = summary.value_counts
 
     # get the most common boolean value and its frequency
     top = value_counts.first()

src/ydata_profiling/model/spark/correlations_spark.py

@@ -4,12 +4,13 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.summary_algorithms import describe_categorical_1d
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 @describe_categorical_1d.register
 def describe_categorical_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a categorical series.
 
     Args:

src/ydata_profiling/model/spark/describe_boolean_spark.py

@@ -6,6 +6,7 @@
 
 from ydata_profiling.config import Settings
 from ydata_profiling.model.summary_algorithms import describe_date_1d
+from ydata_profiling.model.var_description.default import VarDescription
 
 
 def date_stats_spark(df: DataFrame, summary: dict) -> dict:
@@ -21,8 +22,8 @@ def date_stats_spark(df: DataFrame, summary: dict) -> dict:
 
 @describe_date_1d.register
 def describe_date_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a date series.
 
     Args:

src/ydata_profiling/model/spark/describe_categorical_spark.py

@@ -9,9 +9,10 @@
     describe_numeric_1d,
     histogram_compute,
 )
+from ydata_profiling.model.var_description.default import VarDescription
 
 
-def numeric_stats_spark(df: DataFrame, summary: dict) -> dict:
+def numeric_stats_spark(df: DataFrame, summary: VarDescription) -> dict:
     column = df.columns[0]
 
     expr = [
@@ -29,8 +30,8 @@ def numeric_stats_spark(df: DataFrame, summary: dict) -> dict:
 
 @describe_numeric_1d.register
 def describe_numeric_1d_spark(
-    config: Settings, df: DataFrame, summary: dict
-) -> Tuple[Settings, DataFrame, dict]:
+    config: Settings, df: DataFrame, summary: VarDescription
+) -> Tuple[Settings, DataFrame, VarDescription]:
     """Describe a boolean series.
 
     Args:
@@ -51,7 +52,7 @@ def describe_numeric_1d_spark(
     summary["kurtosis"] = stats["kurtosis"]
     summary["sum"] = stats["sum"]
 
-    value_counts = summary["value_counts"]
+    value_counts = summary.value_counts
 
     n_infinite = (
         value_counts.where(F.col(df.columns[0]).isin([np.inf, -np.inf]))
@@ -106,12 +107,12 @@ def describe_numeric_1d_spark(
     ).stat.approxQuantile("abs_dev", [0.5], quantile_threshold)[0]
 
     # FIXME: move to fmt
-    summary["p_negative"] = summary["n_negative"] / summary["n"]
+    summary["p_negative"] = summary["n_negative"] / summary.n
     summary["range"] = summary["max"] - summary["min"]
     summary["iqr"] = summary["75%"] - summary["25%"]
     summary["cv"] = summary["std"] / summary["mean"] if summary["mean"] else np.NaN
-    summary["p_zeros"] = summary["n_zeros"] / summary["n"]
-    summary["p_infinite"] = summary["n_infinite"] / summary["n"]
+    summary["p_zeros"] = summary["n_zeros"] / summary.n
+    summary["p_infinite"] = summary["n_infinite"] / summary.n
 
     # TODO - enable this feature
     # because spark doesn't have an indexing system, there isn't really the idea of monotonic increase/decrease
@@ -124,14 +125,14 @@ def describe_numeric_1d_spark(
     # display in pandas display
     # the alternative is to do this in spark natively, but it is not trivial
     infinity_values = [np.inf, -np.inf]
-    infinity_index = summary["value_counts_without_nan"].index.isin(infinity_values)
+    infinity_index = summary.value_counts_without_nan.index.isin(infinity_values)
 
     summary.update(
         histogram_compute(
             config,
-            summary["value_counts_without_nan"][~infinity_index].index.values,
+            summary.value_counts_without_nan[~infinity_index].index.values,
             summary["n_distinct"],
-            weights=summary["value_counts_without_nan"][~infinity_index].values,
+            weights=summary.value_counts_without_nan[~infinity_index].values,
         )
     )