Malloc/memleak fixes (#169)

Bugfixes:
- Update how finalize and dispose works
- Avoid exception due to disposing dictionary twice when it is linked to the same pointer
- Fix 'contains' implementation for netdict
- Fix incorrect scalar state callback value for on_flushed event
- Fix statevalue leaks

Other:
- Use super().__init__ instaed of <classname>.__init__

Author: peter-quix

src/InteropGenerator/Quix.InteropGenerator/Writers/CsharpInteropWriter/InteropHelpers/ExternalTypes/System/Dictionary.cs

@@ -239,6 +239,39 @@ public static void SetValue(IntPtr dictionaryHPtr, IntPtr keyHPtr, IntPtr valHPt
         }
     }
 
+    [UnmanagedCallersOnly(EntryPoint = "dictionary_contains")]
+    public static bool Contains(IntPtr dictionaryHPtr, IntPtr keyPtr)
+    {
+        object key = null;
+        Type keyType = null;
+        Type dictType = null;
+        try
+        {
+            var target = InteropUtils.FromHPtr<IDictionary>(dictionaryHPtr);
+            var targetType = target.GetType();
+            dictType = (targetType.IsGenericType && targetType.GetGenericTypeDefinition() == typeof(IDictionary<,>))
+                ? targetType
+                : targetType.GetInterfaces().FirstOrDefault(y => y.IsGenericType && y.GetGenericTypeDefinition() == typeof(IDictionary<,>));
+            if (dictType == null)
+            {
+                throw new InvalidOperationException($"Type {targetType} does not implement dictionary with types nor is a dictionary with types");
+            }
+            keyType = dictType.GetGenericArguments()[0];
+
+            key = InteropUtils.PtrToObject(keyPtr, keyType);
+
+            return target.Contains(key);
+        }
+        catch (Exception ex)
+        {
+            InteropUtils.LogDebug("Exception in dictionary_get_value");
+            InteropUtils.LogDebug($"Arg dictionaryHPtr (IntPtr) has value: {dictionaryHPtr}, converted to dict type {dictType}");
+            InteropUtils.LogDebug($"Arg keyPtr (IntPtr) has value: {key}, type {keyType}, converted {key}");
+            InteropUtils.RaiseException(ex);
+            return default;
+        }
+    }
+    
     [UnmanagedCallersOnly(EntryPoint = "dictionary_get_value")]
     public static IntPtr GetValue(IntPtr dictionaryHPtr, IntPtr keyHPtr)
     {

src/InteropGenerator/Quix.InteropGenerator/Writers/PythonWrapperWriter/InteropHelpers/ExternalTypes/System/Dictionary.py

@@ -212,6 +212,14 @@ def SetValue(dictionary_hptr: c_void_p, key_hptr: c_void_p, value_hptr: c_void_p
     def GetValue(dictionary_hptr: c_void_p, key_hptr: c_void_p) -> Any:
         ptr = InteropUtils.invoke("dictionary_get_value", dictionary_hptr, key_hptr)
         return c_void_p(ptr) if ptr is not None else None
+    
+    # ctypes function return type//parameter fix
+    interop_func = InteropUtils.get_function("dictionary_contains")
+    interop_func.argtypes = [c_void_p, c_void_p]
+    interop_func.restype = c_bool
+    @staticmethod
+    def Contains(dictionary_hptr: c_void_p, key_hptr: c_void_p) -> Any:
+        return InteropUtils.invoke("dictionary_contains", dictionary_hptr, key_hptr)
 
     # ctypes function return type//parameter fix
     interop_func = InteropUtils.get_function("dictionary_get_keys")

src/PythonClient/setup.py

@@ -5,7 +5,7 @@
 import re
 import fileinput
 
-package_version = "0.5.5.dev6"
+package_version = "0.5.5.dev11"
 
 with open("README.md", "r") as fh:
     long_description = fh.read()

src/PythonClient/src/quixstreams/__init__.py

@@ -59,3 +59,5 @@
 
 from .logging import Logging, LogLevel
 from .state import *
+from .states import *
+

src/PythonClient/src/quixstreams/helpers/nativedecorator.py

@@ -18,12 +18,17 @@ def new_init(self, *args, **kwargs):
         orig_init(self, *args, **kwargs)
 
     def new_finalizerfunc(self):
+        """
+        Finalizes the underlying object
+        Finalize is implementation specific but generally de-references the underlying objects which
+        may or may not result in garbage collection of the de-referenced objects
+        """
         if self._nativedecorator_finalized:
             return
 
         ptr = getattr(self, "_interop").get_interop_ptr__()
 
-        InteropUtils.log_debug(f"Finalizing {cls.__name__} ({ptr.value})")
+        InteropUtils.log_debug(f"Finalizing {cls.__name__} ({ptr.value}) of object {id(self)}")
         InteropUtils.log_debug_indent_increment()
 
         self._nativedecorator_finalized = True
@@ -32,7 +37,7 @@ def new_finalizerfunc(self):
         getattr(self, "_interop").dispose_ptr__()
 
         InteropUtils.log_debug_indent_decrement()
-        InteropUtils.log_debug(f"Finalized {cls.__name__} ({ptr.value})")
+        InteropUtils.log_debug(f"Finalized {cls.__name__} ({ptr.value}) of object {id(self)}")
 
 
     def new_del(self):
@@ -47,11 +52,16 @@ def new_exit(self, exc_type, exc_val, exc_tb):
         new_dispose(self)
         return result
 
-    def new_dispose(self, *args, **kwargs):
+    def new_dispose(self, *args, **kwargs) -> None:
+        """
+        Disposes the underlying object in addition to finalizing it
+        Dispose is implementation specific but it generally frees underlying resources rather than just
+        de-references
+        """
 
         ptr = getattr(self, "_interop").get_interop_ptr__()
 
-        InteropUtils.log_debug(f"Disposing {cls.__name__} ({ptr.value})")
+        InteropUtils.log_debug(f"Disposing {cls.__name__} ({ptr.value}) of object {id(self)}")
         InteropUtils.log_debug_indent_increment()
 
         if self._nativedecorator_finalized:
@@ -61,7 +71,7 @@ def new_dispose(self, *args, **kwargs):
         new_finalizerfunc(self)
 
         InteropUtils.log_debug_indent_decrement()
-        InteropUtils.log_debug(f"Disposed {cls.__name__} ({ptr.value})")
+        InteropUtils.log_debug(f"Disposed {cls.__name__} ({ptr.value}) of object {id(self)}")
 
     cls.__init__ = new_init
     cls._finalizerfunc = new_finalizerfunc

src/PythonClient/src/quixstreams/models/netdict.py

@@ -18,7 +18,8 @@ def _returnsame(val):
 
     def __init__(self, net_pointer, key_converter_to_python=None, key_converter_from_python=None,
                  val_converter_to_python=None, val_converter_from_python=None,
-                 key_converter_to_python_list=None, val_converter_to_python_list=None):
+                 key_converter_to_python_list=None, val_converter_to_python_list=None,
+                 self_dispose: bool = True):
 
         self._pointer = net_pointer
         self._key_converter_to_python = key_converter_to_python
@@ -49,13 +50,16 @@ def __init__(self, net_pointer, key_converter_to_python=None, key_converter_from
                                                                                      converter_to_python=self._val_converter_to_python,
                                                                                      converter_from_python=self._val_converter_from_python)
 
-        self._finalizer = weakref.finalize(self, self._finalizerfunc)
+        if self_dispose:
+            self._finalizer = weakref.finalize(self, self._finalizerfunc)
+        else:
+            self._finalizer = lambda x: None
 
     def _finalizerfunc(self):
         self._finalizer.detach()
         InteropUtils.free_hptr(self._pointer)
 
-    def dispose(self):
+    def dispose(self) -> None:
         self._finalizer()
 
     def _get_actual_key_from(self, key) -> Any:
@@ -75,9 +79,9 @@ def __getitem__(self, key) -> Any:
         item = di.GetValue(self._pointer, actual_key)
         return self._get_actual_value_to(item)
 
-    def __contains__(self, item) -> bool:
-        actual_item = self._get_actual_value_from(item)
-        return di.GetValue(self._pointer, actual_item)
+    def __contains__(self, key) -> bool:
+        actual_key = self._get_actual_key_from(key)
+        return di.Contains(self._pointer, actual_key)
 
     def __str__(self) -> str:
         text = "{"
@@ -132,16 +136,17 @@ class NetDict(ReadOnlyNetDict):
 
     def __init__(self, net_pointer, key_converter_to_python=None, key_converter_from_python=None,
                  val_converter_to_python=None, val_converter_from_python=None,
-                 key_converter_to_python_list=None, val_converter_to_python_list=None):
-
-        ReadOnlyNetDict.__init__(self,
-                                 net_pointer=net_pointer,
-                                 key_converter_to_python=key_converter_to_python,
-                                 key_converter_from_python=key_converter_from_python,
-                                 val_converter_to_python=val_converter_to_python,
-                                 val_converter_from_python=val_converter_from_python,
-                                 key_converter_to_python_list=key_converter_to_python_list,
-                                 val_converter_to_python_list=val_converter_to_python_list)
+                 key_converter_to_python_list=None, val_converter_to_python_list=None,
+                 self_dispose: bool = True):
+
+        super().__init__(net_pointer=net_pointer,
+                         key_converter_to_python=key_converter_to_python,
+                         key_converter_from_python=key_converter_from_python,
+                         val_converter_to_python=val_converter_to_python,
+                         val_converter_from_python=val_converter_from_python,
+                         key_converter_to_python_list=key_converter_to_python_list,
+                         val_converter_to_python_list=val_converter_to_python_list,
+                         self_dispose=self_dispose)
 
     @staticmethod
     def constructor_for_string_string(net_pointer=None):

src/PythonClient/src/quixstreams/models/netlist.py

@@ -28,7 +28,7 @@ def _finalizerfunc(self):
         self._finalizer.detach()
         InteropUtils.free_hptr(self._pointer)
 
-    def dispose(self):
+    def dispose(self) -> None:
         self._finalizer()
 
     def _get_actual_from(self, value):
@@ -82,7 +82,7 @@ class NetList(NetReadOnlyList):
     """
 
     def __init__(self, net_pointer, converter_to_python=None, converter_from_python=None):
-        NetReadOnlyList.__init__(self, net_pointer, converter_to_python, converter_from_python)
+        super().__init__(net_pointer, converter_to_python, converter_from_python)
 
     @staticmethod
     def constructor_for_string(net_pointer=None):

src/PythonClient/src/quixstreams/models/streamconsumer/streampropertiesconsumer.py

@@ -42,6 +42,9 @@ def __init__(self, stream_consumer: 'StreamConsumer', net_pointer: ctypes.c_void
         self._parents = None
 
     def _finalizerfunc(self):
+        self._on_changed_dispose()
+
+    def dispose(self) -> None:
         if self._metadata is not None:
             self._metadata.dispose()
         if self._parents is not None:

src/PythonClient/src/quixstreams/models/streamconsumer/streamtimeseriesconsumer.py

@@ -56,6 +56,12 @@ def __init__(self, stream_consumer, net_pointer: ctypes.c_void_p):
         self._on_definitions_changed_refs = None  # Keeping references to avoid GC.
 
     def _finalizerfunc(self):
+        self._on_data_received_dispose()
+        self._on_raw_received_dispose()
+        self._on_dataframe_received_dispose()
+        self._on_definitions_changed_dispose()
+
+    def dispose(self) -> None:
         [buffer.dispose() for buffer in self._buffers]
         self._on_data_received_dispose()
         self._on_raw_received_dispose()

src/PythonClient/src/quixstreams/models/streamconsumer/timeseriesbufferconsumer.py

@@ -28,7 +28,7 @@ def __init__(self, stream_consumer, net_pointer: ctypes.c_void_p = None):
             raise Exception("TimeseriesBufferConsumer is none")
 
         self._interop = tsbci(net_pointer)
-        TimeseriesBuffer.__init__(self, stream_consumer, net_pointer)
+        super().__init__(stream_consumer, net_pointer)
 
     def get_net_pointer(self) -> ctypes.c_void_p:
         """