Broadcast shared reflector

examples/Cargo.toml

@@ -130,6 +130,10 @@ path = "log_stream.rs"
 name = "multi_watcher"
 path = "multi_watcher.rs"
 
+[[example]]
+name = "broadcast_reflector"
+path = "broadcast_reflector.rs"
+
 [[example]]
 name = "pod_api"
 path = "pod_api.rs"

examples/broadcast_reflector.rs

@@ -0,0 +1,113 @@
+use futures::{future, lock::Mutex, pin_mut, stream, StreamExt};
+use k8s_openapi::api::{
+    apps::v1::Deployment,
+    core::v1::{ConfigMap, Secret},
+};
+use kube::{
+    api::ApiResource,
+    runtime::{
+        broadcaster,
+        controller::Action,
+        reflector::multi_dispatcher::{BroadcastStream, MultiDispatcher},
+        watcher, Controller,
+    },
+    Api, Client, ResourceExt,
+};
+use std::{fmt::Debug, pin::pin, sync::Arc, time::Duration};
+use thiserror::Error;
+use tracing::*;
+
+#[derive(Debug, Error)]
+enum Infallible {}
+
+// A generic reconciler that can be used with any object whose type is known at
+// compile time. Will simply log its kind on reconciliation.
+async fn reconcile<K>(_obj: Arc<K>, _ctx: Arc<()>) -> Result<Action, Infallible>
+where
+    K: ResourceExt<DynamicType = ()>,
+{
+    let kind = K::kind(&());
+    info!("Reconciled {kind}");
+    Ok(Action::await_change())
+}
+
+fn error_policy<K: ResourceExt>(_: Arc<K>, _: &Infallible, _ctx: Arc<()>) -> Action {
+    info!("error");
+    Action::requeue(Duration::from_secs(10))
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    tracing_subscriber::fmt::init();
+    let client = Client::try_default().await?;
+
+    let writer = MultiDispatcher::new(128);
+
+    // multireflector stream
+    let combo_stream = Arc::new(Mutex::new(stream::select_all(vec![])));
+    let watcher = broadcaster(writer.clone(), BroadcastStream::new(combo_stream.clone()));
+
+    combo_stream.lock().await.push(
+        watcher::watcher(
+            Api::all_with(client.clone(), &ApiResource::erase::<Deployment>(&())),
+            Default::default(),
+        )
+        .boxed(),
+    );
+
+    // watching config maps, but ignoring in the final configuration
+    combo_stream.lock().await.push(
+        watcher::watcher(
+            Api::all_with(client.clone(), &ApiResource::erase::<ConfigMap>(&())),
+            Default::default(),
+        )
+        .boxed(),
+    );
+
+    // Combine duplicate type streams with narrowed down selection
+    combo_stream.lock().await.push(
+        watcher::watcher(
+            Api::default_namespaced_with(client.clone(), &ApiResource::erase::<Secret>(&())),
+            Default::default(),
+        )
+        .boxed(),
+    );
+    combo_stream.lock().await.push(
+        watcher::watcher(
+            Api::namespaced_with(client.clone(), "kube-system", &ApiResource::erase::<Secret>(&())),
+            Default::default(),
+        )
+        .boxed(),
+    );
+
+    let (sub, reader) = writer.subscribe::<Deployment>();
+    let deploy = Controller::for_shared_stream(sub, reader)
+        .shutdown_on_signal()
+        .run(reconcile, error_policy, Arc::new(()))
+        .for_each(|res| async move {
+            match res {
+                Ok(v) => info!("Reconciled deployment {v:?}"),
+                Err(error) => warn!(%error, "Failed to reconcile metadata"),
+            };
+        });
+
+    let (sub, reader) = writer.subscribe::<Secret>();
+    let secret = Controller::for_shared_stream(sub, reader)
+        .shutdown_on_signal()
+        .run(reconcile, error_policy, Arc::new(()))
+        .for_each(|res| async move {
+            match res {
+                Ok(v) => info!("Reconciled secret {v:?}"),
+                Err(error) => warn!(%error, "Failed to reconcile metadata"),
+            };
+        });
+
+    info!("long watches starting");
+    tokio::select! {
+        r = watcher.for_each(|_| future::ready(())) => println!("watcher exit: {r:?}"),
+        x = deploy => println!("deployments exit: {x:?}"),
+        x = secret => println!("secrets exit: {x:?}"),
+    }
+
+    Ok(())
+}

kube-core/src/dynamic.rs

@@ -5,6 +5,7 @@ pub use crate::discovery::ApiResource;
 use crate::{
     metadata::TypeMeta,
     resource::{DynamicResourceScope, Resource},
+    GroupVersionKind,
 };
 
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta;
@@ -73,6 +74,12 @@ impl DynamicObject {
     ) -> Result<K, ParseDynamicObjectError> {
         Ok(serde_json::from_value(serde_json::to_value(self)?)?)
     }
+
+    /// Returns the group, version, and kind (GVK) of this resource.
+    pub fn gvk(&self) -> Option<GroupVersionKind> {
+        let gvk = self.types.clone()?;
+        gvk.try_into().ok()
+    }
 }
 
 impl Resource for DynamicObject {

kube-core/src/gvk.rs

@@ -12,7 +12,7 @@ use thiserror::Error;
 pub struct ParseGroupVersionError(pub String);
 
 /// Core information about an API Resource.
-#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash, Default)]
 pub struct GroupVersionKind {
     /// API group
     pub group: String,

kube-core/src/metadata.rs

@@ -51,6 +51,22 @@ impl TypeMeta {
             kind: K::kind(&()).into(),
         }
     }
+
+    /// Construct a new `TypeMeta` for the object from the list `TypeMeta`.
+    ///
+    /// ```
+    /// # use k8s_openapi::api::core::v1::Pod;
+    /// # use kube_core::TypeMeta;
+    ///
+    /// let mut type_meta = TypeMeta::resource::<Pod>();
+    /// type_meta.kind = "PodList".to_string();
+    /// assert_eq!(type_meta.clone().singular_list().unwrap().kind, "Pod");
+    /// assert_eq!(type_meta.clone().singular_list().unwrap().api_version, "v1");
+    /// ```
+    pub fn singular_list(self) -> Option<Self> {
+        let kind = self.kind.strip_suffix("List")?.to_string();
+        (!kind.is_empty()).then_some(Self { kind, ..self })
+    }
 }
 
 /// A generic representation of any object with `ObjectMeta`.

kube-core/src/params.rs

@@ -8,7 +8,7 @@ use serde::Serialize;
 /// depending on what `resource_version`, `limit`, `continue_token` you include with the list request.
 ///
 /// See <https://kubernetes.io/docs/reference/using-api/api-concepts/#semantics-for-get-and-list> for details.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Hash)]
 pub enum VersionMatch {
     /// Returns data at least as new as the provided resource version.
     ///
@@ -36,7 +36,7 @@ pub enum VersionMatch {
 }
 
 /// Common query parameters used in list/delete calls on collections
-#[derive(Clone, Debug, Default, PartialEq)]
+#[derive(Clone, Debug, Default, PartialEq, Hash)]
 pub struct ListParams {
     /// A selector to restrict the list of returned objects by their labels.
     ///
@@ -305,7 +305,7 @@ impl ValidationDirective {
 }
 
 /// Common query parameters used in watch calls on collections
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Hash)]
 pub struct WatchParams {
     /// A selector to restrict returned objects by their labels.
     ///

kube-core/src/resource.rs

@@ -8,6 +8,8 @@ use std::{borrow::Cow, collections::BTreeMap};
 
 pub use k8s_openapi::{ClusterResourceScope, NamespaceResourceScope, ResourceScope, SubResourceScope};
 
+use crate::GroupVersionKind;
+
 /// Indicates that a [`Resource`] is of an indeterminate dynamic scope.
 pub struct DynamicResourceScope {}
 impl ResourceScope for DynamicResourceScope {}
@@ -54,6 +56,11 @@ pub trait Resource {
     /// This is known as the resource in apimachinery, we rename it for disambiguation.
     fn plural(dt: &Self::DynamicType) -> Cow<'_, str>;
 
+    /// Generates an object reference for the resource
+    fn gvk(dt: &Self::DynamicType) -> GroupVersionKind {
+        GroupVersionKind::gvk(&Self::group(dt), &Self::version(dt), &Self::kind(dt))
+    }
+
     /// Creates a url path for http requests for this resource
     fn url_path(dt: &Self::DynamicType, namespace: Option<&str>) -> String {
         let n = if let Some(ns) = namespace {

kube-runtime/src/controller/mod.rs

@@ -1766,7 +1766,7 @@ mod tests {
             queue_rx.map(Result::<_, Infallible>::Ok),
             Config::default(),
         ));
-        store_tx.apply_watcher_event(&watcher::Event::InitDone);
+        store_tx.apply_watcher_event(&watcher::Event::InitDone(None));
         for i in 0..items {
             let obj = ConfigMap {
                 metadata: ObjectMeta {
@@ -1776,7 +1776,7 @@ mod tests {
                 },
                 ..Default::default()
             };
-            store_tx.apply_watcher_event(&watcher::Event::Apply(obj.clone()));
+            store_tx.apply_watcher_event(&watcher::Event::Apply(obj.clone(), None));
             queue_tx.unbounded_send(ObjectRef::from_obj(&obj)).unwrap();
         }
 

kube-runtime/src/lib.rs

@@ -32,6 +32,7 @@ pub mod watcher;
 
 pub use controller::{applier, Config, Controller};
 pub use finalizer::finalizer;
+#[cfg(feature = "unstable-runtime-subscribe")] pub use reflector::broadcaster;
 pub use reflector::reflector;
 pub use scheduler::scheduler;
 pub use utils::WatchStreamExt;