dns shenanigans to get tenant resolution testing working

Author: jubrad

misc/python/materialize/mzcompose/composition.py

@@ -134,6 +134,9 @@ def __init__(
 
         self.compose: dict[str, Any] = {
             "services": {},
+            "networks": {
+                "mzcompose": {"ipam": {"config": [{"subnet": "10.10.0.0/24"}]}}
+            },
         }
 
         # Add default volumes

misc/python/materialize/mzcompose/service.py

@@ -135,7 +135,7 @@ class ServiceConfig(TypedDict, total=False):
     volumes: list[str]
     """Volumes to attach to the service."""
 
-    networks: dict[str, dict[str, list[str]]]
+    networks: dict[str, dict[str, list[str]]] | dict[str, dict[str, str]] | list[str]
     """Additional networks to join.
 
     TODO(benesch): this should use a nested TypedDict.
@@ -178,6 +178,9 @@ class ServiceConfig(TypedDict, total=False):
     user: str | None
     """The user for the container."""
 
+    dns: list[str] | None
+    """The DNS servers to use for the container."""
+
 
 class Service:
     """A Docker Compose service in a `Composition`.

misc/python/materialize/mzcompose/services/balancerd.py

@@ -16,13 +16,21 @@
 
 
 class Balancerd(Service):
+
     def __init__(
         self,
         name: str = "balancerd",
         mzbuild: str = "balancerd",
         command: list[str] | None = None,
         volumes: list[str] = [],
         depends_on: list[str] = [],
+        networks: (
+            dict[str, dict[str, list[str]]]
+            | dict[str, dict[str, str]]
+            | list[str]
+            | None
+        ) = None,
+        dns: list[str] | None = None,
         https_resolver_template: str | None = None,
         frontegg_resolver_template: str | None = None,
         static_resolver_addr: str | None = None,
@@ -56,6 +64,11 @@ def __init__(
             "volumes": volumes,
             "depends_on": depends_graph,
         }
+        if dns:
+            config["dns"] = dns
+        if networks:
+            config["networks"] = networks
+
         super().__init__(
             name=name,
             config=config,

misc/python/materialize/mzcompose/services/dnsmasq.py

@@ -0,0 +1,72 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+
+from typing import Literal, TypedDict
+
+from materialize.mzcompose.service import (
+    Service,
+    ServiceConfig,
+    ServiceDependency,
+)
+
+
+class DnsmasqEntry(TypedDict):
+    type: Literal["cname", "address"]
+    key: str
+    value: str
+
+
+def dnsmask_entry_to_rec(e: DnsmasqEntry) -> str:
+    match e["type"]:
+        case "address":
+            return f"--entry=address=/{e['key']}/{e['value']}"
+        case "cname":
+            return f"--entry=cname={e['key']},{e['value']}"
+
+
+class Dnsmasq(Service):
+
+    def __init__(
+        self,
+        name: str = "dnsmasq",
+        mzbuild: str = "dnsmasq",
+        command: list[str] | None = None,
+        depends_on: list[str] = [],
+        networks: (
+            dict[str, dict[str, list[str]]]
+            | dict[str, dict[str, str]]
+            | list[str]
+            | None
+        ) = None,
+        dns_overrides: list[DnsmasqEntry] = [],
+    ) -> None:
+
+        command = [dnsmask_entry_to_rec(e) for e in dns_overrides]
+
+        depends_graph: dict[str, ServiceDependency] = {
+            s: {"condition": "service_started"} for s in depends_on
+        }
+        config: ServiceConfig = {
+            "mzbuild": mzbuild,
+            "command": command,
+            "ports": [
+                "53:53/tcp",
+                "53:53/udp",
+            ],
+            "depends_on": depends_graph,
+            "allow_host_ports": True,
+        }
+        if networks:
+            config["networks"] = networks
+
+        super().__init__(
+            name=name,
+            config=config,
+        )

misc/python/materialize/mzcompose/services/materialized.py

@@ -97,13 +97,15 @@ def __init__(
         default_replication_factor: int = 1,
         listeners_config_path: str = f"{MZ_ROOT}/src/materialized/ci/listener_configs/no_auth.json",
         support_external_clusterd: bool = False,
+        networks: (
+            dict[str, dict[str, list[str]]] | dict[str, dict[str, str]] | None
+        ) = None,
     ) -> None:
         if name is None:
             name = "materialized"
 
         if healthcheck is None:
             healthcheck = ["CMD", "curl", "-f", "localhost:6878/api/readyz"]
-
         depends_graph: dict[str, ServiceDependency] = {
             s: {"condition": "service_started"} for s in depends_on
         }
@@ -338,6 +340,9 @@ def __init__(
             volumes += DEFAULT_MZ_VOLUMES
         volumes += volumes_extra
 
+        if networks:
+            config["networks"] = networks
+
         config.update(
             {
                 "depends_on": depends_graph,

src/balancerd/src/lib.rs

@@ -1319,17 +1319,16 @@ impl Resolver {
                         }),
                     ) => {
                         let sni_addr = sni_addr_template.replace("{}", servername);
-                        // let tenant = stub_resolver.tenant(&sni_addr).await;
+                        let tenant = stub_resolver.tenant(&sni_addr).await;
                         let sni_addr = format!("{sni_addr}:{port}");
                         let addr = lookup(&sni_addr).await?;
-                        // if tenant.is_some() {
-                        //     debug!("SNI header found for tenant {:?}", tenant);
-                        // }
+                        if tenant.is_some() {
+                            debug!("SNI header found for tenant {:?}", tenant);
+                        }
                         ResolvedAddr {
                             addr,
                             password: None,
-                            tenant: None,
-                            // tenant,
+                            tenant,
                         }
                     }
                     _ => {

test/balancerd/mzcompose.py

@@ -33,6 +33,7 @@
 from materialize import MZ_ROOT
 from materialize.mzcompose.composition import Composition, Service
 from materialize.mzcompose.services.balancerd import Balancerd
+from materialize.mzcompose.services.dnsmasq import Dnsmasq, DnsmasqEntry
 from materialize.mzcompose.services.frontegg import FronteggMock
 from materialize.mzcompose.services.materialized import Materialized
 from materialize.mzcompose.services.mz import Mz
@@ -84,10 +85,32 @@ def app_password(email: str) -> str:
 SERVICES = [
     TestCerts(),
     Testdrive(
-        materialize_url=f"postgres://{quote(ADMIN_USER)}:{app_password(ADMIN_USER)}@balancerd:6875?sslmode=require",
+        materialize_url=f"postgres://{quote(ADMIN_USER)}:{app_password(ADMIN_USER)}@:6875?sslmode=require",
         materialize_use_https=True,
         no_reset=True,
     ),
+    Dnsmasq(
+        dns_overrides=[
+            # Docker compose will insert into all service pods an arec for
+            # materialized. But what we need is a cname that points to an arec
+            # with the tenant id in the name and the value should point to the
+            # ip of materailized. We're going to use a new network for this
+            # to ensure that we can get a unique ip space and we're going to
+            # use explicit IPs for dnsmasq to set it it as the dns server for
+            # balancerd.
+            DnsmasqEntry(
+                type="cname",
+                key="materialized",
+                value="environmentd.environment-58cd23ff-a4d7-4bd0-ad85-a6ff29cc86c3-0.svc.cluster.local",
+            ),
+            DnsmasqEntry(
+                type="address",
+                key="environmentd.environment-58cd23ff-a4d7-4bd0-ad85-a6ff29cc86c3-0.svc.cluster.local",
+                value="10.10.0.4",
+            ),
+        ],
+        networks={"mzcompose": {"ipv4_address": "10.10.0.2"}},
+    ),
     Balancerd(
         command=[
             "--startup-log-filter=debug",
@@ -115,6 +138,9 @@ def app_password(email: str) -> str:
         volumes=[
             "secrets:/secrets",
         ],
+        # Points to DNSMasq which has an explicit ip
+        dns=["10.10.0.2"],
+        networks={"mzcompose": {"ipv4_address": "10.10.0.3"}},
     ),
     FronteggMock(
         issuer=FRONTEGG_URL,
@@ -147,6 +173,7 @@ def app_password(email: str) -> str:
             "secrets:/secrets",
         ],
         listeners_config_path=f"{MZ_ROOT}/src/materialized/ci/listener_configs/no_auth_https.json",
+        networks={"mzcompose": {"ipv4_address": "10.10.0.4"}},
     ),
 ]
 
@@ -661,10 +688,9 @@ def workflow_user(c: Composition) -> None:
 
 
 def workflow_many_connections(c: Composition) -> None:
-    c.up("balancerd", "frontegg-mock", "materialized")
-
+    c.up("balancerd", "dnsmasq", "frontegg-mock", "materialized")
     cursors = []
-    connections = 1000 - 10  #  Go almost to the limit, but not above
+    connections = 100 - 10  #  Go almost to the limit, but not above
     print(f"Opening {connections} connections.")
     start = time.time()
     for _ in range(connections):

test/dnsmasq/Dockerfile

@@ -0,0 +1,13 @@
+FROM alpine:3.22.1
+
+RUN apk update && apk add dnsmasq
+
+COPY entrypoint.sh /
+
+RUN chmod +x /entrypoint.sh
+
+WORKDIR /
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+

test/dnsmasq/dnsmasq.py

@@ -0,0 +1 @@
+print("dnsmasq")

test/dnsmasq/entrypoint.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+set -euox pipefail
+echo "$@"
+
+OVR=/etc/dnsmasq.conf
+: > "$OVR"
+
+touch /dns_overrides.conf
+chmod 644 /dns_overrides.conf
+
+# Always keep Docker's embedded DNS for container-name resolution.
+echo "server=127.0.0.11" >> "$OVR"
+
+# Collect --entry flags: supports --entry=LINE and --entry LINE
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --entry=*)
+      printf '%s\n' "${1#--entry=}" >> "$OVR"
+      ;;
+    --entry)
+      shift
+      [ $# -gt 0 ] || { echo "missing value after --entry" >&2; exit 2; }
+      echo '%s' "$1" >> "$OVR"
+      ;;
+    --help|-h)
+      cat <<EOF
+Usage: entrypoint.sh [--entry <dnsmasq-directive>]...
+Examples:
+  --entry "address=/target.test.lan/10.0.0.123"
+  --entry "cname=/alias.test.lan/target.test.lan"
+  --entry "txt-record=_acme-challenge.app.test.lan,challenge-token"
+Notes:
+  - Lines are written verbatim to $OVR
+  - Docker DNS passthrough is preserved: server=127.0.0.11
+EOF
+      exit 0
+      ;;
+    *)
+      echo "Unknown arg: $1" >&2
+      exit 2
+      ;;
+  esac
+  shift
+done
+
+# Run dnsmasq using our generated config
+/usr/sbin/dnsmasq \
+  --no-resolv \
+  --keep-in-foreground \
+  --log-queries \
+  --log-facility=- \
+  --conf-file="$OVR"