diff --git a/base-kustomize/ovn/base/ovn-setup.yaml b/base-kustomize/ovn/base/ovn-setup.yaml index 94cf706e2..4a9d5b0bc 100644 --- a/base-kustomize/ovn/base/ovn-setup.yaml +++ b/base-kustomize/ovn/base/ovn-setup.yaml @@ -24,6 +24,16 @@ rules: - services - endpoints - nodes + - apiGroups: + - "" + verbs: + - get + - list + - create + - update + - patch + resources: + - configmaps --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -49,25 +59,284 @@ data: #!/bin/bash set -exo pipefail + # Load configuration from node annotations source /node/ovn + # Read previous state if it exists (written by init container) + if [[ -f /node/previous-state.json ]]; then + PREVIOUS_STATE=$(cat /node/previous-state.json) + FIRST_RUN=false + echo "Previous state found: ${PREVIOUS_STATE}" + else + PREVIOUS_STATE='{}' + FIRST_RUN=true + echo "First run detected - no previous state found" + fi + + # Validation if [[ "${OVN_BRIDGES}" == "null" ]]; then echo "No OVN_BRIDGES defined, exiting" exit 99 fi - IFS=',' read -r -a array <<< "$OVN_BRIDGES" - for i in ${array[@]}; do - ovs-vsctl --no-wait --may-exist add-br $i + + # Create bridges + IFS=',' read -r -a bridges_array <<< "$OVN_BRIDGES" + for bridge in ${bridges_array[@]}; do + ovs-vsctl --no-wait --may-exist add-br $bridge done - if [[ "${OVN_PORT_MAPPINGS}" == "null" ]]; then - echo "No OVN_PORT_MAPPINGS defined, exiting" - exit 99 + + # Parse bond options into associative array + declare -A BOND_OPTIONS_MAP + if [[ "${OVN_BOND_OPTIONS:-null}" != "null" ]]; then + IFS=',' read -r -a bond_options_array <<< "$OVN_BOND_OPTIONS" + for bond_option in ${bond_options_array[@]}; do + # Format: bondname:option1=value1,option2=value2 + IFS=':' read -r -a option_parts <<< "$bond_option" + OPTION_BOND_NAME="${option_parts[0]}" + OPTION_VALUES="${option_parts[1]}" + BOND_OPTIONS_MAP["${OPTION_BOND_NAME}"]="${OPTION_VALUES}" + done + fi + + # Handle bonds if configured + declare -A NEW_BONDS + if [[ "${OVN_BONDS:-null}" != "null" ]]; then + IFS=',' read -r -a bonds_array <<< "$OVN_BONDS" + for bond_config in ${bonds_array[@]}; do + # Format: bridge:bondname:member1+member2+member3:mode[:lacp] + IFS=':' read -r -a bond_parts <<< "$bond_config" + BOND_BRIDGE="${bond_parts[0]}" + BOND_NAME="${bond_parts[1]}" + BOND_MEMBERS="${bond_parts[2]}" + BOND_MODE="${bond_parts[3]:-balance-slb}" + BOND_LACP="${bond_parts[4]:-off}" + + NEW_BONDS["${BOND_BRIDGE}:${BOND_NAME}"]="${BOND_MEMBERS}:${BOND_MODE}:${BOND_LACP}" + done + fi + + # Clean up old bonds that are no longer configured + if [[ "${FIRST_RUN}" == "false" ]]; then + PREV_BONDS=$(echo "${PREVIOUS_STATE}" | jq -r '.bonds // {}' 2>/dev/null || echo '{}') + if [[ "${PREV_BONDS}" != "{}" ]]; then + # Use process substitution instead of pipe to avoid subshell + while IFS=: read -r old_bond old_bridge; do + if [[ -z "${NEW_BONDS[${old_bridge}:${old_bond}]}" ]]; then + echo "Removing old bond: ${old_bond} from bridge ${old_bridge}" + ovs-vsctl --if-exists del-port "${old_bridge}" "${old_bond}" + fi + done < <(echo "${PREV_BONDS}" | jq -r 'to_entries[] | "\(.key):\(.value.bridge)"') + fi + fi + + # Create/update bonds + for bond_key in "${!NEW_BONDS[@]}"; do + IFS=':' read -r -a key_parts <<< "$bond_key" + BOND_BRIDGE="${key_parts[0]}" + BOND_NAME="${key_parts[1]}" + IFS=':' read -r -a value_parts <<< "${NEW_BONDS[$bond_key]}" + BOND_MEMBERS="${value_parts[0]}" + BOND_MODE="${value_parts[1]}" + BOND_LACP="${value_parts[2]}" + + # Replace + with space for members list + MEMBERS_LIST="${BOND_MEMBERS//+/ }" + + # Check if bond already exists and if it needs to be updated + NEEDS_RECREATE=false + if ovs-vsctl port-to-br "${BOND_NAME}" &>/dev/null; then + echo "Bond ${BOND_NAME} exists, checking if update needed" + + # Get current bond configuration + CURRENT_BRIDGE=$(ovs-vsctl port-to-br "${BOND_NAME}") + CURRENT_MODE=$(ovs-vsctl get port "${BOND_NAME}" bond_mode | tr -d '"') + CURRENT_LACP=$(ovs-vsctl get port "${BOND_NAME}" lacp | tr -d '"') + + # Get current members (interfaces in the bond) + CURRENT_MEMBERS=$(ovs-vsctl list-ifaces "${BOND_NAME}" | sort | tr '\n' ' ' | sed 's/ $//') + DESIRED_MEMBERS=$(echo ${MEMBERS_LIST} | tr ' ' '\n' | sort | tr '\n' ' ' | sed 's/ $//') + + # Check if bridge changed + if [[ "${CURRENT_BRIDGE}" != "${BOND_BRIDGE}" ]]; then + echo "Bond ${BOND_NAME} bridge changed from ${CURRENT_BRIDGE} to ${BOND_BRIDGE}" + NEEDS_RECREATE=true + fi + + # Check if members changed + if [[ "${CURRENT_MEMBERS}" != "${DESIRED_MEMBERS}" ]]; then + echo "Bond ${BOND_NAME} members changed from [${CURRENT_MEMBERS}] to [${DESIRED_MEMBERS}]" + NEEDS_RECREATE=true + fi + + # Check if bond mode changed + if [[ "${CURRENT_MODE}" != "${BOND_MODE}" ]]; then + echo "Bond ${BOND_NAME} mode changed from ${CURRENT_MODE} to ${BOND_MODE}" + NEEDS_RECREATE=true + fi + + # Check if LACP setting changed + if [[ "${CURRENT_LACP}" != "${BOND_LACP}" ]]; then + echo "Bond ${BOND_NAME} LACP changed from ${CURRENT_LACP} to ${BOND_LACP}" + NEEDS_RECREATE=true + fi + + if [[ "${NEEDS_RECREATE}" == "true" ]]; then + echo "Recreating bond ${BOND_NAME} due to configuration changes" + ovs-vsctl del-port "${CURRENT_BRIDGE}" "${BOND_NAME}" + else + echo "Bond ${BOND_NAME} configuration unchanged, skipping recreate" + fi + else + # Bond doesn't exist, needs to be created + NEEDS_RECREATE=true + fi + + # Only recreate/create the bond if needed + if [[ "${NEEDS_RECREATE}" == "true" ]]; then + # Remove any member interfaces that are already attached as regular ports + # This prevents "port named X already exists on bridge" errors + for member in ${MEMBERS_LIST}; do + if ovs-vsctl port-to-br "${member}" &>/dev/null; then + CURRENT_BRIDGE=$(ovs-vsctl port-to-br "${member}") + echo "Removing ${member} from bridge ${CURRENT_BRIDGE} (needed for bond ${BOND_NAME})" + ovs-vsctl --if-exists del-port "${CURRENT_BRIDGE}" "${member}" + fi + done + + echo "Creating bond ${BOND_NAME} on bridge ${BOND_BRIDGE} with members: ${MEMBERS_LIST}" + ovs-vsctl --no-wait add-bond "${BOND_BRIDGE}" "${BOND_NAME}" ${MEMBERS_LIST} \ + bond_mode="${BOND_MODE}" lacp="${BOND_LACP}" + fi + + # Apply additional bond options if configured + if [[ -n "${BOND_OPTIONS_MAP[${BOND_NAME}]}" ]]; then + echo "Applying additional options to bond ${BOND_NAME}: ${BOND_OPTIONS_MAP[${BOND_NAME}]}" + IFS=',' read -r -a options_array <<< "${BOND_OPTIONS_MAP[${BOND_NAME}]}" + for option in ${options_array[@]}; do + IFS='=' read -r -a option_kv <<< "$option" + OPTION_KEY="${option_kv[0]}" + OPTION_VALUE="${option_kv[1]}" + + # Determine if this is a bond_ parameter or other_config parameter + case "${OPTION_KEY}" in + bond-updelay|bond_updelay|updelay) + ovs-vsctl set port "${BOND_NAME}" bond_updelay="${OPTION_VALUE}" + ;; + bond-downdelay|bond_downdelay|downdelay) + ovs-vsctl set port "${BOND_NAME}" bond_downdelay="${OPTION_VALUE}" + ;; + lacp-time|lacp_time) + ovs-vsctl set port "${BOND_NAME}" other_config:lacp-time="${OPTION_VALUE}" + ;; + lacp-fallback-ab|lacp_fallback_ab) + ovs-vsctl set port "${BOND_NAME}" other_config:lacp-fallback-ab="${OPTION_VALUE}" + ;; + bond-detect-mode|bond_detect_mode) + ovs-vsctl set port "${BOND_NAME}" other_config:bond-detect-mode="${OPTION_VALUE}" + ;; + bond-miimon-interval|bond_miimon_interval|mii-monitor-interval|miimon-interval) + ovs-vsctl set port "${BOND_NAME}" other_config:bond-miimon-interval="${OPTION_VALUE}" + ;; + bond-rebalance-interval|bond_rebalance_interval|rebalance-interval) + ovs-vsctl set port "${BOND_NAME}" other_config:bond-rebalance-interval="${OPTION_VALUE}" + ;; + transmit-hash-policy|bond-hash-basis|hash-basis) + # Note: OVS uses hash-basis, not transmit-hash-policy like Linux bonding + # The hash algorithm is determined by bond_mode + ovs-vsctl set port "${BOND_NAME}" other_config:bond-hash-basis="${OPTION_VALUE}" + ;; + *) + # Generic other_config parameter + ovs-vsctl set port "${BOND_NAME}" other_config:${OPTION_KEY}="${OPTION_VALUE}" + ;; + esac + done + fi + done + + # Handle regular port mappings + declare -A NEW_PORTS + if [[ "${OVN_PORT_MAPPINGS:-null}" != "null" ]]; then + IFS=',' read -r -a ports_array <<< "$OVN_PORT_MAPPINGS" + for port_mapping in ${ports_array[@]}; do + IFS=':' read -r -a port_parts <<< "$port_mapping" + PORT_BRIDGE="${port_parts[0]}" + PORT_NAME="${port_parts[1]}" + NEW_PORTS["${PORT_BRIDGE}:${PORT_NAME}"]=1 + done + + # Clean up old ports that are no longer configured + if [[ "${FIRST_RUN}" == "false" ]]; then + PREV_PORTS=$(echo "${PREVIOUS_STATE}" | jq -r '.ports // {}' 2>/dev/null || echo '{}') + if [[ "${PREV_PORTS}" != "{}" ]]; then + # Use process substitution instead of pipe to avoid subshell + while IFS=: read -r old_bridge old_port; do + if [[ -z "${NEW_PORTS[${old_bridge}:${old_port}]}" ]]; then + echo "Removing old port: ${old_port} from bridge ${old_bridge}" + # Verify it's not a bond or internal port before removing + PORT_TYPE=$(ovs-vsctl get Interface "${old_port}" type 2>/dev/null || echo "") + if [[ "${PORT_TYPE}" == '""' || "${PORT_TYPE}" == "" ]]; then + ovs-vsctl --if-exists del-port "${old_bridge}" "${old_port}" + else + echo "Skipping removal of ${old_port} - not a physical port (type: ${PORT_TYPE})" + fi + fi + done < <(echo "${PREV_PORTS}" | jq -r 'to_entries[] | .key as $bridge | .value[] | "\($bridge):\(.)"') + fi + fi fi - IFS=',' read -r -a outerarray <<< "$OVN_PORT_MAPPINGS" - for i in ${outerarray[@]}; do - IFS=':' read -r -a innerarray <<< "$i" - ovs-vsctl --no-wait --may-exist add-port ${innerarray[0]} ${innerarray[1]} + + # Clean up orphaned ports on managed bridges (ports not in new config and not system ports) + # This runs regardless of whether OVN_PORT_MAPPINGS is set + IFS=',' read -r -a bridges_array <<< "$OVN_BRIDGES" + for bridge in ${bridges_array[@]}; do + # Get all ports on this bridge + EXISTING_PORTS=$(ovs-vsctl list-ports "${bridge}" 2>/dev/null || echo "") + if [[ -n "${EXISTING_PORTS}" ]]; then + while read -r existing_port; do + [[ -z "${existing_port}" ]] && continue + + # Check if this port is in our new configuration + if [[ -z "${NEW_PORTS[${bridge}:${existing_port}]}" ]] && [[ -z "${NEW_BONDS[${bridge}:${existing_port}]}" ]]; then + # Check if it's a system port (patch, internal, tunnel, etc.) + PORT_TYPE=$(ovs-vsctl get Interface "${existing_port}" type 2>/dev/null || echo "") + if [[ "${PORT_TYPE}" == '""' || "${PORT_TYPE}" == "" ]]; then + # It's a physical port not in our config - remove it + echo "Found orphaned port ${existing_port} on bridge ${bridge}, removing" + ovs-vsctl --if-exists del-port "${bridge}" "${existing_port}" + else + echo "Skipping system port ${existing_port} (type: ${PORT_TYPE})" + fi + fi + done <<< "${EXISTING_PORTS}" + fi done + + # Add new ports + if [[ "${OVN_PORT_MAPPINGS:-null}" != "null" ]]; then + for port_key in "${!NEW_PORTS[@]}"; do + IFS=':' read -r -a port_parts <<< "$port_key" + PORT_BRIDGE="${port_parts[0]}" + PORT_NAME="${port_parts[1]}" + + # Skip if this port is actually a bond that was already created + if [[ -n "${NEW_BONDS[${PORT_BRIDGE}:${PORT_NAME}]}" ]]; then + echo "Skipping ${PORT_NAME} - already created as a bond" + continue + fi + + # Check if port is already on a different bridge + CURRENT_BRIDGE=$(ovs-vsctl port-to-br "${PORT_NAME}" 2>/dev/null || echo "") + if [[ -n "$CURRENT_BRIDGE" ]] && [[ "$CURRENT_BRIDGE" != "$PORT_BRIDGE" ]]; then + echo "Port ${PORT_NAME} is on bridge ${CURRENT_BRIDGE}, moving to ${PORT_BRIDGE}" + ovs-vsctl --if-exists del-port "${CURRENT_BRIDGE}" "${PORT_NAME}" + fi + + ovs-vsctl --no-wait --may-exist add-port "${PORT_BRIDGE}" "${PORT_NAME}" + done + fi + + # Configure OVN settings if [[ "${OVN_BRIDGE}" != "null" ]]; then ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge="${OVN_BRIDGE}" fi @@ -91,6 +360,56 @@ data: if [[ -z $(ovs-vsctl get-manager) ]]; then ovs-vsctl --id @manager create Manager 'target="ptcp:6640:127.0.0.1"' -- add Open_vSwitch . manager_options @manager fi + + # Build current state JSON + BONDS_JSON="{}" + if [[ ${#NEW_BONDS[@]} -gt 0 ]]; then + BONDS_JSON="{" + for bond_key in "${!NEW_BONDS[@]}"; do + IFS=':' read -r -a key_parts <<< "$bond_key" + BOND_BRIDGE="${key_parts[0]}" + BOND_NAME="${key_parts[1]}" + IFS=':' read -r -a value_parts <<< "${NEW_BONDS[$bond_key]}" + BOND_MEMBERS="${value_parts[0]//+/\",\"}" + BOND_MODE="${value_parts[1]}" + BOND_LACP="${value_parts[2]}" + BONDS_JSON="${BONDS_JSON}\"${BOND_NAME}\":{\"bridge\":\"${BOND_BRIDGE}\",\"members\":[\"${BOND_MEMBERS}\"],\"mode\":\"${BOND_MODE}\",\"lacp\":\"${BOND_LACP}\"}," + done + BONDS_JSON="${BONDS_JSON%,}}" + fi + + PORTS_JSON="{}" + if [[ ${#NEW_PORTS[@]} -gt 0 ]]; then + declare -A PORTS_BY_BRIDGE + for port_key in "${!NEW_PORTS[@]}"; do + IFS=':' read -r -a port_parts <<< "$port_key" + PORT_BRIDGE="${port_parts[0]}" + PORT_NAME="${port_parts[1]}" + if [[ -z "${PORTS_BY_BRIDGE[$PORT_BRIDGE]}" ]]; then + PORTS_BY_BRIDGE[$PORT_BRIDGE]="\"${PORT_NAME}\"" + else + PORTS_BY_BRIDGE[$PORT_BRIDGE]="${PORTS_BY_BRIDGE[$PORT_BRIDGE]},\"${PORT_NAME}\"" + fi + done + PORTS_JSON="{" + for bridge in "${!PORTS_BY_BRIDGE[@]}"; do + PORTS_JSON="${PORTS_JSON}\"${bridge}\":[${PORTS_BY_BRIDGE[$bridge]}]," + done + PORTS_JSON="${PORTS_JSON%,}}" + fi + + # Build current state JSON and write to file + cat > /node/current-state.json </dev/null 2>&1; then + kubectl get configmap "${CONFIGMAP_NAME}" -n "${NAMESPACE}" -o jsonpath='{.data.state}' > /node/previous-state.json + echo "Retrieved previous state from ConfigMap ${CONFIGMAP_NAME}" + else + echo "No previous state ConfigMap found - this is a first run" + fi - name: init image: "ghcr.io/rackerlabs/genestack-images/kubernetes-entrypoint:latest" imagePullPolicy: IfNotPresent @@ -206,6 +554,10 @@ spec: echo "export OVN_BRIDGES=${OVN_BRIDGES}" >> /node/ovn OVN_BRIDGE=$(jq '."ovn.openstack.org/int_bridge"' -r /node/annotations.json) echo "export OVN_BRIDGE=${OVN_BRIDGE}" >> /node/ovn + OVN_BONDS=$(jq '."ovn.openstack.org/bonds"' -r /node/annotations.json) + echo "export OVN_BONDS=${OVN_BONDS}" >> /node/ovn + OVN_BOND_OPTIONS=$(jq '."ovn.openstack.org/bond-options"' -r /node/annotations.json) + echo "export OVN_BOND_OPTIONS=${OVN_BOND_OPTIONS}" >> /node/ovn command: - /bin/ash - -c @@ -243,15 +595,38 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: NAMESPACE + value: "openstack" command: - sh - -c - | set -xe + CONFIGMAP_NAME="ovn-state-${NODENAME}" + if [[ -f /node/ovn-setup-completed ]]; then - kubectl label node "$NODENAME" ovn.openstack.org/configured=$(date +%s) + # Save current state to ConfigMap + if [[ -f /node/current-state.json ]]; then + STATE_DATA=$(cat /node/current-state.json) + cat <` +- The setup script compares previous state with new annotations +- Changes are applied incrementally to prevent network disruption + +**Safe Port Migration:** +When changing port assignments (e.g., moving from `eno1` to `eno3`), the setup will: +1. Remove the old port (`eno1`) from the bridge +2. Add the new port (`eno3`) to the bridge +3. Update the state ConfigMap + +This prevents network loops that could occur if both ports were simultaneously connected to the bridge. + +**To reconfigure a node:** + +1. Update the node annotations with new configuration +2. Remove the configured label: + ``` shell + kubectl label node ovn.openstack.org/configured- + ``` +3. The DaemonSet will detect the unlabeled node and reapply configuration +4. Old ports/bonds will be cleaned up, new configuration applied + +**Example: Changing from single interface to bonded interfaces** + +``` shell +# Update annotations +kubectl annotate nodes node1 --overwrite \ + ovn.openstack.org/ports='br-ex:bond0' \ + ovn.openstack.org/bonds='br-ex:bond0:eno1+eno3:balance-tcp:active' \ + ovn.openstack.org/bond-options='bond0:mii-monitor-interval=100,lacp-time=fast' + +# Trigger reconfiguration +kubectl label node node1 ovn.openstack.org/configured- +``` + +The setup will remove any existing single interface ports and create the bond configuration. + !!! tip "Setup your OVN backup" To upload backups to Swift with tempauth, edit