|
| 1 | +#!/bin/bash |
| 2 | +#: |
| 3 | +#: name = "test-interop" |
| 4 | +#: variety = "basic" |
| 5 | +#: target = "lab-2.0-gimlet" |
| 6 | +#: skip_clone = true |
| 7 | +#: output_rules = [ |
| 8 | +#: "/work/*", |
| 9 | +#: ] |
| 10 | +#: |
| 11 | +#: [dependencies.build-interop] |
| 12 | +#: job = "build-interop" |
| 13 | +#: |
| 14 | +#: [dependencies.image] |
| 15 | +#: job = "image" |
| 16 | +#: |
| 17 | + |
| 18 | +set -x |
| 19 | +set -e |
| 20 | + |
| 21 | +ssh() { |
| 22 | + if [ -z "$SSH_BIN" ]; then |
| 23 | + SSH_BIN=$(which ssh) |
| 24 | + fi |
| 25 | + |
| 26 | + $SSH_BIN -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "$@" |
| 27 | +} |
| 28 | + |
| 29 | +net_info() { |
| 30 | + # |
| 31 | + # grab command output to see what's going on from the buildomat logs |
| 32 | + # |
| 33 | + if [ -z "$ARISTA_IP" ]; then |
| 34 | + ARISTA_IF=$(pfexec ./interop exec arista "ip -4 -j route show default | jq '.[0][\"dev\"]' | tr -d '\"'") |
| 35 | + ARISTA_IP=$(pfexec ./interop exec arista "ip -4 -br -j addr show dev $ARISTA_IF | jq '.[0][\"addr_info\"][0][\"local\"]' | tr -d '\"'") |
| 36 | + fi |
| 37 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip interface brief | no-more'" |
| 38 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip bgp summary | no-more'" |
| 39 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip bgp | no-more'" |
| 40 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 Cli -c 'show ip route | no-more'" |
| 41 | + |
| 42 | + if [ -z "$JUNIPER_IP" ]; then |
| 43 | + JUNIPER_IF=$(pfexec ./interop exec juniper "ip -j route show default | jq '.[0][\"dev\"]' | tr -d '\"'") |
| 44 | + JUNIPER_IP=$(pfexec ./interop exec juniper "ip -4 -br -j addr show dev $JUNIPER_IF | jq '.[0][\"addr_info\"][0][\"local\"]' | tr -d '\"'") |
| 45 | + fi |
| 46 | + ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cli -c 'show interfaces terse | no-more'" |
| 47 | + ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cli -c 'show bgp summary | no-more'" |
| 48 | + ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cli -c 'show route | no-more'" |
| 49 | + |
| 50 | + if [ -z "$MGD_IP" ]; then |
| 51 | + MGD_IF=$(pfexec ./interop exec mgd "route get -inet default | grep interface | awk '{print \$NF}'") |
| 52 | + MGD_IP=$(pfexec ./interop exec mgd "ipadm show-addr $MGD_IF/v4 -p -o addr | cut -d / -f 1") |
| 53 | + fi |
| 54 | + ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status neighbors 65100" |
| 55 | + ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status imported 65100" |
| 56 | + ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status selected 65100" |
| 57 | + ssh root@"$MGD_IP" "/opt/cargo-bay/mgadm bgp status exported 65100" |
| 58 | +} |
| 59 | + |
| 60 | +_exit_trap() { |
| 61 | + local status=$? |
| 62 | + [[ $status -eq 0 ]] && exit 0 |
| 63 | + |
| 64 | + set +o errexit |
| 65 | + |
| 66 | + banner 'debug' |
| 67 | + |
| 68 | + # |
| 69 | + # collect general info about runner |
| 70 | + # |
| 71 | + pfexec df -h |
| 72 | + pfexec diskinfo |
| 73 | + pfexec zfs list |
| 74 | + pfexec zpool list |
| 75 | + pfexec ls -l /ci |
| 76 | + |
| 77 | + # |
| 78 | + # collect falcon info |
| 79 | + # |
| 80 | + find /ci/testbed/interop/.falcon -ls |
| 81 | + cp /ci/testbed/interop/.falcon/{arista,juniper,mgd}* /work/ |
| 82 | + |
| 83 | + # |
| 84 | + # check if propolis is running |
| 85 | + # |
| 86 | + pgrep -lf propolis-server |
| 87 | + |
| 88 | + # |
| 89 | + # get network info from nodes |
| 90 | + # |
| 91 | + net_info |
| 92 | + |
| 93 | + # |
| 94 | + # grab platform-specific logs |
| 95 | + # variables set in prior call to net_info |
| 96 | + # |
| 97 | + ssh root@"$ARISTA_IP" "cp /tmp/init.log /tmp/arista.init.log" |
| 98 | + ssh root@"$ARISTA_IP" "docker ps -a > /tmp/arista.docker-ps.log" |
| 99 | + ssh root@"$ARISTA_IP" "docker logs ceos1 > /tmp/arista.docker.logs" |
| 100 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/account.log > /tmp/arista.account.log" |
| 101 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/messages > /tmp/arista.messages" |
| 102 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/nginx-error.log > /tmp/arista.nginx-error.log" |
| 103 | + ssh root@"$ARISTA_IP" "docker exec -t ceos1 cat /var/log/nginx-access.log > /tmp/arista.nginx-access.log" |
| 104 | + scp root@"$ARISTA_IP":/tmp/*.log /work |
| 105 | + |
| 106 | + ssh root@"$JUNIPER_IP" "cp /tmp/init.log /tmp/juniper.init.log" |
| 107 | + ssh root@"$JUNIPER_IP" "docker ps -a > /tmp/juniper.docker-ps.log" |
| 108 | + ssh root@"$JUNIPER_IP" "docker logs crpd1 > /tmp/juniper.docker-logs.log" |
| 109 | + ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cat /var/log/messages > /tmp/juniper-messages.log" |
| 110 | + ssh root@"$JUNIPER_IP" "docker exec -t crpd1 cat /var/log/na-grpcd > /tmp/juniper-na-grpcd.log" |
| 111 | + scp root@"$JUNIPER_IP":/tmp/*.log /work |
| 112 | + |
| 113 | + # /tmp filepaths chosen in testbed/interop/src/interop.rs and testbed/interop/cargo-bay/mgd/init.sh |
| 114 | + ssh root@"$MGD_IP" "cp /tmp/init.log /tmp/mgd.init.log" |
| 115 | + scp root@"$MGD_IP":/tmp/{mgd.init,mgd}.log /work |
| 116 | + |
| 117 | + find /work -ls |
| 118 | + |
| 119 | + exit 1 |
| 120 | +} |
| 121 | + |
| 122 | +trap _exit_trap EXIT |
| 123 | + |
| 124 | +banner 'inputs' |
| 125 | + |
| 126 | +find /input -ls |
| 127 | + |
| 128 | +banner 'zpool' |
| 129 | + |
| 130 | +# pick the largest disk available |
| 131 | +DISK=$(pfexec diskinfo -pH | sort -k8 -n -r | head -1 | awk '{print $2}') |
| 132 | +export DISK |
| 133 | +pfexec zpool create -o ashift=12 -f cpool "$DISK" |
| 134 | +pfexec zfs create -o mountpoint=/ci cpool/ci |
| 135 | + |
| 136 | +if [[ $(curl -s http://catacomb.eng.oxide.computer:12346/trim-me) =~ "true" ]]; then |
| 137 | + pfexec zpool trim cpool |
| 138 | + while [[ ! $(zpool status -t cpool) =~ "100%" ]]; do sleep 10; done |
| 139 | +fi |
| 140 | + |
| 141 | +pfexec chown "$UID" /ci |
| 142 | +cd /ci |
| 143 | +export FALCON_DATASET="cpool/falcon" |
| 144 | + |
| 145 | +banner 'setup' |
| 146 | + |
| 147 | +tar xvfz /input/build-interop/work/testbed.tar.gz |
| 148 | + |
| 149 | +mkdir -p image/mgd |
| 150 | +(cd image/mgd && tar xvfz /input/image/out/mgd.tar.gz) |
| 151 | +for bin in mgadm mgd; do |
| 152 | + mv "image/mgd/root/opt/oxide/mgd/bin/$bin" \ |
| 153 | + "testbed/interop/cargo-bay/mgd/$bin" |
| 154 | +done |
| 155 | +cd testbed |
| 156 | +mkdir -p target/debug |
| 157 | +mv out/{interop,wrangler} target/debug |
| 158 | +mv out/baseline interop |
| 159 | + |
| 160 | +banner 'dhcp-server' |
| 161 | + |
| 162 | +export EXT_INTERFACE=${EXT_INTERFACE:-igb0} |
| 163 | + |
| 164 | +cp /input/build-interop/work/dhcp-server . |
| 165 | +chmod +x dhcp-server |
| 166 | +first=$(bmat address ls -f extra -Ho first) |
| 167 | +last=$(bmat address ls -f extra -Ho last) |
| 168 | +gw=$(bmat address ls -f extra -Ho gateway) |
| 169 | +server=$(ipadm show-addr "$EXT_INTERFACE"/dhcp -po ADDR | sed 's#/.*##g') |
| 170 | +pfexec ./dhcp-server "$first" "$last" "$gw" "$server" &> /work/dhcp-server.log & |
| 171 | + |
| 172 | +banner 'launch' |
| 173 | + |
| 174 | +cd interop |
| 175 | +pfexec ./interop launch |
| 176 | + |
| 177 | +banner 'status' |
| 178 | + |
| 179 | +net_info |
| 180 | + |
| 181 | +banner 'test' |
| 182 | + |
| 183 | +./baseline --show-output |
0 commit comments