Skip to content

Commit 2c043a5

Browse files
NathanFlurryMasterPtato
NathanFlurry
authored and
MasterPtato
committed
chore: add standalone toolchain build scripts
1 parent ac3d4cd commit 2c043a5

File tree

7 files changed

+277
-123
lines changed

7 files changed

+277
-123
lines changed

‎.github/workflows/release.yaml‎

Lines changed: 42 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,11 @@ jobs:
7070
npmAuthToken: "${{ secrets.NPM_TOKEN }}"
7171
EOF
7272
73+
# TODO: CI yarn.lock differs from local for some reason & causes errors
7374
# Install dependencies
74-
yarn install --immutable
75+
# yarn install --immutable
76+
yarn config set -H enableImmutableInstalls false
77+
yarn install
7578
7679
if [ "${{ inputs.latest }}" = "true" ]; then
7780
./scripts/release/main.ts --version "${{ github.event.inputs.version }}" --setupCi
@@ -86,142 +89,68 @@ jobs:
8689
strategy:
8790
matrix:
8891
include:
89-
- platform: x86_64-unknown-linux-gnu
92+
- platform: linux
9093
runner: [self-hosted, Linux, X64]
91-
suffix: x86-linux
92-
# - platform: x86_64-pc-windows-gnu
94+
target: x86_64-unknown-linux-musl
95+
# - platform: windows
9396
# runner: [self-hosted, Linux, X64]
94-
# suffix: x86-windows
95-
- platform: x86_64-apple-darwin
97+
# target: x86_64-pc-windows-musl
98+
- platform: macos
9699
runner: [self-hosted, Linux, X64]
97-
suffix: x86-mac
98-
- platform: aarch64-apple-darwin
100+
target: x86_64-apple-darwin
101+
- platform: macos
99102
runner: [self-hosted, Linux, ARM64]
100-
suffix: aarch64-mac
103+
target: aarch64-apple-darwin
104+
arch: arm64
101105
runs-on: ${{ matrix.runner }}
102-
container:
103-
image: rust:1.82.0
104106
steps:
105107
- uses: actions/checkout@v4
106-
107-
# Required for running in Docker
108-
- name: Git LFS Pull
109-
run: |
110-
apt-get update
111-
apt-get install git-lfs
112-
git lfs pull
113-
114-
- uses: actions/setup-node@v4
115108
with:
116-
node-version: 20
117-
118-
- run: corepack enable
109+
lfs: 'true'
119110

120-
- name: Build
121-
env:
122-
# Expose correct target since we're building cross-platform
123-
OVERRIDE_TARGET: ${{ matrix.platform }}
124-
125-
# Expose env
126-
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: x86_64-linux-gnu-gcc
127-
CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER: x86_64-apple-darwin20.4-clang
128-
CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER: aarch64-apple-darwin20.4-clang
129-
CC_x86_64_apple_darwin: x86_64-apple-darwin20.4-clang
130-
CXX_x86_64_apple_darwin: x86_64-apple-darwin20.4-clang++
131-
CC_aarch64_apple_darwin: aarch64-apple-darwin20.4-clang
132-
CXX_aarch64_apple_darwin: aarch64-apple-darwin20.4-clang++
133-
MACOSX_DEPLOYMENT_TARGET: "10.7"
134-
# Use RUSTFLAGS to skip building aws-lc-rs when targeting macOS
135-
# CARGO_FEATURE_RUSTLS_NATIVE_CERTS: 0
136-
# CARGO_RUSTLS_NATIVE_CERTS: 0
137-
# RUSTFLAGS: "--cfg tokio_unstable ${{ contains(matrix.platform, 'apple-darwin') && '--cfg tokio_unstable --cfg rustls_native_certs --cfg aws_lc_rs' || '' }}"
138-
139-
# Expose token for pulling GH artifacts
140-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
111+
- name: Set up Docker Buildx
112+
uses: docker/setup-buildx-action@v3
141113

142-
# Pull via Git CLI to improve reliability in CI
143-
ENV CARGO_NET_GIT_FETCH_WITH_CLI: "true"
114+
- name: Build using toolchain Dockerfile
144115
run: |
145-
# Install required components for each platform
146-
apt-get update
147-
if [[ "${{ matrix.platform }}" == "x86_64-pc-windows-gnu" ]]; then
148-
echo "Installing Windows components"
149-
150-
apt-get install -y gcc-mingw-w64-x86-64
151-
elif [[ "${{ matrix.platform }}" == "x86_64-unknown-linux-gnu" ]]; then
152-
echo "Installing Linux components"
153-
154-
apt-get install -y gcc-x86-64-linux-gnu libc6-dev-amd64-cross
155-
elif [[ "${{ matrix.platform }}" == "x86_64-apple-darwin" || "${{ matrix.platform }}" == "aarch64-apple-darwin" ]]; then
156-
echo "Installing macOS components"
157-
158-
apt-get install -y clang cmake patch libxml2-dev wget xz-utils curl
159-
160-
# Install osxcross
161-
git config --global --add safe.directory '*'
162-
git clone https://github.com/tpoechtrager/osxcross /root/osxcross
163-
(
164-
cd /root/osxcross
165-
wget -nc https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz
166-
mv MacOSX11.3.sdk.tar.xz tarballs/
167-
UNATTENDED=yes OSX_VERSION_MIN=10.7 ./build.sh
168-
)
169-
170-
# Add osxcross to PATH permanently
171-
echo 'export PATH="/root/osxcross/target/bin:$PATH"' >> ~/.bashrc
172-
export PATH="/root/osxcross/target/bin:$PATH"
173-
174-
# Verify compiler installation
175-
echo 'clang path:'
176-
which x86_64-apple-darwin20.4-clang
177-
echo 'clang version:'
178-
x86_64-apple-darwin20.4-clang --version
179-
fi
180-
181-
# Install targets
182-
rustup target add ${{ matrix.platform }}
183-
184-
mkdir -p /root/.cargo && \
185-
echo '\
186-
[target.x86_64-unknown-linux-gnu]\n\
187-
linker = "x86_64-linux-gnu-gcc"\n\
188-
\n\
189-
[target.x86_64-pc-windows-gnu]\n\
190-
linker = "x86_64-w64-mingw32-gcc"\n\
191-
\n\
192-
[target.x86_64-apple-darwin]\n\
193-
linker = "x86_64-apple-darwin20.4-clang"\n\
194-
ar = "x86_64-apple-darwin20.4-ar"\n\
195-
\n\
196-
[target.aarch64-apple-darwin]\n\
197-
linker = "aarch64-apple-darwin20.4-clang"\n\
198-
ar = "aarch64-apple-darwin20.4-ar"\n\
199-
' > /root/.cargo/config.toml
200-
201-
# Build
202-
cargo build --bin rivet --release --target ${{ matrix.platform }}
116+
# Use Docker BuildKit
117+
export DOCKER_BUILDKIT=1
118+
119+
# Build the binary using our Dockerfile
120+
ARCH_ARG="${{ matrix.arch }}"
121+
docker/toolchain/build.sh ${{ matrix.target }}
122+
123+
# Make sure dist directory exists and binary is there
124+
ls -la dist/
203125
204126
- name: Upload artifacts
205127
env:
206128
AWS_ACCESS_KEY_ID: ${{ secrets.R2_RELEASES_ACCESS_KEY_ID }}
207129
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_RELEASES_SECRET_ACCESS_KEY }}
208130
run: |
131+
# Install dependencies for AWS CLI
209132
apt-get update
210-
apt-get install -y awscli
133+
apt-get install -y unzip curl
211134
212-
COMMIT_SHA_SHORT="${GITHUB_SHA::7}"
213-
BINARY_PATH="target/${{ matrix.platform }}/release/rivet"
214-
BINARY_NAME="rivet-${{ matrix.suffix }}"
215-
if [[ "${{ matrix.platform }}" == "x86_64-pc-windows-gnu" ]]; then
216-
BINARY_PATH="${BINARY_PATH}.exe"
217-
BINARY_NAME="${BINARY_NAME}.exe"
135+
# Install AWS CLI - use ARM version if running on ARM architecture
136+
if [ "${{ matrix.arch }}" = "arm64" ]; then
137+
curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"
138+
else
139+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
218140
fi
141+
unzip awscliv2.zip
142+
./aws/install --update
143+
144+
COMMIT_SHA_SHORT="${GITHUB_SHA::7}"
145+
BINARY_PATH="dist/rivet-${{ matrix.target }}"
219146
147+
# Must specify --checksum-algorithm for compatibility with R2
220148
aws s3 cp \
221149
"${BINARY_PATH}" \
222-
"s3://rivet/${COMMIT_SHA_SHORT}/${BINARY_NAME}" \
150+
"s3://rivet/${COMMIT_SHA_SHORT}/rivet-${{ matrix.target }}" \
223151
--region auto \
224-
--endpoint-url https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com/rivet-releases
152+
--endpoint-url https://2a94c6a0ced8d35ea63cddc86c2681e7.r2.cloudflarestorage.com/rivet-releases \
153+
--checksum-algorithm CRC32
225154
226155
docker:
227156
name: "Build & Push Docker Images"
@@ -253,12 +182,6 @@ jobs:
253182
apt-get install git-lfs
254183
git lfs pull
255184
256-
- uses: actions/setup-node@v4
257-
with:
258-
node-version: 20
259-
260-
- run: corepack enable
261-
262185
- name: Set outputs
263186
id: vars
264187
run: echo "sha_short=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
@@ -394,4 +317,3 @@ jobs:
394317
else
395318
./scripts/release/main.ts --version "${{ github.event.inputs.version }}" --no-latest --completeCi
396319
fi
397-

‎docker/toolchain/build.sh‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Default to x86_64-unknown-linux-gnu if no target specified
5+
TARGET=${1:-x86_64-unknown-linux-musl}
6+
7+
case $TARGET in
8+
x86_64-unknown-linux-musl)
9+
echo "Building for Linux x86_64 platform"
10+
DOCKERFILE="linux.Dockerfile"
11+
BINARY="rivet-$TARGET"
12+
;;
13+
aarch64-apple-darwin)
14+
echo "Building for macOS ARM64 platform"
15+
DOCKERFILE="macos.Dockerfile"
16+
BINARY="rivet-$TARGET"
17+
;;
18+
x86_64-apple-darwin)
19+
echo "Building for macOS x86_64 platform"
20+
DOCKERFILE="macos.Dockerfile"
21+
BINARY="rivet-$TARGET"
22+
;;
23+
x86_64-pc-windows-gnu)
24+
echo "Building for Windows platform"
25+
DOCKERFILE="windows.Dockerfile"
26+
BINARY="rivet-$TARGET.exe"
27+
;;
28+
*)
29+
echo "Unsupported target: $TARGET"
30+
exit 1
31+
;;
32+
esac
33+
34+
# Build docker image
35+
DOCKER_BUILDKIT=1 docker build -f docker/toolchain/$DOCKERFILE -t rivet-cli-builder-$TARGET .
36+
37+
# Extract binary
38+
CONTAINER_ID=$(docker create rivet-cli-builder-$TARGET)
39+
mkdir -p dist
40+
docker cp "$CONTAINER_ID:/artifacts/$BINARY" dist/
41+
docker rm "$CONTAINER_ID"
42+
43+
# Make binary executable (skip for Windows .exe files)
44+
if [[ ! "$BINARY" == *.exe ]]; then
45+
chmod +x dist/$BINARY
46+
fi
47+
48+
echo "Binary saved to: dist/$BINARY"

‎docker/toolchain/linux.Dockerfile‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# syntax=docker/dockerfile:1.4
2+
FROM rust:1.82.0
3+
4+
# Install dependencies
5+
RUN apt-get update && apt-get install -y \
6+
musl-tools \
7+
libssl-dev \
8+
pkg-config \
9+
protobuf-compiler \
10+
ca-certificates \
11+
git-lfs \
12+
musl-dev \
13+
&& rm -rf /var/lib/apt/lists/*
14+
15+
# Install musl target
16+
RUN rustup target add x86_64-unknown-linux-musl
17+
18+
# Set up OpenSSL for musl target
19+
ENV SSL_VER=1.1.1w
20+
RUN wget https://www.openssl.org/source/openssl-$SSL_VER.tar.gz \
21+
&& tar -xzf openssl-$SSL_VER.tar.gz \
22+
&& cd openssl-$SSL_VER \
23+
&& ./Configure no-shared no-async --prefix=/musl --openssldir=/musl/ssl linux-x86_64 \
24+
&& make -j$(nproc) \
25+
&& make install_sw \
26+
&& cd .. \
27+
&& rm -rf openssl-$SSL_VER*
28+
29+
# Configure OpenSSL env vars for the build
30+
ENV OPENSSL_DIR=/musl \
31+
OPENSSL_INCLUDE_DIR=/musl/include \
32+
OPENSSL_LIB_DIR=/musl/lib \
33+
PKG_CONFIG_ALLOW_CROSS=1
34+
35+
# Set environment variables
36+
ENV CARGO_INCREMENTAL=0 \
37+
RUSTFLAGS="--cfg tokio_unstable" \
38+
CARGO_NET_GIT_FETCH_WITH_CLI=true
39+
40+
# Set working directory
41+
WORKDIR /build
42+
43+
# Build CLI instructions
44+
COPY . .
45+
46+
# Build for Linux with musl (static binary)
47+
RUN --mount=type=cache,target=/usr/local/cargo/registry \
48+
--mount=type=cache,target=/usr/local/cargo/git \
49+
--mount=type=cache,target=/build/target \
50+
cargo build --bin rivet --release --target x86_64-unknown-linux-musl -v && \
51+
mkdir -p /artifacts && \
52+
cp target/x86_64-unknown-linux-musl/release/rivet /artifacts/rivet-x86_64-unknown-linux-musl
53+
54+
# Default command to show help
55+
CMD ["ls", "-la", "/artifacts"]

‎docker/toolchain/macos.Dockerfile‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# syntax=docker/dockerfile:1.4
2+
FROM rust:1.82.0
3+
4+
# Install dependencies
5+
RUN apt-get update && apt-get install -y \
6+
git-lfs \
7+
protobuf-compiler \
8+
clang \
9+
cmake \
10+
patch \
11+
libxml2-dev \
12+
wget \
13+
xz-utils \
14+
curl \
15+
&& rm -rf /var/lib/apt/lists/*
16+
17+
# Install osxcross
18+
RUN git config --global --add safe.directory '*' && \
19+
git clone https://github.com/tpoechtrager/osxcross /root/osxcross && \
20+
cd /root/osxcross && \
21+
wget -nc https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz && \
22+
mv MacOSX11.3.sdk.tar.xz tarballs/ && \
23+
UNATTENDED=yes OSX_VERSION_MIN=10.7 ./build.sh
24+
25+
# Add osxcross to PATH
26+
ENV PATH="/root/osxcross/target/bin:$PATH"
27+
28+
# Install macOS target
29+
RUN rustup target add x86_64-apple-darwin aarch64-apple-darwin
30+
31+
# Configure Cargo for cross-compilation
32+
RUN mkdir -p /root/.cargo && \
33+
echo '\
34+
[target.x86_64-apple-darwin]\n\
35+
linker = "x86_64-apple-darwin20.4-clang"\n\
36+
ar = "x86_64-apple-darwin20.4-ar"\n\
37+
\n\
38+
[target.aarch64-apple-darwin]\n\
39+
linker = "aarch64-apple-darwin20.4-clang"\n\
40+
ar = "aarch64-apple-darwin20.4-ar"\n\
41+
' > /root/.cargo/config.toml
42+
43+
# Set environment variables for cross-compilation
44+
ENV CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=x86_64-apple-darwin20.4-clang \
45+
CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=aarch64-apple-darwin20.4-clang \
46+
CC_x86_64_apple_darwin=x86_64-apple-darwin20.4-clang \
47+
CXX_x86_64_apple_darwin=x86_64-apple-darwin20.4-clang++ \
48+
CC_aarch64_apple_darwin=aarch64-apple-darwin20.4-clang \
49+
CXX_aarch64_apple_darwin=aarch64-apple-darwin20.4-clang++ \
50+
MACOSX_DEPLOYMENT_TARGET=10.7 \
51+
# Skip aws-lc-rs with rustls certs config when building for macOS
52+
RUSTFLAGS="--cfg tokio_unstable --cfg rustls_native_certs --cfg aws_lc_rs" \
53+
CARGO_FEATURE_RUSTLS_NATIVE_CERTS=0 \
54+
CARGO_RUSTLS_NATIVE_CERTS=0 \
55+
CARGO_INCREMENTAL=0 \
56+
CARGO_NET_GIT_FETCH_WITH_CLI=true
57+
58+
# Set working directory
59+
WORKDIR /build
60+
61+
# Copy the source code
62+
COPY . .
63+
64+
# Build for x86_64 macOS
65+
RUN --mount=type=cache,target=/usr/local/cargo/registry \
66+
--mount=type=cache,target=/usr/local/cargo/git \
67+
--mount=type=cache,target=/build/target \
68+
cargo build --bin rivet --release --target x86_64-apple-darwin && \
69+
mkdir -p /artifacts && \
70+
cp target/x86_64-apple-darwin/release/rivet /artifacts/rivet-x86_64-apple-darwin
71+
72+
# Build for ARM64 macOS
73+
RUN --mount=type=cache,target=/usr/local/cargo/registry \
74+
--mount=type=cache,target=/usr/local/cargo/git \
75+
--mount=type=cache,target=/build/target \
76+
cargo build --bin rivet --release --target aarch64-apple-darwin && \
77+
cp target/aarch64-apple-darwin/release/rivet /artifacts/rivet-aarch64-apple-darwin
78+
79+
# Default command to show help
80+
CMD ["ls", "-la", "/artifacts"]

0 commit comments

Comments
 (0)