Skip to content

Commit ee80b4f

Browse files
olexandr-havryliakolexandr-havryliak
authored
PSMDB-1690 (#436)
1 parent e759c5e commit ee80b4f

File tree

2 files changed

+136
-66
lines changed

2 files changed

+136
-66
lines changed

‎.github/workflows/LINT.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ on:
66
pull_request:
77
branches:
88
- main
9+
paths:
10+
- '**.py'
911

1012
jobs:
1113
lint:

‎regression-tests/build_image/Dockerfile_gcc_from_scratch‎

Lines changed: 134 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,35 @@
11
ARG OS=debian:12
22
FROM $OS AS toolchain_builder
33
ARG toolchain_version=v4
4+
ARG bazel_version=7.6.1
5+
ARG TARGETARCH
46
RUN set -ex; \
57
if [ -f "/usr/bin/yum" ] ; \
68
then \
7-
yum -y update && yum install -y --allowerasing diffutils wget curl file glibc-devel gmp-devel mpfr-devel libmpc-devel gcc gcc-c++ bzip2 bzip2-devel openssl-devel zlib-devel libffi-devel xz-devel make tzdata ; \
9+
yum -y update && yum install -y --allowerasing diffutils wget curl file glibc-devel gmp-devel mpfr-devel libmpc-devel gcc gcc-c++ bzip2 bzip2-devel openssl-devel zlib-devel libffi-devel xz-devel make tzdata; \
810
else \
911
apt-get update && apt-get -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -y install wget curl file libgmp-dev libmpfr-dev libmpc-dev gcc g++ bzip2 libbz2-dev libssl-dev zlib1g-dev libreadline-dev libffi-dev make tzdata ; \
1012
fi
13+
ARG build_platform=bazel
1114
RUN set -ex; \
12-
if [ "$toolchain_version" = "v3" ]; then \
13-
wget https://gcc.gnu.org/pub/gcc/releases/gcc-8.5.0/gcc-8.5.0.tar.gz -O gcc.tar.gz; \
14-
fi; \
15-
if [ "$toolchain_version" = "v4" ]; then \
16-
wget https://gcc.gnu.org/pub/gcc/releases/gcc-11.3.0/gcc-11.3.0.tar.gz -O gcc.tar.gz; \
17-
fi; \
18-
if [ "$toolchain_version" = "v5" ]; then \
19-
wget https://gcc.gnu.org/pub/gcc/releases/gcc-14.2.0/gcc-14.2.0.tar.gz -O gcc.tar.gz; \
20-
fi; \
21-
mkdir -p /opt/src/gcc && tar -xf gcc.tar.gz -C /opt/src/gcc --strip-components=1; \
22-
cd /opt/src/gcc && ./contrib/download_prerequisites; \
23-
mkdir /opt/objdir && cd /opt/objdir; \
24-
CFLAGS="-O2" CXXFLAGS="-O2" /opt/src/gcc/configure --disable-multilib --prefix=/opt/gcc --enable-languages=c,c++; \
25-
make -j "$(nproc)"; \
26-
make install-strip; \
27-
cd /opt && rm -rf /opt/src/gcc && rm -rf /opt/gcc.tar.gz && rm -rf /opt/objdir
15+
if [ "$build_platform" = "scons" ]; then \
16+
if [ "$toolchain_version" = "v3" ]; then \
17+
wget https://gcc.gnu.org/pub/gcc/releases/gcc-8.5.0/gcc-8.5.0.tar.gz -O gcc.tar.gz; \
18+
fi; \
19+
if [ "$toolchain_version" = "v4" ]; then \
20+
wget https://gcc.gnu.org/pub/gcc/releases/gcc-11.3.0/gcc-11.3.0.tar.gz -O gcc.tar.gz; \
21+
fi; \
22+
if [ "$toolchain_version" = "v5" ]; then \
23+
wget https://gcc.gnu.org/pub/gcc/releases/gcc-14.2.0/gcc-14.2.0.tar.gz -O gcc.tar.gz; \
24+
fi; \
25+
mkdir -p /opt/src/gcc && tar -xf gcc.tar.gz -C /opt/src/gcc --strip-components=1; \
26+
cd /opt/src/gcc && ./contrib/download_prerequisites; \
27+
mkdir /opt/objdir && cd /opt/objdir; \
28+
CFLAGS="-O2" CXXFLAGS="-O2" /opt/src/gcc/configure --disable-multilib --prefix=/opt/gcc --enable-languages=c,c++; \
29+
make -j "$(nproc)"; \
30+
make install-strip; \
31+
cd /opt && rm -rf /opt/src/gcc && rm -rf /opt/gcc.tar.gz && rm -rf /opt/objdir; \
32+
fi
2833
ENV PATH="/opt/gcc/bin:${PATH}"
2934
RUN set -ex; \
3035
if [ "$toolchain_version" = "v3" ]; then \
@@ -37,27 +42,67 @@ RUN set -ex; \
3742
./configure --prefix=/opt/python --enable-optimizations; \
3843
make -j "$(nproc)"; \
3944
make install; \
40-
cd /opt/ && rm -rf /opt/src/python && rm -rf /opt/python.tgz
45+
cd /opt/ && rm -rf /opt/src/python && rm -rf /opt/python.tgz
46+
# if [ "$TARGETARCH" = "amd64" ]; then \
47+
# TARGETARCH="x86_64"; \
48+
# fi; \
49+
# if [ "$toolchain_version" = "v5" ]; then \
50+
# mkdir -p /opt/bazel && \
51+
# wget https://github.com/bazelbuild/bazel/releases/download/${bazel_version}/bazel-${bazel_version}-linux-${TARGETARCH} -O /opt/bazel/bazel && \
52+
# chmod +x /opt/bazel/bazel ;\
53+
# fi
54+
RUN set -ex; \
55+
if [ -f "/usr/bin/yum" ] ; then \
56+
yum -y update && yum install -y --allowerasing diffutils wget git curl cmake lld binutils glibc-devel tzdata openssl-devel libcurl-devel; \
57+
else \
58+
apt-get update && apt-get -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -y install wget curl git cmake binutils lld tzdata libssl-dev libcurl4-openssl-dev; \
59+
fi
60+
ENV CMAKE_C_FLAGS=" -Wno-error=uninitialized "
61+
ENV CMAKE_CXX_FLAGS=" -Wno-error=deprecated-declarations -Wno-error=uninitialized "
62+
ARG AWS_VERSION=1.9.379
63+
RUN cd /opt && git clone --branch $AWS_VERSION --single-branch https://github.com/aws/aws-sdk-cpp.git && \
64+
mkdir -p /opt/aws && \
65+
cd aws-sdk-cpp && git submodule update --init --recursive && \
66+
mkdir build && cd build && \
67+
cmake .. -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=OFF -DMINIMIZE_SIZE=ON -DCMAKE_INSTALL_PREFIX="/opt/aws" -DAUTORUN_UNIT_TESTS=OFF && \
68+
make -j "$(nproc)" && make install && \
69+
cd /opt && rm -rf aws-sdk-cpp
70+
ARG E2FS_VERSION=v1.47.2
71+
RUN cd /opt && git clone --branch $E2FS_VERSION --single-branch https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git && \
72+
mkdir -p /opt/e2fs && cd e2fsprogs && \
73+
mkdir build && cd build && \
74+
../configure --prefix=/opt/e2fs && \
75+
make -j "$(nproc)" && make install-libs && \
76+
cd /opt && rm -rf e2fsprogs
4177

4278
ARG OS=debian:12
43-
FROM $OS AS base_image
44-
ARG branch=v8.0
45-
ARG psm_ver=8.0.3
46-
ARG psm_release=1
47-
ARG repo=https://github.com/percona/percona-server-mongodb.git
79+
FROM $OS AS toolchain
80+
COPY --from=toolchain_builder /opt /opt
81+
ENV PATH="/opt/python/bin:/opt/gcc/bin:/opt/bazel:${PATH}"
4882
RUN set -ex; \
83+
cp -r /opt/aws/* /usr/ && cp -r /opt/e2fs/* /usr/ && \
84+
rm -rf /opt/aws && rm -rf /opt/e2fs && \
4985
if [ -f "/usr/bin/yum" ] ; then \
5086
yum -y update && yum install -y --allowerasing diffutils wget git curl cmake lld binutils glibc-devel tzdata; \
5187
else \
5288
apt-get update && apt-get -y upgrade && DEBIAN_FRONTEND=noninteractive apt-get -y install wget curl git cmake binutils lld tzdata; \
53-
fi
54-
COPY --from=toolchain_builder /opt /opt
55-
ENV PATH="/opt/python/bin:/opt/gcc/bin:${PATH}"
56-
RUN pip3 install --upgrade pip && pip3 install virtualenv && python3 -m venv /opt/venv
57-
ENV VIRTUAL_ENV /opt/venv
89+
fi; \
90+
pip3 install --upgrade pip && pip3 install virtualenv && python3 -m venv /opt/venv
91+
ENV VIRTUAL_ENV=/opt/venv
5892
ENV PATH="/opt/venv/bin:${PATH}"
93+
ENV POETRY_VIRTUALENVS_CREATE=false
94+
ENV POETRY_INSTALLER_MAX_WORKERS=10
95+
ENV PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
96+
ENV GLIBC_TUNABLES=glibc.pthread.rseq=0
97+
ENV PYTHON_BIN_PATH=/opt/venv/bin/python
98+
99+
FROM toolchain AS base_image
100+
ARG branch=v8.0
101+
ARG psm_ver=8.0.13
102+
ARG psm_release=4
103+
ARG repo=https://github.com/percona/percona-server-mongodb.git
59104
RUN cd /opt && mkdir -p percona-server-mongodb && \
60-
git clone $repo percona-server-mongodb && \
105+
git clone --branch $branch --single-branch $repo percona-server-mongodb && \
61106
cd percona-server-mongodb/ && \
62107
git checkout $branch && git pull && \
63108
REVISION_LONG=$(git rev-parse HEAD) && \
@@ -66,17 +111,13 @@ RUN cd /opt && mkdir -p percona-server-mongodb && \
66111
\"githash\": \"${REVISION_LONG}\"\n\
67112
}" > version.json && \
68113
echo "mongo_version: \"$psm_ver-$psm_release\"" > .resmoke_mongo_version.yml && \
69-
rm -rf /opt/percona-server-mongodb/.git && mkdir -p /data/db
114+
rm -rf /opt/percona-server-mongodb/.git && mkdir -p /data/db
70115
WORKDIR /opt/percona-server-mongodb
71-
ENV POETRY_VIRTUALENVS_CREATE=false
72-
ENV POETRY_INSTALLER_MAX_WORKERS=10
73-
ENV PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
74-
ENV GLIBC_TUNABLES=glibc.pthread.rseq=0
75116
RUN set -ex; \
76117
if [ -f "/usr/bin/yum" ] ; then \
77118
yum install -y cmake openssl-devel openldap-devel krb5-devel libcurl-devel cyrus-sasl-devel bzip2-devel zlib-devel lz4-devel xz-devel e2fsprogs-devel iproute; \
78119
else \
79-
DEBIAN_FRONTEND=noninteractive apt-get -y install cmake libssl-dev libldap2-dev libkrb5-dev libcurl4-openssl-dev libsasl2-dev liblz4-dev libbz2-dev libsnappy-dev zlib1g-dev libzlcore-dev liblzma-dev e2fslibs-dev iproute2; \
120+
DEBIAN_FRONTEND=noninteractive apt-get -y install cmake libssl-dev libldap2-dev libkrb5-dev libcurl4-openssl-dev libsasl2-dev liblz4-dev libbz2-dev libsnappy-dev zlib1g-dev libzlcore-dev liblzma-dev e2fslibs-dev iproute2 libc6-dbg; \
80121
fi
81122
RUN set -ex; \
82123
pip install --upgrade pip; \
@@ -92,58 +133,85 @@ RUN set -ex; \
92133
pip install --upgrade referencing==0.35.1 && \
93134
pip install -r etc/pip/dev-requirements.txt; \
94135
fi
95-
ENV CMAKE_C_FLAGS=" -Wno-error=uninitialized "
96-
ENV CMAKE_CXX_FLAGS=" -Wno-error=deprecated-declarations -Wno-error=uninitialized "
97-
ENV AWS_LIBS=/opt/aws
98-
ARG AWS_VERSION=1.9.379
99-
RUN cd /opt && git clone https://github.com/aws/aws-sdk-cpp.git && \
100-
mkdir -p /opt/aws && \
101-
cd aws-sdk-cpp && git checkout $AWS_VERSION && git submodule update --init --recursive && \
102-
mkdir build && cd build && \
103-
cmake .. -DCMAKE_C_FLAGS="$CMAKE_C_FLAGS" -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS" -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=OFF -DMINIMIZE_SIZE=ON -DCMAKE_INSTALL_PREFIX="$AWS_LIBS" -DAUTORUN_UNIT_TESTS=OFF && \
104-
make -j "$(nproc)" && make install && \
105-
cd /opt && rm -rf aws-sdk-cpp
106-
ENV E2FS_LIBS=/opt/e2fs
107-
ARG E2FS_VERSION=v1.47.2
108-
RUN cd /opt && git clone https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git && \
109-
mkdir -p /opt/e2fs && \
110-
cd e2fsprogs && git checkout $E2FS_VERSION && \
111-
mkdir build && cd build && \
112-
../configure --prefix=$E2FS_LIBS && \
113-
make -j "$(nproc)" && make install-libs && \
114-
cd /opt && rm -rf e2fsprogs
115136
RUN /usr/bin/echo -e "\n\
116137
CC = \"/opt/gcc/bin/gcc\"\n\
117138
CXX = \"/opt/gcc/bin/g++\"\n\
118-
CPPPATH = \"/opt/aws/include /opt/e2fs/include\"\n\
119-
LIBPATH = \"/opt/aws/lib /opt/aws/lib64 /opt/e2fs/lib /opt/e2fs/lib64\"\n\
139+
CPPPATH = \"/usr/include\"\n\
140+
LIBPATH = \"/usr/lib /usr/lib64\"\n\
120141
" > gcc.vars
142+
ENV MONGO_VERSION=$psm_ver-$psm_release
143+
ADD https://raw.githubusercontent.com/Percona-QA/psmdb-testing/main/regression-tests/resmoke2junit.py .
121144

122145
FROM base_image AS psmdb_builder
123-
ARG psm_ver=8.0.3
124-
ARG pro=""
146+
ARG build_platform=bazel
147+
ENV PATH="/root/.local/bin:${PATH}"
125148
RUN set -ex; \
126-
VER=$(echo $psm_ver | cut -d"." -f1) && \
149+
VER=$(echo $MONGO_VERSION | cut -d"." -f1) && \
127150
if [ $VER -ge 8 ]; then \
128151
SCONS_TARGETS="install-mongod install-mongos install-mongo install-dbtest build/install/bin/mongobridge build/install/bin/wt build/install/bin/mongotmock" ;\
129152
else \
130153
SCONS_TARGETS="install-mongod install-mongos install-mongo install-dbtest build/install/bin/mongobridge build/install/bin/wt" ;\
131154
fi; \
132-
buildscripts/scons.py --variables-files=gcc.vars -j"$(nproc)" --disable-warnings-as-errors \
133-
--ssl --opt=on $pro --use-sasl-client --wiredtiger --audit --inmemory --hotbackup ${SCONS_TARGETS}
155+
if [ "$build_platform" = "bazel" ]; then \
156+
buildscripts/install_bazel.py && \
157+
bazel build --config=psmdb_opt_release --full_featured install-dist-test install-dbtest --define=MONGO_VERSION=$MONGO_VERSION && \
158+
mkdir -p build/install/bin && cp -RL bazel-bin/install/bin/* build/install/bin/ ;\
159+
else \
160+
buildscripts/scons.py --variables-files=gcc.vars -j"$(nproc)" --disable-warnings-as-errors \
161+
--ssl --opt=on --full-featured --use-sasl-client --wiredtiger --audit --inmemory --hotbackup ${SCONS_TARGETS} ;\
162+
fi
134163
RUN find build/install/bin -type f | xargs strip --strip-debug || true
135164

136-
FROM base_image
165+
FROM base_image AS unittests
166+
ARG build_platform=bazel
167+
RUN useradd -u 1001 -g 0 -m -c "Default Application User" mongodb && \
168+
chown -R mongodb /opt && chown -R mongodb /data/db && mkdir /work && chown -R mongodb /work
169+
USER mongodb
170+
ENV PATH="/home/mongodb/.local/bin:${PATH}"
171+
RUN set -ex; \
172+
if [ "$build_platform" = "bazel" ]; then \
173+
buildscripts/install_bazel.py && \
174+
bazel build --define=MONGO_VERSION=$MONGO_VERSION --config=psmdb_dev --include_autogenerated_targets=True --full_featured install-mongo_unittest ;\
175+
else \
176+
buildscripts/scons.py --variables-files=gcc.vars -j"$(nproc)" --disable-warnings-as-errors \
177+
--ssl --link-model=dynamic --linker=gold --dbg=off --opt=off --full-featured --use-sasl-client --wiredtiger --audit --inmemory --hotbackup install-unittests ;\
178+
fi
179+
180+
FROM base_image AS jstests
137181
COPY --from=psmdb_builder /opt/percona-server-mongodb/build/install/bin/* /usr/bin/
138182
RUN find jstests -type f | xargs chmod 400 && \
139183
cp /usr/bin/dbtest . || true && \
140184
support-files/ldap-sasl/deploy_ldap_and_sasl.sh || true && \
141-
echo "START=yes" >> /etc/default/saslauthd || true
185+
echo "START=yes" >> /etc/default/saslauthd || true
142186
ENV PATH="${PATH}:/data/multiversion"
143187
ENV PYTHONPATH="/opt/percona-server-mongodb:/opt/percona-server-mongodb/src"
144188
ENV TZ=America/New_York
145189
ENV PORTABLE=1
146190
ENV USE_SSE=1
147-
ADD https://raw.githubusercontent.com/Percona-QA/psmdb-testing/main/regression-tests/resmoke2junit.py .
148191
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
149-
RUN useradd -u 1001 -r -g 0 -m -s /sbin/nologin -c "Default Application User" mongodb || true
192+
RUN useradd -u 1001 -r -g 0 -m -s /sbin/nologin -c "Default Application User" mongodb || true
193+
194+
FROM base_image AS integrationtests
195+
COPY --from=psmdb_builder /opt/percona-server-mongodb/build/install/bin/* /usr/bin/
196+
ARG build_platform=bazel
197+
ENV PATH="/root/.local/bin:${PATH}"
198+
RUN set -ex; \
199+
if [ "$build_platform" = "bazel" ]; then \
200+
buildscripts/install_bazel.py && \
201+
bazel build --define=MONGO_VERSION=$MONGO_VERSION --config=psmdb_dev --include_autogenerated_targets=True --full_featured install-mongo_integration_test ;\
202+
else \
203+
buildscripts/scons.py --variables-files=gcc.vars -j"$(nproc)" --disable-warnings-as-errors \
204+
--ssl --link-model=dynamic --linker=gold --dbg=off --opt=off --full-featured --use-sasl-client --wiredtiger --audit --inmemory --hotbackup install-integration-tests ;\
205+
fi
206+
207+
FROM base_image AS benchmarks
208+
ARG build_platform=bazel
209+
ENV PATH="/root/.local/bin:${PATH}"
210+
RUN set -ex; \
211+
if [ "$build_platform" = "bazel" ]; then \
212+
buildscripts/install_bazel.py && \
213+
bazel build --define=MONGO_VERSION=$MONGO_VERSION --config=psmdb_opt_release --include_autogenerated_targets=True --full_featured install-mongo_benchmark-stripped ;\
214+
else \
215+
buildscripts/scons.py --variables-files=gcc.vars -j"$(nproc)" --disable-warnings-as-errors \
216+
--ssl --link-model=dynamic --linker=gold --dbg=off --opt=off --full-featured --use-sasl-client --wiredtiger --audit --inmemory --hotbackup install-benchmarks ;\
217+
fi

0 commit comments

Comments
 (0)