upgrade numpy to 2.0.0 and fix xgboost

docker/3.0.5/base/Dockerfile.cpu

@@ -0,0 +1,205 @@
+ARG UBUNTU_VERSION=20.04
+ARG CUDA_VERSION=12.8.0
+ARG IMAGE_DIGEST=c2d95c9c6ff77da41cf0f2f9e8c5088f5b4db20c16a7566b808762f05b9032ef
+
+# Build stage for SQLite compilation
+FROM ubuntu:${UBUNTU_VERSION} as sqlite-builder
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential \
+        wget \
+        ca-certificates \
+        && \
+    cd /tmp && \
+    wget https://www.sqlite.org/2025/sqlite-autoconf-3500200.tar.gz && \
+    tar xzf sqlite-autoconf-3500200.tar.gz && \
+    cd sqlite-autoconf-3500200 && \
+    ./configure --prefix=/usr/local && \
+    make && \
+    make install && \
+    ldconfig && \
+    cd / && \
+    rm -rf /tmp/sqlite-autoconf-3500200 /tmp/sqlite-autoconf-3500200.tar.gz && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Main image
+FROM nvidia/cuda:${CUDA_VERSION}-base-ubuntu${UBUNTU_VERSION}
+
+ARG MINICONDA_VERSION=25.9.1
+ARG CONDA_CHECKSUM=04a8b03d8b0ec062d923e592201a6fd88b7247c309ef8848afb25c424c40ac39
+ARG CONDA_PY_VERSION=310
+ARG CONDA_PKG_VERSION=25.9.1
+ARG PYTHON_VERSION=3.10
+ARG PYARROW_VERSION=22.0.0
+ARG MLIO_VERSION=0.9.0
+ARG XGBOOST_VERSION=3.0.5
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
+
+# Python won’t try to write .pyc or .pyo files on the import of source modules
+# Force stdin, stdout and stderr to be totally unbuffered. Good for logging
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONIOENCODING='utf-8'
+
+RUN apt-key del 7fa2af80 && \
+        apt-get update && apt-get install -y --no-install-recommends wget && \
+        wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-keyring_1.0-1_all.deb && \
+        dpkg -i cuda-keyring_1.0-1_all.deb && \
+        apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install --no-install-recommends \
+        build-essential \
+        curl \
+        git \
+        jq \
+        libatlas-base-dev \
+        expat \
+        nginx \
+        openjdk-8-jdk-headless \
+        unzip \
+        wget \
+        apparmor \
+        linux-libc-dev \
+        libxml2 \
+        libgstreamer1.0-0 \
+        linux-libc-dev \
+        && \
+    # MLIO build dependencies
+    # Official Ubuntu APT repositories do not contain an up-to-date version of CMake required to build MLIO.
+    # Kitware contains the latest version of CMake.
+    wget http://es.archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi7_3.3-4_amd64.deb && \
+    dpkg -i libffi7_3.3-4_amd64.deb && \
+    apt-get -y install --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        gnupg \
+        software-properties-common \
+        && \
+    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
+        gpg --dearmor - | \
+        tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null && \
+    echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ bionic main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null && \
+    apt-get update && \
+    rm /usr/share/keyrings/kitware-archive-keyring.gpg && \
+    apt-get install -y --no-install-recommends \
+        autoconf \
+        automake \
+        build-essential \
+        cmake \
+        cmake-data \
+        doxygen \
+        kitware-archive-keyring \
+        libcurl4-openssl-dev \
+        libssl-dev \
+        libtool \
+        ninja-build \
+        python3-dev \
+        python3-distutils \
+        python3-pip \
+        zlib1g-dev \
+        libxml2 \
+        zstd \
+        libsqlite3-0 \
+        && \
+    python3 -m pip install --upgrade pip && \
+    python3 -m pip install --upgrade certifi && \
+    apt-get clean && \
+    # Node.js setup
+    mkdir -p /etc/apt/keyrings && \
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | \
+        gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | \
+        tee /etc/apt/sources.list.d/nodesource.list && \
+    apt-get update && \
+    apt-get install -y nodejs && \
+    npm install -g npm@latest && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install conda
+RUN cd /tmp && \
+    curl -L --output /tmp/Miniconda3.sh https://repo.anaconda.com/miniconda/Miniconda3-py${CONDA_PY_VERSION}_${MINICONDA_VERSION}-1-Linux-x86_64.sh && \ 
+    echo "${CONDA_CHECKSUM} /tmp/Miniconda3.sh" | sha256sum -c - && \
+    bash /tmp/Miniconda3.sh -bfp /miniconda3 && \
+    rm /tmp/Miniconda3.sh
+
+ENV PATH=/miniconda3/bin:${PATH}
+# Install MLIO with Apache Arrow integration
+RUN conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main
+RUN conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
+# We could install mlio-py from conda, but it comes  with extra support such as image reader that increases image size
+# which increases training time. We build from source to minimize the image size.
+RUN conda config --system --set channel_priority strict && \
+    conda config --system --set auto_update_conda false && \
+    conda config --system --set show_channel_urls true && \
+    conda install -y -c conda-forge \
+        python=${PYTHON_VERSION} \
+        requests=2.32.3 \
+        conda=${CONDA_PKG_VERSION} \
+        pyarrow=${PYARROW_VERSION} \
+    --solver=libmamba && \
+    conda clean -afy
+
+# Then handle the grpc and npm parts separately
+RUN git clone -b v1.65.4 https://github.com/grpc/grpc.git && \
+    LIBGRPC_DIR=$(find /miniconda3/pkgs -name "libgrpc-*" -type d | head -n 1) && \
+    mkdir -p ${LIBGRPC_DIR}/info/test/examples && \
+    cp -r grpc/examples/* ${LIBGRPC_DIR}/info/test/examples/
+
+    RUN cd ${LIBGRPC_DIR}/info/test/examples/node && \
+    npm cache clean --force && \
+    npm install minimist@latest protobufjs@latest \
+    apt-get purge -y nodejs npm && \
+    apt-get autoremove -y && \
+    rm -rf /etc/apt/sources.list.d/nodesource.list \
+        /etc/apt/keyrings/nodesource.gpg \
+        /etc/apt/sources.list.d/kitware.list && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    # Continue with the rest of the build process
+    cd /tmp && \
+    git clone --branch v${MLIO_VERSION} https://github.com/awslabs/ml-io.git mlio && \
+    cd mlio && \
+    sed -i 's/find_package(Arrow 14.0.1 REQUIRED/find_package(Arrow 22.0.0 REQUIRED/g' CMakeLists.txt && \
+    sed -i 's/pyarrow==14.0.1/pyarrow==22.0.0/g' src/mlio-py/setup.py && \
+    build-tools/build-dependency build/third-party all && \
+    mkdir -p build/release && \
+    cd build/release && \
+    cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_PREFIX_PATH="$(pwd)/../third-party" ../.. && \
+    cmake --build . && \
+    cmake --build . --target install && \
+    cmake -DMLIO_INCLUDE_PYTHON_EXTENSION=ON -DPYTHON_EXECUTABLE="/miniconda3/bin/python3" \
+        -DMLIO_INCLUDE_ARROW_INTEGRATION=ON ../.. && \
+    cmake --build . --target mlio-py && \
+    cmake --build . --target mlio-arrow && \
+    cd ../../src/mlio-py && \
+    python3 setup.py bdist_wheel && \
+    python3 -m pip install typing && \
+    python3 -m pip install --upgrade pip && \
+    python3 -m pip install dist/*.whl && \
+    cp -r /tmp/mlio/build/third-party/lib/libtbb* /usr/local/lib/ && \
+    ldconfig && \
+    rm -rf /tmp/mlio
+
+# Copy compiled SQLite from builder stage
+COPY --from=sqlite-builder /usr/local/bin/sqlite3 /usr/local/bin/sqlite3
+COPY --from=sqlite-builder /usr/local/lib/libsqlite3.* /usr/local/lib/
+COPY --from=sqlite-builder /usr/local/include/sqlite3*.h /usr/local/include/
+
+# Update library cache and ensure /usr/local/bin is in PATH
+RUN ldconfig && \
+    echo "/usr/local/lib" > /etc/ld.so.conf.d/sqlite3.conf && \
+    ldconfig
+
+ENV PATH="/usr/local/bin:${PATH}"
+
+RUN echo "sqlite3 " 
+# This command will check the version and print it to the build logs
+RUN sqlite3 --version
+
+RUN apt list --installed
+
+# Install latest version of XGBoost
+RUN python3 -m pip install --no-cache -I xgboost==${XGBOOST_VERSION} numpy==2.1.0 pyarrow==22.0.0 pandas==2.2.3

docker/3.0.5/final/Dockerfile.cpu

@@ -0,0 +1,100 @@
+ARG SAGEMAKER_XGBOOST_VERSION=3.0-5
+ARG PYTHON_VERSION=3.10
+
+FROM xgboost-container-base:${SAGEMAKER_XGBOOST_VERSION}-cpu-py3
+
+ARG SAGEMAKER_XGBOOST_VERSION=3.0.5
+
+########################
+# Install dependencies #
+########################
+
+# Fix Python 3.10 compatibility for sagemaker-containers
+# RUN python3 -c "import sys; sys.path.insert(0, '/miniconda3/lib/python3.10/site-packages'); \
+#     import sagemaker_containers._mapping as m; \
+#     import collections.abc; \
+#     setattr(collections, 'Mapping', collections.abc.Mapping); \
+#     exec(open('/miniconda3/lib/python3.10/site-packages/sagemaker_containers/_mapping.py').read().replace('collections.Mapping', 'collections.abc.Mapping'))" || \
+#     sed -i 's/collections\.Mapping/collections.abc.Mapping/g' /miniconda3/lib/python3.10/site-packages/sagemaker_containers/_mapping.py
+
+
+# Install smdebug from source
+RUN python3 -m pip install git+https://github.com/awslabs/[email protected]
+
+COPY requirements.txt /requirements.txt
+RUN python3 -m pip install -r /requirements.txt && rm /requirements.txt
+
+RUN pip install --no-cache-dir "protobuf>=3.20.0,<=3.20.3"
+# ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
+RUN sed -i 's/collections\.Mapping/collections.abc.Mapping/g' /miniconda3/lib/python3.10/site-packages/sagemaker_containers/_mapping.py
+
+###########################
+# Copy wheel to container #
+###########################
+COPY dist/sagemaker_xgboost_container-2.0-py2.py3-none-any.whl /sagemaker_xgboost_container-1.0-py2.py3-none-any.whl
+RUN rm -rf /miniconda3/lib/python${PYTHON_VERSION}/site-packages/numpy-1.21.2.dist-info && \
+    python3 -m pip install --force-reinstall PyYAML==6.0.1 && \
+    python3 -m pip install --no-cache --no-deps /sagemaker_xgboost_container-1.0-py2.py3-none-any.whl && \
+    python3 -m pip uninstall -y typing && \
+    rm /sagemaker_xgboost_container-1.0-py2.py3-none-any.whl
+
+##############
+# DMLC PATCH #
+##############
+# TODO: remove after making contributions back to xgboost for tracker.py
+# COPY src/sagemaker_xgboost_container/dmlc_patch/tracker.py \
+#    /miniconda3/lib/python${PYTHON_VERSION}/site-packages/xgboost/dmlc-core/tracker/dmlc_tracker/tracker.py
+
+# # Include DMLC python code in PYTHONPATH to use RabitTracker
+# ENV PYTHONPATH=$PYTHONPATH:/miniconda3/lib/python${PYTHON_VERSION}/site-packages/xgboost/dmlc-core/tracker
+
+#######
+# MMS #
+#######
+# Create MMS user directory
+RUN useradd -m model-server
+RUN mkdir -p /home/model-server/tmp && chown -R model-server /home/model-server
+
+# Copy MMS configs
+COPY docker/${SAGEMAKER_XGBOOST_VERSION}/resources/mms/config.properties.tmp /home/model-server
+ENV XGBOOST_MMS_CONFIG=/home/model-server/config.properties
+
+# Copy execution parameters endpoint plugin for MMS
+RUN mkdir -p /tmp/plugins
+COPY docker/${SAGEMAKER_XGBOOST_VERSION}/resources/mms/endpoints-1.0.jar /tmp/plugins
+RUN chmod +x /tmp/plugins/endpoints-1.0.jar
+
+# Create directory for models
+RUN mkdir -p /opt/ml/models
+RUN chmod +rwx /opt/ml/models
+
+# Copy Dask configs
+RUN mkdir /etc/dask
+COPY docker/configs/dask_configs.yaml /etc/dask/
+
+# Required label for multi-model loading
+LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
+
+#####################
+# Required ENV vars #
+#####################
+# Set SageMaker training environment variables
+ENV SM_INPUT /opt/ml/input
+ENV SM_INPUT_TRAINING_CONFIG_FILE $SM_INPUT/config/hyperparameters.json
+ENV SM_INPUT_DATA_CONFIG_FILE $SM_INPUT/config/inputdataconfig.json
+ENV SM_CHECKPOINT_CONFIG_FILE $SM_INPUT/config/checkpointconfig.json
+# See: https://github.com/dmlc/xgboost/issues/7982#issuecomment-1379390906 https://github.com/dmlc/xgboost/pull/8257
+ENV NCCL_SOCKET_IFNAME eth 
+
+
+# Set SageMaker serving environment variables
+ENV SM_MODEL_DIR /opt/ml/model
+
+# Set SageMaker entrypoints
+ENV SAGEMAKER_TRAINING_MODULE sagemaker_xgboost_container.training:main
+ENV SAGEMAKER_SERVING_MODULE sagemaker_xgboost_container.serving:main
+
+EXPOSE 8080
+ENV TEMP=/home/model-server/tmp
+LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true