Skip to content

Docker optimizations #566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Docker optimizations #566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f5ec139
Decrease docker image size
otargowski Oct 8, 2025
800afb3
Address a warning for old ENV syntax in Dockerfile
otargowski Nov 10, 2025
65a4e19
Parallelize uploading sandboxes to filetracker
otargowski Oct 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 26 additions & 16 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
FROM python:3.11 AS base
FROM python:3.11-slim AS base

ENV PYTHONUNBUFFERED 1
ENV PYTHONUNBUFFERED=1

#RUN dpkg --add-architecture i386
RUN apt-get update && \
apt-get install -y \
apt-get install --no-install-recommends -y \
git \
libpq-dev \
postgresql-client \
Expand All @@ -16,9 +16,13 @@ RUN apt-get update && \
texlive-lang-european \
texlive-lang-czechslovak \
texlive-pstricks \
ghostscript \
texlive-fonts-recommended \
tex-gyre \
ghostscript \
make \
gcc \
g++ \
libc6-dev \
sudo \
libstdc++6 \
zlib1g \
Expand All @@ -28,7 +32,8 @@ RUN apt-get update && \
python3-pip \
nodejs \
npm && \
apt-get clean
apt-get clean && \
rm -rf /usr/share/doc/texlive*

# This is oioioi user linux uid. Setting it is useful in development.
# By default we use an unused uid of 1234.
Expand All @@ -50,9 +55,9 @@ RUN sed -i -e "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen && \
# Installing python dependencies
USER oioioi

ENV PATH $PATH:/home/oioioi/.local/bin/
ENV PATH=$PATH:/home/oioioi/.local/bin/

ENV BERKELEYDB_DIR /usr
ENV BERKELEYDB_DIR=/usr
RUN pip3 install --user psycopg2-binary twisted uwsgi
RUN pip3 install --user bsddb3==6.2.7

Expand All @@ -63,7 +68,7 @@ RUN pip3 install --user -r requirements.txt filetracker[server]
RUN pip3 install --user -r requirements_static.txt

# Installing node dependencies
ENV PATH $PATH:/sio2/oioioi/node_modules/.bin
ENV PATH=$PATH:/sio2/oioioi/node_modules/.bin

RUN npm ci
RUN npm run build
Expand All @@ -75,7 +80,7 @@ WORKDIR /sio2/deployment
RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver}

# The stage below is independent of base and can be built in parallel to optimize build time.
FROM python:3.11 AS development-sandboxes
FROM python:3.11-slim AS development-sandboxes

ENV DOWNLOAD_DIR=/sio2/sandboxes
ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest
Expand All @@ -84,7 +89,7 @@ ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest
ADD $MANIFEST_URL /sio2/Manifest

RUN apt-get update && \
apt-get install -y curl wget bash && \
apt-get install --no-install-recommends -y curl wget bash && \
apt-get clean

COPY download_sandboxes.sh /download_sandboxes.sh
Expand All @@ -93,14 +98,19 @@ RUN chmod +x /download_sandboxes.sh
# Run script to download sandbox data from the given Manifest.
RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL

FROM base AS development

# For production (or dev with filetracker): Upload sandboxes to built-in filetracker during build
# For dev: The sandboxes will be re-uploaded to s3dedup at runtime via oioioi_init.sh
FROM base AS base_with_sandboxes
COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes
RUN chmod +x /sio2/oioioi/download_sandboxes.sh

# For production: Upload sandboxes to built-in filetracker during build
# For dev: These will be re-uploaded to s3dedup at runtime via oioioi_init.sh
RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \
FROM base_with_sandboxes AS base_with_populated_filetracker
RUN ./manage.py supervisor > /dev/null --daemonize \
--nolaunch={uwsgi,unpackmgr,evalmgr,rankingsd,mailnotifyd,sioworkersd,receive_from_workers} && \
/sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \
./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \
./manage.py supervisor stop all

FROM base_with_sandboxes AS development

FROM base AS development_filetracker
COPY --from=base_with_populated_filetracker /sio2/deployment/media /sio2/deployment/media
22 changes: 11 additions & 11 deletions oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
import os
import os.path
from concurrent.futures import ProcessPoolExecutor
from pathlib import Path

from django.core.management.base import BaseCommand

from oioioi.filetracker.client import get_client


def upload_sandbox(file):
filetracker = get_client()
filetracker.put_file("/sandboxes/" + file.name, str(file))


class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
Expand All @@ -19,17 +24,12 @@ def add_arguments(self, parser):

help = "Upload sandboxes to the Filetracker."

def handle(self, *args, **options):
filetracker = get_client()

def handle(self, *args, **options):
print("--- Saving sandboxes to the Filetracker ...", file=self.stdout)

sandboxes_dir = os.fsencode(options["sandboxes_dir"])
for file in os.listdir(sandboxes_dir):
filename = os.fsdecode(file)
if not filename.endswith(".tar.gz"):
continue

filetracker.put_file("/sandboxes/" + filename, os.path.join(options["sandboxes_dir"], filename))
sandboxes_dir = Path(options["sandboxes_dir"])
with ProcessPoolExecutor() as executor:
executor.map(upload_sandbox, sandboxes_dir.glob('*.tar.gz'))

print("--- Done.", file=self.stdout)