Merge branch 'master' into notifications-python

Author: twalen

.gitattributes

@@ -0,0 +1,7 @@
+# This file exists for multi-system support
+
+# default file behaviour
+* text=auto
+
+# make all '.sh' files LF, regardless of users' OS
+*.sh text eol=lf

.github/workflows/ruff.yml

@@ -15,4 +15,4 @@ jobs:
       - uses: astral-sh/ruff-action@v3
         with:
           src: "./oioioi"
-          version: 0.11.12
+          version: 0.13.2

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.12
+    rev: v0.13.2
     hooks:
     -   id: ruff-check
         files: ^oioioi/

Dockerfile

@@ -58,21 +58,16 @@ RUN pip3 install --user bsddb3==6.2.7
 
 WORKDIR /sio2/oioioi
 
-COPY --chown=oioioi:oioioi setup.py requirements.txt ./
-RUN pip3 install -r requirements.txt --user filetracker[server]
-COPY --chown=oioioi:oioioi requirements_static.txt ./
-RUN pip3 install -r requirements_static.txt --user
-RUN pip3 install --user -U "gevent==25.5.1"  # override version of gevent
+COPY --chown=oioioi:oioioi . ./
+RUN pip3 install --user -r requirements.txt filetracker[server]
+RUN pip3 install --user -r requirements_static.txt
 
 # Installing node dependencies
 ENV PATH $PATH:/sio2/oioioi/node_modules/.bin
 
-COPY --chown=oioioi:oioioi package.json package-lock.json ./
 RUN npm ci
-
-COPY --chown=oioioi:oioioi . /sio2/oioioi
-
 RUN npm run build
+
 RUN oioioi-create-config /sio2/deployment
 
 WORKDIR /sio2/deployment
@@ -103,6 +98,8 @@ FROM base AS development
 COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes
 RUN chmod +x /sio2/oioioi/download_sandboxes.sh
 
+# For production: Upload sandboxes to built-in filetracker during build
+# For dev: These will be re-uploaded to s3dedup at runtime via oioioi_init.sh
 RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \
     /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \
     ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \

UPGRADING.rst

@@ -347,7 +347,7 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
         # directory in which necessary files will be stored.
         #FILETRACKER_CACHE_ROOT = '__DIR__/cache'
 
-        # When using a remote storage it's recommended to enable a cache cleaner deamon
+        # When using a remote storage it's recommended to enable a cache cleaner daemon
         # which will periodically scan cache directory and remove files what aren't
         # used. For a detailed description of each option, please read a cache cleaner
         # configuration section in the sioworkersd documentation.
@@ -439,7 +439,7 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
 
         -# Filetracker server settings.
         -#
-        -# Determines which filetracker database use, availible options are:
+        -# Determines which filetracker database use, available options are:
         -# - 'oioioi.filetracker.client.media_root_factory' (the default)
         -#    Stores files on local filesystem under MEDIA_ROOT, optionally
         -#    exposing them with a filetracker server (see section below).
@@ -588,7 +588,7 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
 
         #    'oioioi.usergroups',
 
-#. * Introduced `DEFAULT_COMPILERS` to settings, which should be set for every language supoorted::
+#. * Introduced `DEFAULT_COMPILERS` to settings, which should be set for every language supported::
 
         --- a/oioioi/default_settings.py
         +++ b/oioioi/default_settings.py
@@ -896,7 +896,7 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
         +# The server to be run. Options are:
         +# 'django' - django's http server
         +# 'uwsgi' - uwsgi daemon
-        +# 'uwsgi-http' - uwsgi deamon with built-in http server
+        +# 'uwsgi-http' - uwsgi daemon with built-in http server
         +# None - nothing will be run
         +SERVER = 'django'
 
@@ -1121,3 +1121,49 @@ List of changes since the *CONFIG_VERSION* numbering was introduced:
         +# )
 
      * Remove all sentry and raven reminiscent from settings.py in LOGGING SECTION.
+
+Migrating from Filetracker to s3dedup (currently only for docker-compose-dev.yml deployments)
+----------------------------------------------------------------
+
+This guide covers migrating from the built-in Filetracker to s3dedup in Docker Compose
+development deployments. This is currently only needed deployments using ``docker-compose-dev.yml``.
+If you don't have any data you want to keep, you don't have to do anything.
+
+#. Stop the s3dedup container::
+
+    docker-compose -f docker-compose-dev.yml stop s3dedup
+
+#. In ``docker-compose-dev.yml``, modify the web container environment variables:
+
+   * Remove ``FILETRACKER_SERVER_ENABLED=False`` (or set to ``True``)
+   * Set ``FILETRACKER_LISTEN_ADDR=0.0.0.0``
+   * Set ``FILETRACKER_LISTEN_PORT=9999``
+
+#. Restart the web container::
+
+    docker-compose -f docker-compose-dev.yml down web
+    docker-compose -f docker-compose-dev.yml up -d web
+
+#. Run the migration::
+
+    docker-compose -f docker-compose-dev.yml run --rm s3dedup migrate --env \
+      --filetracker-url http://web:9999 \
+      --max-concurrency 10
+
+   The migration will display progress and statistics about files migrated and
+   deduplication savings.
+
+#. Once migration completes, restore the environment variables in ``docker-compose-dev.yml``:
+
+   * Remove or comment out ``FILETRACKER_LISTEN_ADDR`` and ``FILETRACKER_LISTEN_PORT``
+   * Set ``FILETRACKER_SERVER_ENABLED=False``
+
+#. Restart the web container and start s3dedup::
+
+    docker-compose -f docker-compose-dev.yml down web
+    docker-compose -f docker-compose-dev.yml up -d web s3dedup
+
+#. Verify the migration by checking that files are accessible through s3dedup.
+
+For more details about the migration process, see the s3dedup documentation at
+``https://github.com/sio2project/s3dedup``.

conftest.py

@@ -1,10 +1,7 @@
-from __future__ import print_function
-
 import pytest
-
+from django.conf import settings
 from oioioi.base.tests import pytest_plugin as base_plugin
 from oioioi.contests.tests import pytest_plugin as contests_plugin
-from django.conf import settings
 
 
 def pytest_addoption(parser):

docker-compose-dev.yml

@@ -1,5 +1,5 @@
 x-common-envs: &common-envs
- FILETRACKER_URL: 'http://web:9999'
+ FILETRACKER_URL: 'http://s3dedup:9999/ft'
  DATABASE_HOST: 'db'
  DATABASE_PORT: '5432'
 
@@ -30,8 +30,7 @@ services:
       RABBITMQ_PORT: '5672'
       RABBITMQ_USER: 'oioioi'
       RABBITMQ_PASSWORD: 'oioioi'
-      FILETRACKER_LISTEN_ADDR: '0.0.0.0'
-      FILETRACKER_LISTEN_PORT: '9999'
+      FILETRACKER_SERVER_ENABLED: 'False'
       <<: *common-envs
     ports:
     # web server
@@ -49,6 +48,7 @@ services:
     depends_on:
       - db
       - broker
+      - s3dedup
   worker:
     image: sio2project/oioioi-dev
     command: ["/sio2/oioioi/worker_init.sh"]
@@ -68,8 +68,55 @@ services:
       RABBITMQ_DEFAULT_USER: oioioi
       RABBITMQ_DEFAULT_PASS: oioioi
     stop_grace_period: 1m
+  minio:
+    image: minio/minio:RELEASE.2024-10-02T17-50-41Z
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    volumes:
+      - minio-data-dev:/data
+    command: server /data --console-address ":9001"
+    ports:
+      - "9000:9000"  # MinIO API
+      - "9001:9001"  # MinIO Console
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 5s
+    stop_grace_period: 1m
+  s3dedup:
+    image: ghcr.io/sio2project/s3dedup:latest-dev
+    ports:
+      - "9999:9999"
+    environment:
+      LOG_LEVEL: debug
+      BUCKET_NAME: oioioi-filetracker
+      LISTEN_ADDRESS: 0.0.0.0
+      LISTEN_PORT: 9999
+      KVSTORAGE_TYPE: sqlite
+      SQLITE_PATH: /app/data/kv.db
+      SQLITE_MAX_CONNECTIONS: 10
+      S3_ENDPOINT: http://minio:9000
+      S3_ACCESS_KEY: minioadmin
+      S3_SECRET_KEY: minioadmin
+      S3_FORCE_PATH_STYLE: "true"
+      CLEANER_ENABLED: "true"
+      CLEANER_INTERVAL: 3600
+      CLEANER_BATCH_SIZE: 1000
+      CLEANER_MAX_DELETES: 10000
+    volumes:
+      - s3dedup-data-dev:/app/data
+    depends_on:
+      minio:
+        condition: service_healthy
+    restart: on-failure:5
+    stop_grace_period: 1m
 volumes:
   postgress-data-dev:
   filetracker-data-dev:
   node_modules-dev:
   dist_webpack-dev:
+  minio-data-dev:
+  s3dedup-data-dev:

easy_toolbox.py

@@ -19,7 +19,13 @@
 import sys
 from shlex import quote
 
-BASE_DOCKER_COMMAND = "OIOIOI_UID=$(id -u) docker compose" + " -f docker-compose-dev.yml"
+if sys.platform == "win32":
+    # for Windows, a different syntax command_prefix is needed
+    command_prefix = "set OIOIOI_UID=1000 &&"
+else:
+    command_prefix = "OIOIOI_UID=$(id -u)"
+
+BASE_DOCKER_COMMAND = f"{command_prefix} docker compose -f docker-compose-dev.yml"
 
 RAW_COMMANDS = [
     ("build", "Build OIOIOI container from source.", "build", True),
@@ -45,7 +51,7 @@
         "Run coverage tests.",
         "{exec} 'web' ../oioioi/test.sh oioioi/problems --cov-report term --cov-report xml:coverage.xml --cov=oioioi {extra_args}",
     ),
-    ("cypress-apply-settings", "Apply settings for CyPress.", '{exec} web bash -c "echo CAPTCHA_TEST_MODE=True >> settings.py"'),
+    ("cypress-apply-settings", "Apply settings for CyPress.", '{exec} web bash -c "echo >> settings.py && echo CAPTCHA_TEST_MODE=True >> settings.py"'),
     ("npm", "Run npm command.", "{exec} web npm --prefix ../oioioi {extra_args}"),
     ("eslint", "Run javascript linter.", "{exec} web npm --prefix ../oioioi run lint"),
     (
@@ -132,15 +138,22 @@ def get_action_from_gui() -> Option:
         ),
     ]
     answers = inquirer.prompt(questions)
-    return answers["action"]
+    return COMMANDS[answers["action"]]
 
 
 def run_command(command) -> None:
     print("Running command", command)
     if not NO_INPUT:
         width = os.get_terminal_size().columns
         print("=" * width)
-    sys.exit(os.WEXITSTATUS(os.system(command)))
+
+
+    if sys.platform == "win32":
+        # for Windows, a different syntax for exiting is needed
+        exit_code = os.system(command)
+        sys.exit(exit_code)
+    else:
+        sys.exit(os.WEXITSTATUS(os.system(command)))
 
 
 def warn_user(action: Option) -> bool:
@@ -189,6 +202,5 @@ def main() -> None:
     except Exception as e:
         print(f"An error occurred during execution: {e}", file=sys.stderr)
 
-
 if __name__ == "__main__":
     main()

extra/sample-configs/runnable-test-settings.py

@@ -37,7 +37,7 @@
 #
 
 #
-# To workarund a bug, where sioworkers is not passed
+# To workaround a bug, where sioworkers is not passed
 # FILETRACKER_CACHE_ROOT, use this to start the website:
 #
 # FILETRACKER_URL=/tmp/oioioi-filetracker-cache ./manage.py runserver

ez_setup.py

@@ -14,8 +14,6 @@
 This file can also be run as a script to install or upgrade setuptools.
 """
 
-from __future__ import print_function
-
 import os
 import sys
 
@@ -25,7 +23,7 @@
     from md5 import md5
 
 DEFAULT_VERSION = "0.6c11"
-DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3]
+DEFAULT_URL = f"http://pypi.python.org/packages/{sys.version[:3]}/s/setuptools/"
 
 md5_data = {
     "setuptools-0.6b1-py2.3.egg": "8822caf901250d848b996b7f25c6e6ca",
@@ -78,7 +76,7 @@ def _validate_md5(egg_name, data):
         digest = md5(data).hexdigest()
         if digest != md5_data[egg_name]:
             print(
-                ("md5 validation of %s failed!  (Possible download problem?)" % egg_name),
+                (f"md5 validation of {egg_name} failed!  (Possible download problem?)"),
                 file=sys.stderr,
             )
             sys.exit(2)
@@ -122,12 +120,11 @@ def do_download():
         if was_imported:
             print(
                 (
-                    "The required version of setuptools (>=%s) is not available, and\n"
+                    f"The required version of setuptools (>={version}) is not available, and\n"
                     "can't be installed while this script is running. Please install\n"
                     " a more recent version first, using 'easy_install -U setuptools'."
-                    "\n\n(Currently using %r)"
-                )
-                % (version, e.args[0]),
+                    f"\n\n(Currently using {e.args[0]!r})"
+                ),
                 file=sys.stderr,
             )
             sys.exit(2)
@@ -148,7 +145,7 @@ def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_d
     """
     import urllib.request
 
-    egg_name = "setuptools-%s-py%s.egg" % (version, sys.version[:3])
+    egg_name = f"setuptools-{version}-py{sys.version[:3]}.egg"
     url = download_base + egg_name
     saveto = os.path.join(to_dir, egg_name)
     src = dst = None
@@ -251,7 +248,7 @@ def update_md5(filenames):
         md5_data[base] = md5(f.read()).hexdigest()
         f.close()
 
-    data = ["    %r: %r,\n" % it for it in md5_data.items()]
+    data = ["    {!r}: {!r},\n".format(*it) for it in md5_data.items()]
     data.sort()
     repl = "".join(data)