From 50f032e26b31773a4f03d010eba9fe4a496a9815 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Tue, 14 Feb 2023 15:54:57 +0000 Subject: [PATCH 1/9] Add git submodules for Merlin projects --- .gitmodules | 18 ++++++++++++++++++ merlin/NVTabular | 1 + merlin/Transformers4Rec | 1 + merlin/core | 1 + merlin/dataloader | 1 + merlin/models | 1 + merlin/systems | 1 + 7 files changed, 24 insertions(+) create mode 100644 .gitmodules create mode 160000 merlin/NVTabular create mode 160000 merlin/Transformers4Rec create mode 160000 merlin/core create mode 160000 merlin/dataloader create mode 160000 merlin/models create mode 160000 merlin/systems diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..3678fd50e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,18 @@ +[submodule "merlin/systems"] + path = merlin/systems + url = git@github.com:NVIDIA-Merlin/systems.git +[submodule "merlin/NVTabular"] + path = merlin/NVTabular + url = git@github.com:NVIDIA-Merlin/NVTabular.git +[submodule "merlin/core"] + path = merlin/core + url = git@github.com:NVIDIA-Merlin/core.git +[submodule "merlin/models"] + path = merlin/models + url = git@github.com:NVIDIA-Merlin/models.git +[submodule "merlin/dataloader"] + path = merlin/dataloader + url = git@github.com:NVIDIA-Merlin/dataloader.git +[submodule "merlin/Transformers4Rec"] + path = merlin/Transformers4Rec + url = git@github.com:NVIDIA-Merlin/Transformers4Rec.git diff --git a/merlin/NVTabular b/merlin/NVTabular new file mode 160000 index 000000000..57e5514aa --- /dev/null +++ b/merlin/NVTabular @@ -0,0 +1 @@ +Subproject commit 57e5514aa6e5f6a05779a5baaf48a50a8acc2888 diff --git a/merlin/Transformers4Rec b/merlin/Transformers4Rec new file mode 160000 index 000000000..ae5f30508 --- /dev/null +++ b/merlin/Transformers4Rec @@ -0,0 +1 @@ +Subproject commit ae5f305083aaa74c7b4562c70492b42b970d7701 diff --git a/merlin/core b/merlin/core new file mode 160000 index 000000000..0f6ee9820 --- /dev/null +++ b/merlin/core @@ -0,0 +1 @@ +Subproject commit 0f6ee9820094baca2a083ecdd3c82db338f7e6aa diff --git a/merlin/dataloader b/merlin/dataloader new file mode 160000 index 000000000..02aad2124 --- /dev/null +++ b/merlin/dataloader @@ -0,0 +1 @@ +Subproject commit 02aad2124e247e6a4f229d6638eaaec0931aca8c diff --git a/merlin/models b/merlin/models new file mode 160000 index 000000000..d8d53575d --- /dev/null +++ b/merlin/models @@ -0,0 +1 @@ +Subproject commit d8d53575d78f5bb8883c661f334edebc14946b8e diff --git a/merlin/systems b/merlin/systems new file mode 160000 index 000000000..e4bf45a9d --- /dev/null +++ b/merlin/systems @@ -0,0 +1 @@ +Subproject commit e4bf45a9d112b689036228189659fe9669791b82 From abee44f3e78fb5911133e71ad0e462f9d4d644fa Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Tue, 14 Feb 2023 15:58:53 +0000 Subject: [PATCH 2/9] Add docker-compose.yml and dev Dockerfile for TensorFlow image --- docker-compose.yml | 31 +++++++++++++++++ docker/dev.tf.Dockerfile | 74 ++++++++++++++++++++++++++++++++++++++++ requirements.in | 0 3 files changed, 105 insertions(+) create mode 100644 docker-compose.yml create mode 100644 docker/dev.tf.Dockerfile create mode 100644 requirements.in diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..14a137138 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,31 @@ +--- +services: + + dev-tf: + build: + context: . + dockerfile: docker/dev.tf.Dockerfile + args: + - UID + - GID + - USER + volumes: + - .:/workspace/dev + - /dev/shm:/dev/shm # required for memory mapping to work as non-root user + ports: + - "8888:8888" + command: "jupyter-nbclassic --ip 0.0.0.0 --allow-root" + entrypoint: "/opt/nvidia/nvidia_entrypoint.sh" + environment: + HISTFILE: "/workspace/.bash_history" + IPYTHONDIR: "/workspace/.ipython" + BASE_DIR: "/workspace/data/base" + FEAST_USAGE: "False" # disale usage logging + TF_CPP_MIN_LOG_LEVEL: "1" # disable INFO logging + ipc: "host" + shm_size: '200mb' + deploy: + resources: + reservations: + devices: + - capabilities: [gpu] diff --git a/docker/dev.tf.Dockerfile b/docker/dev.tf.Dockerfile new file mode 100644 index 000000000..e1615813c --- /dev/null +++ b/docker/dev.tf.Dockerfile @@ -0,0 +1,74 @@ +FROM nvcr.io/nvidia/merlin/merlin-tensorflow:22.12 + +# ----------------------------------------------------------------------------- +# Install system packages +# ----------------------------------------------------------------------------- + +RUN apt-get update +RUN apt-get install -y \ + graphviz `# For visualizing merlin graphs` \ + sudo `# for installing deps from git repos` + +# Update pip +RUN pip install --upgrade \ + pip \ + virtualenv + +# remove default installation of merlin packages +RUN pip uninstall -y \ + merlin-core \ + merlin-models \ + merlin-systems \ + nvtabular \ + transformers4rec \ + ipython \ + jupyter + +# ----------------------------------------------------------------------------- +# Setup User +# ----------------------------------------------------------------------------- + +ARG UID +ARG GID +ARG USER + +RUN addgroup --gid $GID $USER +RUN adduser --disabled-password --gecos '' --uid $UID --gid $GID $USER +RUN adduser $USER sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +USER $USER + +# ----------------------------------------------------------------------------- +# Setup Virtualenv +# ----------------------------------------------------------------------------- + +# The virtual env is required to allow us to install the merlin packages +# in development (editable) mode succesfully + +ENV VIRTUALENV_PATH=/home/$USER/venv +RUN python -m virtualenv $VIRTUALENV_PATH +ENV PATH="${VIRTUALENV_PATH}/bin:${PATH}" + +# Extending the PYTHONPATH so that we can continue to find packages already installed +# e.g 'cudf', 'cuda-python' and others in /usr/local/lib/ +ENV PYTHONPATH="${VIRTUALENV_PATH}/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages" + +WORKDIR /workspace/dev + +# install common dev requirements +COPY requirements.in /tmp/requirements.in +RUN python -m pip install -r /tmp/requirements.in + +# ----------------------------------------------------------------------------- +# Install Merlin Repos +# ----------------------------------------------------------------------------- + +COPY --chown=$USER merlin merlin + +# # # install merlin packages in development mode +RUN python -m pip install -e merlin/models --no-deps +RUN python -m pip install -e merlin/core --no-deps +RUN python -m pip install -e merlin/NVTabular --no-deps +RUN python -m pip install -e merlin/systems --no-deps +RUN python -m pip install -e merlin/dataloader --no-deps diff --git a/requirements.in b/requirements.in new file mode 100644 index 000000000..e69de29bb From 34169b9a15e571a71f04bdf8a6a1d7391e90d686 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 15 Feb 2023 10:05:39 +0000 Subject: [PATCH 3/9] Add ipython and ipdb to requirements.in --- requirements.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.in b/requirements.in index e69de29bb..fe96c42d1 100644 --- a/requirements.in +++ b/requirements.in @@ -0,0 +1,2 @@ +ipython +ipdb From 110543f4574209f0389e84315beffab2cee3506b Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 15 Feb 2023 10:07:00 +0000 Subject: [PATCH 4/9] Move dev requirements file --- docker/dev.tf.Dockerfile | 4 ++-- requirements.in => requirements/dev.txt | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename requirements.in => requirements/dev.txt (100%) diff --git a/docker/dev.tf.Dockerfile b/docker/dev.tf.Dockerfile index e1615813c..bed280fb4 100644 --- a/docker/dev.tf.Dockerfile +++ b/docker/dev.tf.Dockerfile @@ -57,8 +57,8 @@ ENV PYTHONPATH="${VIRTUALENV_PATH}/lib/python3.8/site-packages:/usr/local/lib/py WORKDIR /workspace/dev # install common dev requirements -COPY requirements.in /tmp/requirements.in -RUN python -m pip install -r /tmp/requirements.in +COPY requirements/dev.txt /tmp/requirements.txt +RUN python -m pip install -r /tmp/requirements.txt # ----------------------------------------------------------------------------- # Install Merlin Repos diff --git a/requirements.in b/requirements/dev.txt similarity index 100% rename from requirements.in rename to requirements/dev.txt From 3844a0a0add755e96cb13a74898c0cc86436a9a6 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 15 Feb 2023 10:07:33 +0000 Subject: [PATCH 5/9] Add `.bash_history` and `.ipython/` to `.gitignore` --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 1f1e69f72..8fa9c4004 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ __pycache__/ /.*_checkpoints/ .ipynb_checkpoints/* */.jupyter/* + +.bash_history +.ipython/ \ No newline at end of file From fe8757310a555de2c93df55d9996f30685895a18 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 15 Feb 2023 11:49:15 +0000 Subject: [PATCH 6/9] Change workdir to /workspace --- docker-compose.yml | 2 +- docker/dev.tf.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 14a137138..30df81582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - GID - USER volumes: - - .:/workspace/dev + - .:/workspace - /dev/shm:/dev/shm # required for memory mapping to work as non-root user ports: - "8888:8888" diff --git a/docker/dev.tf.Dockerfile b/docker/dev.tf.Dockerfile index bed280fb4..8652a3bfe 100644 --- a/docker/dev.tf.Dockerfile +++ b/docker/dev.tf.Dockerfile @@ -54,7 +54,7 @@ ENV PATH="${VIRTUALENV_PATH}/bin:${PATH}" # e.g 'cudf', 'cuda-python' and others in /usr/local/lib/ ENV PYTHONPATH="${VIRTUALENV_PATH}/lib/python3.8/site-packages:/usr/local/lib/python3.8/dist-packages" -WORKDIR /workspace/dev +WORKDIR /workspace # install common dev requirements COPY requirements/dev.txt /tmp/requirements.txt From f6cd776a2100524a70873528219ae8d746373f1d Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Wed, 15 Feb 2023 11:49:51 +0000 Subject: [PATCH 7/9] Move base dir location --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 30df81582..ac34769b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: environment: HISTFILE: "/workspace/.bash_history" IPYTHONDIR: "/workspace/.ipython" - BASE_DIR: "/workspace/data/base" + BASE_DIR: "/workspace/base" FEAST_USAGE: "False" # disale usage logging TF_CPP_MIN_LOG_LEVEL: "1" # disable INFO logging ipc: "host" From 999352ff2e9069ccc26df25cfa755edf07a0a5b9 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Thu, 16 Feb 2023 18:46:34 +0000 Subject: [PATCH 8/9] Use useradd instead of adduser --- docker/dev.tf.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/dev.tf.Dockerfile b/docker/dev.tf.Dockerfile index 8652a3bfe..5978f9f0e 100644 --- a/docker/dev.tf.Dockerfile +++ b/docker/dev.tf.Dockerfile @@ -33,8 +33,8 @@ ARG GID ARG USER RUN addgroup --gid $GID $USER -RUN adduser --disabled-password --gecos '' --uid $UID --gid $GID $USER -RUN adduser $USER sudo +RUN useradd --no-log-init --no-user-group --create-home --home-dir /home/$USER --password '' --uid $UID --gid $GID $USER +RUN usermod -a -G sudo $USER RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers USER $USER From b17836f43ccf75663f30f5a64d10c47e02ff9b61 Mon Sep 17 00:00:00 2001 From: Oliver Holworthy Date: Thu, 9 Mar 2023 13:37:25 +0000 Subject: [PATCH 9/9] Add requirements files and dev.cpu and dev.gpu Dockerfiles --- docker-compose.yml | 42 +- docker/dev.cpu.Dockerfile | 44 ++ docker/dev.gpu.Dockerfile | 63 +++ docker/requirements.Dockerfile | 3 + requirements/compile.sh | 3 + requirements/dev.cpu.in | 9 + requirements/dev.cpu.txt | 972 ++++++++++++++++++++++++++++++++ requirements/dev.gpu.in | 2 + requirements/dev.gpu.txt | 989 +++++++++++++++++++++++++++++++++ requirements/dev.txt | 2 - requirements/gpu.in | 4 + requirements/merlin-base.in | 2 + requirements/merlin-dev.in | 6 + 13 files changed, 2129 insertions(+), 12 deletions(-) create mode 100644 docker/dev.cpu.Dockerfile create mode 100644 docker/dev.gpu.Dockerfile create mode 100644 docker/requirements.Dockerfile create mode 100755 requirements/compile.sh create mode 100644 requirements/dev.cpu.in create mode 100644 requirements/dev.cpu.txt create mode 100644 requirements/dev.gpu.in create mode 100644 requirements/dev.gpu.txt delete mode 100644 requirements/dev.txt create mode 100644 requirements/gpu.in create mode 100644 requirements/merlin-base.in create mode 100644 requirements/merlin-dev.in diff --git a/docker-compose.yml b/docker-compose.yml index ac34769b5..e3e074d70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,20 +1,34 @@ --- services: - dev-tf: + dev-cpu: build: context: . - dockerfile: docker/dev.tf.Dockerfile - args: - - UID - - GID - - USER + dockerfile: docker/dev.cpu.Dockerfile volumes: - .:/workspace - - /dev/shm:/dev/shm # required for memory mapping to work as non-root user + working_dir: "/workspace" ports: - "8888:8888" - command: "jupyter-nbclassic --ip 0.0.0.0 --allow-root" + command: "jupyter notebook --ip 0.0.0.0 --allow-root" + environment: + HISTFILE: "/workspace/.bash_history" + IPYTHONDIR: "/workspace/.ipython" + BASE_DIR: "/workspace/base" + FEAST_USAGE: "False" # disale usage logging + TF_CPP_MIN_LOG_LEVEL: "1" # disable INFO logging + shm_size: '1gb' + + dev-gpu: + build: + context: . + dockerfile: docker/dev.gpu.Dockerfile + volumes: + - .:/workspace + working_dir: "/workspace" + ports: + - "8888:8888" + command: "jupyter notebook --ip 0.0.0.0 --allow-root" entrypoint: "/opt/nvidia/nvidia_entrypoint.sh" environment: HISTFILE: "/workspace/.bash_history" @@ -22,10 +36,18 @@ services: BASE_DIR: "/workspace/base" FEAST_USAGE: "False" # disale usage logging TF_CPP_MIN_LOG_LEVEL: "1" # disable INFO logging - ipc: "host" - shm_size: '200mb' + shm_size: '1gb' deploy: resources: reservations: devices: - capabilities: [gpu] + + requirements: + build: + context: . + dockerfile: docker/requirements.Dockerfile + volumes: + - ./requirements:/workspace + - ./merlin:/workspace/merlin + working_dir: "/workspace" diff --git a/docker/dev.cpu.Dockerfile b/docker/dev.cpu.Dockerfile new file mode 100644 index 000000000..b424580a3 --- /dev/null +++ b/docker/dev.cpu.Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.8.16-slim-buster + +RUN apt-get update && apt-get install -y \ + curl \ + git \ + build-essential `For building NVTabular C++ code` \ + graphviz `# For visualizing merlin graphs` \ + ripgrep + +# Update pip +RUN pip install --no-cache-dir --upgrade \ + pip \ + virtualenv + +# ----------------------------------------------------------------------------- +# Setup Virtualenv +# ----------------------------------------------------------------------------- + +# The virtual env is required to allow us to install the merlin packages +# in development (editable) mode succesfully + +ENV VIRTUALENV_PATH=/venv +RUN python -m virtualenv $VIRTUALENV_PATH +ENV PATH="${VIRTUALENV_PATH}/bin:${PATH}" + +WORKDIR /workspace + +# install common dev requirements +COPY requirements/dev.cpu.txt /tmp/requirements.txt +RUN python -m pip install --no-cache-dir -r /tmp/requirements.txt + +# ----------------------------------------------------------------------------- +# Install Merlin Repos +# ----------------------------------------------------------------------------- + +COPY merlin merlin + +# install merlin packages in development mode +RUN python -m pip install -e merlin/models --no-deps +RUN python -m pip install -e merlin/core --no-deps +RUN python -m pip install -e merlin/NVTabular --no-deps +RUN python -m pip install -e merlin/systems --no-deps +RUN python -m pip install -e merlin/dataloader --no-deps +RUN python -m pip install -e merlin/Transformers4Rec --no-deps diff --git a/docker/dev.gpu.Dockerfile b/docker/dev.gpu.Dockerfile new file mode 100644 index 000000000..c53ea2b0a --- /dev/null +++ b/docker/dev.gpu.Dockerfile @@ -0,0 +1,63 @@ +FROM nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04 + +# ----------------------------------------------------------------------------- +# Install system packages +# ----------------------------------------------------------------------------- + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + curl \ + git \ + software-properties-common \ + graphviz `# For visualizing merlin graphs` \ + libcudnn8 \ + ripgrep + +RUN add-apt-repository ppa:deadsnakes/ppa -y + +RUN apt-get update && TZ=Etc/UTC apt install -y \ + python3.8 \ + python3.8-dev \ + python3.8-distutils + +RUN ln -s /usr/bin/python3.8 /usr/bin/python + +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py + +# Update pip +RUN pip install --no-cache-dir --upgrade \ + pip \ + virtualenv + +# ----------------------------------------------------------------------------- +# Setup Virtualenv +# ----------------------------------------------------------------------------- + +# The virtual env is required to allow us to install the merlin packages +# in development (editable) mode succesfully + +ENV VIRTUALENV_PATH=/venv +RUN python -m virtualenv $VIRTUALENV_PATH +ENV PATH="${VIRTUALENV_PATH}/bin:${PATH}" + +WORKDIR /workspace + +# install common dev requirements +COPY requirements/dev.gpu.txt /tmp/requirements.txt +RUN python -m pip install --no-cache-dir -r /tmp/requirements.txt + +# ----------------------------------------------------------------------------- +# Install Merlin Repos +# ----------------------------------------------------------------------------- + +COPY merlin merlin + +# install merlin packages in development mode +RUN python -m pip install -e merlin/models --no-deps +RUN python -m pip install -e merlin/core --no-deps +RUN python -m pip install -e merlin/NVTabular --no-deps +RUN python -m pip install -e merlin/systems --no-deps +RUN python -m pip install -e merlin/dataloader --no-deps +RUN python -m pip install -e merlin/Transformers4Rec --no-deps diff --git a/docker/requirements.Dockerfile b/docker/requirements.Dockerfile new file mode 100644 index 000000000..69f0604fa --- /dev/null +++ b/docker/requirements.Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.8.16-slim-buster +RUN apt-get update && apt-get install -y git +RUN pip install --upgrade pip && pip install pip-tools==6.12.3 diff --git a/requirements/compile.sh b/requirements/compile.sh new file mode 100755 index 000000000..9263514c7 --- /dev/null +++ b/requirements/compile.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker compose run --rm requirements sh -c "pip-compile --output-file=$1.txt --resolver=backtracking $1.in" diff --git a/requirements/dev.cpu.in b/requirements/dev.cpu.in new file mode 100644 index 000000000..1c47efc7c --- /dev/null +++ b/requirements/dev.cpu.in @@ -0,0 +1,9 @@ +-r merlin-base.in +-r merlin-dev.in + +# conflict between TensorFlow and cudf +protobuf==3.20.3 + +scipy +tensorflow +torch diff --git a/requirements/dev.cpu.txt b/requirements/dev.cpu.txt new file mode 100644 index 000000000..f00521f27 --- /dev/null +++ b/requirements/dev.cpu.txt @@ -0,0 +1,972 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --output-file=dev.cpu.txt --resolver=backtracking dev.cpu.in +# +absl-py==1.4.0 + # via + # tensorboard + # tensorflow + # tensorflow-metadata +aiohttp==3.8.4 + # via tritonclient +aiosignal==1.3.1 + # via aiohttp +alabaster==0.7.13 + # via sphinx +alembic==1.10.1 + # via optuna +anyio==3.6.2 + # via jupyter-server +appdirs==1.4.4 + # via + # black + # wandb +argon2-cffi==21.3.0 + # via + # jupyter-server + # nbclassic + # notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.2.3 + # via isoduration +asttokens==2.2.1 + # via stack-data +astunparse==1.6.3 + # via tensorflow +async-timeout==4.0.2 + # via aiohttp +attrs==21.4.0 + # via + # aiohttp + # interrogate + # jsonschema + # jupyter-cache + # markdown-it-py + # pytest +babel==2.12.1 + # via sphinx +backcall==0.2.0 + # via ipython +bandit==1.7.4 + # via -r merlin/Transformers4Rec/requirements/dev.txt +beautifulsoup4==4.11.2 + # via nbconvert +betterproto==1.2.5 + # via -r merlin/core/requirements.txt +black==20.8b1 + # via -r merlin/models/requirements/dev.txt +bleach==6.0.0 + # via nbconvert +bokeh==3.0.3 + # via -r merlin/models/requirements/dev.txt +boto3==1.26.85 + # via moto +botocore==1.29.85 + # via + # boto3 + # moto + # s3transfer +brotli==1.0.9 + # via geventhttpclient +build==0.10.0 + # via check-manifest +cachetools==5.3.0 + # via google-auth +certifi==2022.12.7 + # via + # geventhttpclient + # requests + # sentry-sdk +cffi==1.15.1 + # via + # argon2-cffi-bindings + # cryptography +charset-normalizer==3.1.0 + # via + # aiohttp + # requests +check-manifest==0.49 + # via + # -r merlin/Transformers4Rec/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +click==8.1.3 + # via + # black + # dask + # distributed + # interrogate + # sacremoses + # sphinx-external-toc + # wandb +cloudpickle==2.2.1 + # via + # dask + # distributed +cmaes==0.9.1 + # via optuna +colorama==0.4.6 + # via + # interrogate + # nbdime +colorlog==6.7.0 + # via optuna +comm==0.1.2 + # via ipykernel +contourpy==1.0.7 + # via bokeh +coverage[toml]==7.2.1 + # via pytest-cov +cpplint==1.6.1 + # via + # -r merlin/NVTabular/requirements/dev.txt + # -r merlin/dataloader/requirements/dev.txt +cryptography==39.0.2 + # via moto +dask==2023.3.0 + # via + # -r merlin/core/requirements.txt + # distributed +debugpy==1.6.6 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +distributed==2023.3.0 + # via -r merlin/core/requirements.txt +docker-pycreds==0.4.0 + # via wandb +docutils==0.16 + # via + # myst-nb + # myst-parser + # nbsphinx + # sphinx + # sphinx-rtd-theme + # sphinx-togglebutton +entrypoints==0.4 + # via nbconvert +exceptiongroup==1.1.0 + # via pytest +execnet==1.9.0 + # via pytest-xdist +executing==1.2.0 + # via stack-data +fastjsonschema==2.16.3 + # via nbformat +fiddle==0.2.2 + # via -r merlin/models/requirements/dev.txt +filelock==3.9.0 + # via + # huggingface-hub + # transformers +flake8==6.0.0 + # via -r merlin/models/requirements/dev.txt +flatbuffers==1.12 + # via tensorflow +fqdn==1.5.1 + # via jsonschema +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2022.5.0 + # via + # -r merlin/core/requirements.txt + # dask +gast==0.4.0 + # via tensorflow +gevent==22.10.2 + # via geventhttpclient +geventhttpclient==2.0.2 + # via tritonclient +gitdb==4.0.10 + # via gitpython +gitpython==3.1.31 + # via + # bandit + # nbdime + # wandb +google-auth==2.16.2 + # via + # google-auth-oauthlib + # tensorboard +google-auth-oauthlib==0.4.6 + # via tensorboard +google-pasta==0.2.0 + # via tensorflow +googleapis-common-protos==1.58.0 + # via tensorflow-metadata +greenlet==2.0.2 + # via + # gevent + # sqlalchemy +grpcio==1.51.3 + # via + # tensorboard + # tensorflow + # tritonclient +grpclib==0.4.3 + # via betterproto +h2==4.1.0 + # via grpclib +h5py==3.8.0 + # via tensorflow +heapdict==1.0.1 + # via zict +hpack==4.0.0 + # via h2 +huggingface-hub==0.12.1 + # via transformers +hyperframe==6.0.1 + # via h2 +idna==3.4 + # via + # anyio + # jsonschema + # requests + # yarl +imagesize==1.4.1 + # via sphinx +importlib-metadata==6.0.0 + # via + # alembic + # jupyter-client + # markdown + # myst-nb + # numba +importlib-resources==5.12.0 + # via + # alembic + # jsonschema +iniconfig==2.0.0 + # via pytest +interrogate==1.5.0 + # via -r merlin/models/requirements/dev.txt +ipykernel==6.21.3 + # via + # ipywidgets + # jupyter + # jupyter-console + # nbclassic + # notebook + # qtconsole +ipython==8.11.0 + # via + # ipykernel + # ipywidgets + # jupyter-console + # jupyter-sphinx + # myst-nb +ipython-genutils==0.2.0 + # via + # -r merlin/dataloader/requirements/dev.txt + # ipywidgets + # nbclassic + # notebook + # qtconsole +ipywidgets==7.7.3 + # via + # jupyter + # jupyter-sphinx + # myst-nb +isoduration==20.11.0 + # via jsonschema +isort==5.12.0 + # via -r merlin/models/requirements/dev.txt +jedi==0.18.2 + # via ipython +jinja2==3.0.3 + # via + # -r merlin/dataloader/requirements/dev.txt + # bokeh + # distributed + # jupyter-server + # moto + # myst-parser + # nbclassic + # nbconvert + # nbdime + # nbsphinx + # notebook + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +joblib==1.2.0 + # via + # sacremoses + # scikit-learn +jsonpointer==2.3 + # via jsonschema +jsonschema[format-nongpl]==4.17.3 + # via + # jupyter-events + # nbformat +jupyter==1.0.0 + # via -r merlin/models/requirements/dev.txt +jupyter-cache==0.4.3 + # via myst-nb +jupyter-client==8.0.3 + # via + # ipykernel + # jupyter-console + # jupyter-server + # nbclassic + # nbclient + # notebook + # qtconsole +jupyter-console==6.6.3 + # via jupyter +jupyter-core==5.2.0 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # nbclassic + # nbconvert + # nbformat + # notebook + # qtconsole +jupyter-events==0.6.3 + # via jupyter-server +jupyter-server==2.4.0 + # via + # jupyter-server-mathjax + # nbclassic + # nbdime + # notebook-shim +jupyter-server-mathjax==0.2.6 + # via nbdime +jupyter-server-terminals==0.4.4 + # via jupyter-server +jupyter-sphinx==0.3.2 + # via myst-nb +jupyterlab-pygments==0.2.2 + # via nbconvert +jupyterlab-widgets==1.1.2 + # via ipywidgets +keras==2.9.0 + # via tensorflow +keras-preprocessing==1.1.2 + # via tensorflow +libclang==15.0.6.1 + # via tensorflow +libcst==0.4.9 + # via fiddle +linkify-it-py==1.0.3 + # via -r merlin/dataloader/requirements/dev.txt +llvmlite==0.39.1 + # via numba +locket==1.0.0 + # via + # distributed + # partd +lxml==4.9.2 + # via nbconvert +mako==1.2.4 + # via alembic +markdown==3.3.7 + # via + # -r merlin/dataloader/requirements/dev.txt + # sphinx-markdown-tables + # tensorboard +markdown-it-py==1.1.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.0.1 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt + # jinja2 + # mako + # nbconvert +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +mccabe==0.7.0 + # via flake8 +mdit-py-plugins==0.2.8 + # via myst-parser +mistune==0.8.4 + # via nbconvert +moto==4.1.4 + # via -r merlin/dataloader/requirements/dev.txt +msgpack==1.0.4 + # via distributed +multidict==6.0.4 + # via + # aiohttp + # grpclib + # yarl +mypy==1.1.1 + # via -r merlin/models/requirements/dev.txt +mypy-extensions==1.0.0 + # via + # black + # mypy + # typing-inspect +myst-nb==0.13.2 + # via -r merlin/dataloader/requirements/dev.txt +myst-parser==0.15.2 + # via myst-nb +natsort==8.1.0 + # via -r merlin/dataloader/requirements/dev.txt +nbclassic==0.5.3 + # via notebook +nbclient==0.5.13 + # via + # jupyter-cache + # nbconvert + # testbook +nbconvert==6.5.4 + # via + # jupyter + # jupyter-server + # jupyter-sphinx + # myst-nb + # nbclassic + # nbsphinx + # notebook +nbdime==3.1.1 + # via jupyter-cache +nbformat==5.7.3 + # via + # jupyter-cache + # jupyter-server + # jupyter-sphinx + # myst-nb + # nbclassic + # nbclient + # nbconvert + # nbdime + # nbsphinx + # notebook + # testbook +nbsphinx==0.8.12 + # via -r merlin/dataloader/requirements/dev.txt +nest-asyncio==1.5.6 + # via + # ipykernel + # nbclassic + # nbclient + # notebook +notebook==6.5.3 + # via + # jupyter + # widgetsnbextension +notebook-shim==0.2.2 + # via nbclassic +npy-append-array==0.9.16 + # via -r merlin/dataloader/requirements/dev.txt +numba==0.56.4 + # via -r merlin/core/requirements.txt +numpy==1.23.5 + # via + # bokeh + # cmaes + # contourpy + # h5py + # keras-preprocessing + # numba + # opt-einsum + # optuna + # pandas + # pyarrow + # scikit-learn + # scipy + # tensorboard + # tensorflow + # transformers + # tritonclient +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +oauthlib==3.2.2 + # via requests-oauthlib +opt-einsum==3.3.0 + # via tensorflow +optuna==3.1.0 + # via -r merlin/models/requirements/dev.txt +packaging==23.0 + # via + # -r merlin/core/requirements.txt + # bokeh + # build + # dask + # distributed + # huggingface-hub + # ipykernel + # jupyter-server + # nbconvert + # optuna + # pytest + # qtpy + # sphinx + # tensorflow + # transformers + # tritonclient +pandas==1.5.3 + # via + # -r merlin/core/requirements.txt + # bokeh +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi +partd==1.3.0 + # via dask +pathspec==0.11.0 + # via black +pathtools==0.1.2 + # via wandb +pbr==5.11.1 + # via + # sphinxcontrib-copydirs + # stevedore +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pillow==9.4.0 + # via bokeh +pkgutil-resolve-name==1.3.10 + # via jsonschema +platformdirs==3.1.0 + # via jupyter-core +plotly==5.13.1 + # via -r merlin/models/requirements/dev.txt +pluggy==1.0.0 + # via pytest +prometheus-client==0.16.0 + # via + # jupyter-server + # nbclassic + # notebook +prompt-toolkit==3.0.38 + # via + # ipython + # jupyter-console +protobuf==3.20.3 + # via + # -r dev.cpu.in + # -r merlin/core/requirements.txt + # googleapis-common-protos + # tensorboard + # tensorflow + # tensorflow-metadata + # tritonclient + # wandb +psutil==5.9.4 + # via + # distributed + # ipykernel + # wandb +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via interrogate +pyarrow==11.0.0 + # via + # -r merlin/Transformers4Rec/requirements/base_external.txt + # -r merlin/core/requirements.txt +pyasn1==0.4.8 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via google-auth +pycodestyle==2.10.0 + # via flake8 +pycparser==2.21 + # via cffi +pyflakes==3.0.1 + # via flake8 +pygments==2.14.0 + # via + # ipython + # jupyter-console + # nbconvert + # nbdime + # qtconsole + # sphinx +pynvml==11.5.0 + # via -r merlin/core/requirements.txt +pyproject-hooks==1.0.0 + # via build +pyrsistent==0.19.3 + # via jsonschema +pytest==7.2.2 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt + # pytest-cov + # pytest-xdist +pytest-cov==4.0.0 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +pytest-xdist==3.2.0 + # via + # -r merlin/core/requirements-dev.txt + # -r merlin/models/requirements/dev.txt +python-dateutil==2.8.2 + # via + # arrow + # botocore + # jupyter-client + # moto + # pandas +python-json-logger==2.0.7 + # via jupyter-events +python-rapidjson==1.9 + # via tritonclient +pytz==2022.7.1 + # via + # babel + # pandas +pyyaml==6.0 + # via + # bandit + # bokeh + # dask + # distributed + # huggingface-hub + # jupyter-events + # libcst + # myst-nb + # myst-parser + # optuna + # sphinx-external-toc + # transformers + # wandb +pyzmq==25.0.0 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # nbclassic + # notebook + # qtconsole +qtconsole==5.4.0 + # via jupyter +qtpy==2.3.0 + # via qtconsole +regex==2022.10.31 + # via + # black + # sacremoses + # transformers +requests==2.28.2 + # via + # huggingface-hub + # moto + # nbdime + # requests-oauthlib + # responses + # sphinx + # tensorboard + # transformers + # wandb +requests-oauthlib==1.3.1 + # via google-auth-oauthlib +responses==0.22.0 + # via moto +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +rsa==4.9 + # via google-auth +s3transfer==0.6.0 + # via boto3 +sacremoses==0.0.53 + # via transformers +scikit-learn==1.2.1 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +scipy==1.10.1 + # via + # -r dev.cpu.in + # -r merlin/dataloader/requirements/dev.txt + # scikit-learn +send2trash==1.8.0 + # via + # jupyter-server + # nbclassic + # notebook +sentry-sdk==1.16.0 + # via wandb +setproctitle==1.3.2 + # via wandb +six==1.16.0 + # via + # asttokens + # astunparse + # bleach + # docker-pycreds + # geventhttpclient + # google-auth + # google-pasta + # keras-preprocessing + # python-dateutil + # rfc3339-validator + # sacremoses + # tensorflow +smmap==5.0.0 + # via gitdb +sniffio==1.3.0 + # via anyio +snowballstemmer==2.2.0 + # via sphinx +sortedcontainers==2.4.0 + # via distributed +soupsieve==2.4 + # via beautifulsoup4 +sphinx==3.5.4 + # via + # -r merlin/dataloader/requirements/dev.txt + # jupyter-sphinx + # myst-nb + # myst-parser + # nbsphinx + # sphinx-external-toc + # sphinx-multiversion + # sphinx-rtd-theme + # sphinx-togglebutton +sphinx-external-toc==0.3.0 + # via -r merlin/dataloader/requirements/dev.txt +sphinx-markdown-tables==0.0.15 + # via -r merlin/dataloader/requirements/dev.txt +sphinx-multiversion @ git+https://github.com/mikemckiernan/sphinx-multiversion.git + # via -r merlin/dataloader/requirements/dev.txt +sphinx-rtd-theme==1.2.0 + # via -r merlin/dataloader/requirements/dev.txt +sphinx-togglebutton==0.3.2 + # via myst-nb +sphinxcontrib-applehelp==1.0.4 + # via sphinx +sphinxcontrib-copydirs @ git+https://github.com/mikemckiernan/sphinxcontrib-copydirs.git + # via -r merlin/dataloader/requirements/dev.txt +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.1 + # via sphinx +sphinxcontrib-jquery==2.0.0 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +sqlalchemy==1.4.46 + # via + # alembic + # jupyter-cache + # optuna +stack-data==0.6.2 + # via ipython +stevedore==5.0.0 + # via bandit +stringcase==1.2.0 + # via betterproto +tabulate==0.9.0 + # via interrogate +tblib==1.7.0 + # via distributed +tenacity==8.2.2 + # via plotly +tensorboard==2.9.0 + # via tensorflow +tensorboard-data-server==0.6.1 + # via tensorboard +tensorboard-plugin-wit==1.8.1 + # via tensorboard +tensorflow==2.9.0 + # via -r dev.cpu.in +tensorflow-estimator==2.9.0 + # via tensorflow +tensorflow-io-gcs-filesystem==0.31.0 + # via tensorflow +tensorflow-metadata==1.12.0 + # via -r merlin/core/requirements.txt +termcolor==2.2.0 + # via tensorflow +terminado==0.17.1 + # via + # jupyter-server + # jupyter-server-terminals + # nbclassic + # notebook +testbook==0.4.2 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +threadpoolctl==3.1.0 + # via scikit-learn +tinycss2==1.2.1 + # via nbconvert +tokenizers==0.12.1 + # via transformers +toml==0.10.2 + # via + # black + # interrogate + # responses +tomli==2.0.1 + # via + # build + # check-manifest + # coverage + # mypy + # pyproject-hooks + # pytest +toolz==0.12.0 + # via + # dask + # distributed + # partd +torch==1.13.1 + # via -r dev.cpu.in +tornado==6.2 + # via + # bokeh + # distributed + # ipykernel + # jupyter-client + # jupyter-server + # nbclassic + # nbdime + # notebook + # terminado +tqdm==4.65.0 + # via + # -r merlin/Transformers4Rec/requirements/base_external.txt + # -r merlin/core/requirements.txt + # huggingface-hub + # optuna + # sacremoses + # transformers +traitlets==5.9.0 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-console + # jupyter-core + # jupyter-events + # jupyter-server + # matplotlib-inline + # nbclassic + # nbclient + # nbconvert + # nbformat + # nbsphinx + # notebook + # qtconsole +transformers==4.18.0 + # via -r merlin/Transformers4Rec/requirements/base_external.txt +tritonclient[all]==2.31.0 + # via -r merlin/systems/requirements/dev.txt +typed-ast==1.5.4 + # via black +types-toml==0.10.8.5 + # via responses +typing-extensions==4.5.0 + # via + # alembic + # black + # fiddle + # huggingface-hub + # libcst + # mypy + # tensorflow + # torch + # typing-inspect + # wandb +typing-inspect==0.8.0 + # via libcst +uc-micro-py==1.0.1 + # via linkify-it-py +uri-template==1.2.0 + # via jsonschema +urllib3==1.26.14 + # via + # botocore + # distributed + # requests + # responses + # sentry-sdk +wandb==0.13.10 + # via -r merlin/models/requirements/dev.txt +wcwidth==0.2.6 + # via prompt-toolkit +webcolors==1.12 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.5.1 + # via jupyter-server +werkzeug==2.1.2 + # via + # moto + # tensorboard +wheel==0.38.4 + # via + # astunparse + # nvidia-cublas-cu11 + # nvidia-cuda-runtime-cu11 + # sphinx-togglebutton + # tensorboard +widgetsnbextension==3.6.2 + # via ipywidgets +wrapt==1.15.0 + # via tensorflow +xmltodict==0.13.0 + # via moto +xyzservices==2023.2.0 + # via bokeh +yarl==1.8.2 + # via aiohttp +zict==2.2.0 + # via distributed +zipp==3.15.0 + # via + # importlib-metadata + # importlib-resources +zope-event==4.6 + # via gevent +zope-interface==5.5.2 + # via gevent + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/dev.gpu.in b/requirements/dev.gpu.in new file mode 100644 index 000000000..e1a96685d --- /dev/null +++ b/requirements/dev.gpu.in @@ -0,0 +1,2 @@ +-r dev.cpu.in +-r gpu.in diff --git a/requirements/dev.gpu.txt b/requirements/dev.gpu.txt new file mode 100644 index 000000000..24647995a --- /dev/null +++ b/requirements/dev.gpu.txt @@ -0,0 +1,989 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --output-file=dev.gpu.txt --resolver=backtracking dev.gpu.in +# +--extra-index-url https://pypi.nvidia.com + +absl-py==1.4.0 + # via + # tensorboard + # tensorflow + # tensorflow-metadata +aiohttp==3.8.4 + # via tritonclient +aiosignal==1.3.1 + # via aiohttp +alabaster==0.7.13 + # via sphinx +alembic==1.10.1 + # via optuna +anyio==3.6.2 + # via jupyter-server +appdirs==1.4.4 + # via + # black + # wandb +argon2-cffi==21.3.0 + # via + # jupyter-server + # nbclassic + # notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +asttokens==2.2.1 + # via stack-data +astunparse==1.6.3 + # via tensorflow +async-timeout==4.0.2 + # via aiohttp +attrs==21.4.0 + # via + # aiohttp + # interrogate + # jsonschema + # jupyter-cache + # markdown-it-py + # pytest +babel==2.12.1 + # via sphinx +backcall==0.2.0 + # via ipython +bandit==1.7.4 + # via -r merlin/Transformers4Rec/requirements/dev.txt +beautifulsoup4==4.11.2 + # via nbconvert +betterproto==1.2.5 + # via -r merlin/core/requirements.txt +black==20.8b1 + # via -r merlin/models/requirements/dev.txt +bleach==6.0.0 + # via nbconvert +bokeh==3.0.3 + # via -r merlin/models/requirements/dev.txt +boto3==1.26.85 + # via moto +botocore==1.29.85 + # via + # boto3 + # moto + # s3transfer +brotli==1.0.9 + # via geventhttpclient +build==0.10.0 + # via check-manifest +cachetools==5.3.0 + # via + # cudf-cu11 + # google-auth +certifi==2022.12.7 + # via + # geventhttpclient + # requests + # sentry-sdk +cffi==1.15.1 + # via + # argon2-cffi-bindings + # cryptography +charset-normalizer==3.1.0 + # via + # aiohttp + # requests +check-manifest==0.49 + # via + # -r merlin/Transformers4Rec/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +click==8.1.3 + # via + # black + # distributed + # interrogate + # sacremoses + # sphinx-external-toc + # wandb +cloudpickle==2.2.1 + # via + # dask + # distributed +cmaes==0.9.1 + # via optuna +colorama==0.4.6 + # via + # interrogate + # nbdime +colorlog==6.7.0 + # via optuna +comm==0.1.2 + # via ipykernel +contourpy==1.0.7 + # via bokeh +coverage[toml]==7.2.1 + # via pytest-cov +cpplint==1.6.1 + # via + # -r merlin/NVTabular/requirements/dev.txt + # -r merlin/dataloader/requirements/dev.txt +cryptography==39.0.2 + # via moto +cubinlinker-cu11==0.3.0.post1 + # via cudf-cu11 +cuda-python==11.7.0 + # via + # cudf-cu11 + # rmm-cu11 +cudf-cu11==22.10.0 + # via + # -r gpu.in + # dask-cudf-cu11 +cupy-cuda115==10.6.0 + # via + # cudf-cu11 + # dask-cudf-cu11 +cython==0.29.33 + # via cuda-python +dask==2022.9.2 + # via + # -r merlin/core/requirements.txt + # dask-cudf-cu11 + # distributed +dask-cudf-cu11==22.10.0 + # via -r gpu.in +debugpy==1.6.6 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +distributed==2022.9.2 + # via + # -r merlin/core/requirements.txt + # dask-cudf-cu11 +docker-pycreds==0.4.0 + # via wandb +docutils==0.16 + # via + # myst-nb + # myst-parser + # nbsphinx + # sphinx + # sphinx-rtd-theme + # sphinx-togglebutton +entrypoints==0.4 + # via + # jupyter-client + # nbconvert +exceptiongroup==1.1.0 + # via pytest +execnet==1.9.0 + # via pytest-xdist +executing==1.2.0 + # via stack-data +fastjsonschema==2.16.3 + # via nbformat +fastrlock==0.8.1 + # via cupy-cuda115 +fiddle==0.2.2 + # via -r merlin/models/requirements/dev.txt +filelock==3.9.0 + # via + # huggingface-hub + # transformers +flake8==6.0.0 + # via -r merlin/models/requirements/dev.txt +flatbuffers==1.12 + # via tensorflow +frozenlist==1.3.3 + # via + # aiohttp + # aiosignal +fsspec==2022.5.0 + # via + # -r merlin/core/requirements.txt + # cudf-cu11 + # dask + # dask-cudf-cu11 +gast==0.4.0 + # via tensorflow +gevent==22.10.2 + # via geventhttpclient +geventhttpclient==2.0.2 + # via tritonclient +gitdb==4.0.10 + # via gitpython +gitpython==3.1.31 + # via + # bandit + # nbdime + # wandb +google-auth==2.16.2 + # via + # google-auth-oauthlib + # tensorboard +google-auth-oauthlib==0.4.6 + # via tensorboard +google-pasta==0.2.0 + # via tensorflow +googleapis-common-protos==1.58.0 + # via tensorflow-metadata +greenlet==2.0.2 + # via + # gevent + # sqlalchemy +grpcio==1.51.3 + # via + # tensorboard + # tensorflow + # tritonclient +grpclib==0.4.3 + # via betterproto +h2==4.1.0 + # via grpclib +h5py==3.8.0 + # via tensorflow +heapdict==1.0.1 + # via zict +hpack==4.0.0 + # via h2 +huggingface-hub==0.12.1 + # via transformers +hyperframe==6.0.1 + # via h2 +idna==3.4 + # via + # anyio + # requests + # yarl +imagesize==1.4.1 + # via sphinx +importlib-metadata==6.0.0 + # via + # alembic + # markdown + # myst-nb + # numba +importlib-resources==5.12.0 + # via + # alembic + # jsonschema +iniconfig==2.0.0 + # via pytest +interrogate==1.5.0 + # via -r merlin/models/requirements/dev.txt +ipykernel==6.21.3 + # via + # ipywidgets + # jupyter + # jupyter-console + # nbclassic + # notebook + # qtconsole +ipython==8.11.0 + # via + # ipykernel + # ipywidgets + # jupyter-console + # jupyter-sphinx + # myst-nb +ipython-genutils==0.2.0 + # via + # -r merlin/dataloader/requirements/dev.txt + # ipywidgets + # nbclassic + # notebook + # qtconsole +ipywidgets==7.7.3 + # via + # jupyter + # jupyter-sphinx + # myst-nb +isort==5.12.0 + # via -r merlin/models/requirements/dev.txt +jedi==0.18.2 + # via ipython +jinja2==3.0.3 + # via + # -r merlin/dataloader/requirements/dev.txt + # bokeh + # distributed + # jupyter-server + # moto + # myst-parser + # nbclassic + # nbconvert + # nbdime + # nbsphinx + # notebook + # sphinx +jmespath==1.0.1 + # via + # boto3 + # botocore +joblib==1.2.0 + # via + # sacremoses + # scikit-learn +jsonschema==4.17.3 + # via nbformat +jupyter==1.0.0 + # via -r merlin/models/requirements/dev.txt +jupyter-cache==0.4.3 + # via myst-nb +jupyter-client==7.3.4 + # via + # ipykernel + # jupyter-console + # jupyter-server + # nbclassic + # nbclient + # notebook + # qtconsole +jupyter-console==6.6.3 + # via jupyter +jupyter-core==5.2.0 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # nbclassic + # nbconvert + # nbformat + # notebook + # qtconsole +jupyter-server==1.23.6 + # via + # jupyter-server-mathjax + # nbclassic + # nbdime + # notebook-shim +jupyter-server-mathjax==0.2.6 + # via nbdime +jupyter-sphinx==0.3.2 + # via myst-nb +jupyterlab-pygments==0.2.2 + # via nbconvert +jupyterlab-widgets==1.1.2 + # via ipywidgets +keras==2.9.0 + # via tensorflow +keras-preprocessing==1.1.2 + # via tensorflow +libclang==15.0.6.1 + # via tensorflow +libcst==0.4.9 + # via fiddle +linkify-it-py==1.0.3 + # via -r merlin/dataloader/requirements/dev.txt +llvmlite==0.39.1 + # via numba +locket==1.0.0 + # via + # distributed + # partd +lxml==4.9.2 + # via nbconvert +mako==1.2.4 + # via alembic +markdown==3.3.7 + # via + # -r merlin/dataloader/requirements/dev.txt + # sphinx-markdown-tables + # tensorboard +markdown-it-py==1.1.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.0.1 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt + # jinja2 + # mako + # nbconvert +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +mccabe==0.7.0 + # via flake8 +mdit-py-plugins==0.2.8 + # via myst-parser +mistune==0.8.4 + # via nbconvert +moto==4.1.4 + # via -r merlin/dataloader/requirements/dev.txt +msgpack==1.0.4 + # via distributed +multidict==6.0.4 + # via + # aiohttp + # grpclib + # yarl +mypy==1.1.1 + # via -r merlin/models/requirements/dev.txt +mypy-extensions==1.0.0 + # via + # black + # mypy + # typing-inspect +myst-nb==0.13.2 + # via -r merlin/dataloader/requirements/dev.txt +myst-parser==0.15.2 + # via myst-nb +natsort==8.1.0 + # via -r merlin/dataloader/requirements/dev.txt +nbclassic==0.5.3 + # via notebook +nbclient==0.5.13 + # via + # jupyter-cache + # nbconvert + # testbook +nbconvert==6.5.4 + # via + # jupyter + # jupyter-server + # jupyter-sphinx + # myst-nb + # nbclassic + # nbsphinx + # notebook +nbdime==3.1.1 + # via jupyter-cache +nbformat==5.7.3 + # via + # jupyter-cache + # jupyter-server + # jupyter-sphinx + # myst-nb + # nbclassic + # nbclient + # nbconvert + # nbdime + # nbsphinx + # notebook + # testbook +nbsphinx==0.8.12 + # via -r merlin/dataloader/requirements/dev.txt +nest-asyncio==1.5.6 + # via + # ipykernel + # jupyter-client + # nbclassic + # nbclient + # notebook +notebook==6.5.3 + # via + # jupyter + # widgetsnbextension +notebook-shim==0.2.2 + # via nbclassic +npy-append-array==0.9.16 + # via -r merlin/dataloader/requirements/dev.txt +numba==0.56.4 + # via + # -r merlin/core/requirements.txt + # cudf-cu11 + # rmm-cu11 +numpy==1.23.5 + # via + # bokeh + # cmaes + # contourpy + # cudf-cu11 + # cupy-cuda115 + # dask-cudf-cu11 + # h5py + # keras-preprocessing + # numba + # opt-einsum + # optuna + # pandas + # pyarrow + # rmm-cu11 + # scikit-learn + # scipy + # tensorboard + # tensorflow + # transformers + # tritonclient +nvidia-cublas-cu11==11.10.3.66 + # via + # nvidia-cudnn-cu11 + # torch +nvidia-cuda-nvrtc-cu11==11.7.99 + # via torch +nvidia-cuda-runtime-cu11==11.7.99 + # via torch +nvidia-cudnn-cu11==8.5.0.96 + # via torch +nvtx==0.2.5 + # via cudf-cu11 +oauthlib==3.2.2 + # via requests-oauthlib +opt-einsum==3.3.0 + # via tensorflow +optuna==3.1.0 + # via -r merlin/models/requirements/dev.txt +packaging==23.0 + # via + # -r merlin/core/requirements.txt + # bokeh + # build + # cudf-cu11 + # dask + # distributed + # huggingface-hub + # ipykernel + # jupyter-server + # nbconvert + # optuna + # pytest + # qtpy + # sphinx + # tensorflow + # transformers + # tritonclient +pandas==1.5.3 + # via + # -r merlin/core/requirements.txt + # bokeh + # cudf-cu11 + # dask-cudf-cu11 +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi +partd==1.3.0 + # via dask +pathspec==0.11.0 + # via black +pathtools==0.1.2 + # via wandb +pbr==5.11.1 + # via + # sphinxcontrib-copydirs + # stevedore +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pillow==9.4.0 + # via bokeh +pkgutil-resolve-name==1.3.10 + # via jsonschema +platformdirs==3.1.0 + # via jupyter-core +plotly==5.13.1 + # via -r merlin/models/requirements/dev.txt +pluggy==1.0.0 + # via pytest +prometheus-client==0.16.0 + # via + # jupyter-server + # nbclassic + # notebook +prompt-toolkit==3.0.38 + # via + # ipython + # jupyter-console +protobuf==3.20.3 + # via + # -r dev.cpu.in + # -r merlin/core/requirements.txt + # cudf-cu11 + # googleapis-common-protos + # tensorboard + # tensorflow + # tensorflow-metadata + # tritonclient + # wandb +psutil==5.9.4 + # via + # distributed + # ipykernel + # wandb +ptxcompiler-cu11==0.7.0.post1 + # via cudf-cu11 +ptyprocess==0.7.0 + # via + # pexpect + # terminado +pure-eval==0.2.2 + # via stack-data +py==1.11.0 + # via interrogate +pyarrow==9.0.0 + # via + # -r merlin/Transformers4Rec/requirements/base_external.txt + # -r merlin/core/requirements.txt + # cudf-cu11 +pyasn1==0.4.8 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.2.8 + # via google-auth +pycodestyle==2.10.0 + # via flake8 +pycparser==2.21 + # via cffi +pyflakes==3.0.1 + # via flake8 +pygments==2.14.0 + # via + # ipython + # jupyter-console + # nbconvert + # nbdime + # qtconsole + # sphinx +pynvml==11.4.1 + # via -r merlin/core/requirements.txt +pyproject-hooks==1.0.0 + # via build +pyrsistent==0.19.3 + # via jsonschema +pytest==7.2.2 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt + # pytest-cov + # pytest-xdist +pytest-cov==4.0.0 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +pytest-xdist==3.2.0 + # via + # -r merlin/core/requirements-dev.txt + # -r merlin/models/requirements/dev.txt +python-dateutil==2.8.2 + # via + # botocore + # jupyter-client + # moto + # pandas +python-rapidjson==1.9 + # via tritonclient +pytz==2022.7.1 + # via + # babel + # pandas +pyyaml==6.0 + # via + # bandit + # bokeh + # dask + # distributed + # huggingface-hub + # libcst + # myst-nb + # myst-parser + # optuna + # sphinx-external-toc + # transformers + # wandb +pyzmq==25.0.0 + # via + # ipykernel + # jupyter-client + # jupyter-console + # jupyter-server + # nbclassic + # notebook + # qtconsole +qtconsole==5.4.0 + # via jupyter +qtpy==2.3.0 + # via qtconsole +regex==2022.10.31 + # via + # black + # sacremoses + # transformers +requests==2.28.2 + # via + # huggingface-hub + # moto + # nbdime + # requests-oauthlib + # responses + # sphinx + # tensorboard + # transformers + # wandb +requests-oauthlib==1.3.1 + # via google-auth-oauthlib +responses==0.22.0 + # via moto +rmm-cu11==22.10.0 + # via + # -r gpu.in + # cudf-cu11 +rsa==4.9 + # via google-auth +s3transfer==0.6.0 + # via boto3 +sacremoses==0.0.53 + # via transformers +scikit-learn==1.2.1 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +scipy==1.10.1 + # via + # -r dev.cpu.in + # -r merlin/dataloader/requirements/dev.txt + # scikit-learn +send2trash==1.8.0 + # via + # jupyter-server + # nbclassic + # notebook +sentry-sdk==1.16.0 + # via wandb +setproctitle==1.3.2 + # via wandb +six==1.16.0 + # via + # asttokens + # astunparse + # bleach + # docker-pycreds + # geventhttpclient + # google-auth + # google-pasta + # keras-preprocessing + # python-dateutil + # sacremoses + # tensorflow +smmap==5.0.0 + # via gitdb +sniffio==1.3.0 + # via anyio +snowballstemmer==2.2.0 + # via sphinx +sortedcontainers==2.4.0 + # via distributed +soupsieve==2.4 + # via beautifulsoup4 +sphinx==3.5.4 + # via + # -r merlin/dataloader/requirements/dev.txt + # jupyter-sphinx + # myst-nb + # myst-parser + # nbsphinx + # sphinx-external-toc + # sphinx-multiversion + # sphinx-rtd-theme + # sphinx-togglebutton +sphinx-external-toc==0.3.0 + # via -r merlin/dataloader/requirements/dev.txt +sphinx-markdown-tables==0.0.15 + # via -r merlin/dataloader/requirements/dev.txt +sphinx-multiversion @ git+https://github.com/mikemckiernan/sphinx-multiversion.git + # via -r merlin/dataloader/requirements/dev.txt +sphinx-rtd-theme==1.2.0 + # via -r merlin/dataloader/requirements/dev.txt +sphinx-togglebutton==0.3.2 + # via myst-nb +sphinxcontrib-applehelp==1.0.4 + # via sphinx +sphinxcontrib-copydirs @ git+https://github.com/mikemckiernan/sphinxcontrib-copydirs.git + # via -r merlin/dataloader/requirements/dev.txt +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.1 + # via sphinx +sphinxcontrib-jquery==2.0.0 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +sqlalchemy==1.4.46 + # via + # alembic + # jupyter-cache + # optuna +stack-data==0.6.2 + # via ipython +stevedore==5.0.0 + # via bandit +stringcase==1.2.0 + # via betterproto +tabulate==0.9.0 + # via interrogate +tblib==1.7.0 + # via distributed +tenacity==8.2.2 + # via plotly +tensorboard==2.9.0 + # via tensorflow +tensorboard-data-server==0.6.1 + # via tensorboard +tensorboard-plugin-wit==1.8.1 + # via tensorboard +tensorflow==2.9.0 + # via -r dev.cpu.in +tensorflow-estimator==2.9.0 + # via tensorflow +tensorflow-io-gcs-filesystem==0.31.0 + # via tensorflow +tensorflow-metadata==1.12.0 + # via -r merlin/core/requirements.txt +termcolor==2.2.0 + # via tensorflow +terminado==0.17.1 + # via + # jupyter-server + # nbclassic + # notebook +testbook==0.4.2 + # via + # -r merlin/dataloader/requirements/dev.txt + # -r merlin/models/requirements/dev.txt +threadpoolctl==3.1.0 + # via scikit-learn +tinycss2==1.2.1 + # via nbconvert +tokenizers==0.12.1 + # via transformers +toml==0.10.2 + # via + # black + # interrogate + # responses +tomli==2.0.1 + # via + # build + # check-manifest + # coverage + # mypy + # pyproject-hooks + # pytest +toolz==0.12.0 + # via + # dask + # distributed + # partd +torch==1.13.1 + # via -r dev.cpu.in +tornado==6.1 + # via + # bokeh + # distributed + # ipykernel + # jupyter-client + # jupyter-server + # nbclassic + # nbdime + # notebook + # terminado +tqdm==4.65.0 + # via + # -r merlin/Transformers4Rec/requirements/base_external.txt + # -r merlin/core/requirements.txt + # huggingface-hub + # optuna + # sacremoses + # transformers +traitlets==5.9.0 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-console + # jupyter-core + # jupyter-server + # matplotlib-inline + # nbclassic + # nbclient + # nbconvert + # nbformat + # nbsphinx + # notebook + # qtconsole +transformers==4.18.0 + # via -r merlin/Transformers4Rec/requirements/base_external.txt +tritonclient[all]==2.31.0 + # via -r merlin/systems/requirements/dev.txt +typed-ast==1.5.4 + # via black +types-toml==0.10.8.5 + # via responses +typing-extensions==4.5.0 + # via + # alembic + # black + # cudf-cu11 + # fiddle + # huggingface-hub + # libcst + # mypy + # tensorflow + # torch + # typing-inspect + # wandb +typing-inspect==0.8.0 + # via libcst +uc-micro-py==1.0.1 + # via linkify-it-py +urllib3==1.26.14 + # via + # botocore + # distributed + # requests + # responses + # sentry-sdk +wandb==0.13.10 + # via -r merlin/models/requirements/dev.txt +wcwidth==0.2.6 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.5.1 + # via jupyter-server +werkzeug==2.1.2 + # via + # moto + # tensorboard +wheel==0.38.4 + # via + # astunparse + # nvidia-cublas-cu11 + # nvidia-cuda-runtime-cu11 + # sphinx-togglebutton + # tensorboard +widgetsnbextension==3.6.2 + # via ipywidgets +wrapt==1.15.0 + # via tensorflow +xmltodict==0.13.0 + # via moto +xyzservices==2023.2.0 + # via bokeh +yarl==1.8.2 + # via aiohttp +zict==2.2.0 + # via distributed +zipp==3.15.0 + # via + # importlib-metadata + # importlib-resources +zope-event==4.6 + # via gevent +zope-interface==5.5.2 + # via gevent + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/dev.txt b/requirements/dev.txt deleted file mode 100644 index fe96c42d1..000000000 --- a/requirements/dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -ipython -ipdb diff --git a/requirements/gpu.in b/requirements/gpu.in new file mode 100644 index 000000000..526b1a803 --- /dev/null +++ b/requirements/gpu.in @@ -0,0 +1,4 @@ +--extra-index-url=https://pypi.nvidia.com +rmm-cu11==22.10.0 +cudf-cu11==22.10.0 +dask-cudf-cu11==22.10.0 diff --git a/requirements/merlin-base.in b/requirements/merlin-base.in new file mode 100644 index 000000000..cb5e19aac --- /dev/null +++ b/requirements/merlin-base.in @@ -0,0 +1,2 @@ +-r merlin/core/requirements.txt +-r merlin/Transformers4Rec/requirements/base_external.txt diff --git a/requirements/merlin-dev.in b/requirements/merlin-dev.in new file mode 100644 index 000000000..912d160f1 --- /dev/null +++ b/requirements/merlin-dev.in @@ -0,0 +1,6 @@ +-r merlin/models/requirements/dev.txt +-r merlin/core/requirements-dev.txt +-r merlin/systems/requirements/dev.txt +-r merlin/dataloader/requirements/dev.txt +-r merlin/Transformers4Rec/requirements/dev.txt +-r merlin/NVTabular/requirements/dev.txt