Updates to the Lingvo Dockerfiles: fixes broken tests caused by unfrozen pip dependencies becoming inconsistent.

PiperOrigin-RevId: 480183184

Author: lingvo-bot

WORKSPACE

@@ -12,7 +12,8 @@ load(
 git_repository(
     name = "subpar",
     remote = "https://github.com/google/subpar",
-    tag = "2.0.0",
+    commit = "35bb9f0092f71ea56b742a520602da9b3638a24f",
+    shallow_since = "1557863961 -0400",
 )
 
 cc_tf_configure()

docker/dev.dockerfile renamed to docker/dev.Dockerfile

@@ -68,49 +68,20 @@ RUN mkdir /bazel && \
     cd / && \
     rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
 
-# TODO(austinwaters): Remove graph-compression-google-research once
-# model-pruning-google-research properly declares it as a dependency.
-ARG pip_dependencies=' \
-      apache-beam[gcp]>=2.8 \
-      attrs \
-      contextlib2 \
-      dataclasses \
-      google-api-python-client \
-      graph-compression-google-research \
-      h5py \
-      ipykernel \
-      jupyter \
-      jupyter_http_over_ws \
-      matplotlib \
-      mock \
-      model-pruning-google-research \
-      numpy \
-      oauth2client \
-      pandas \
-      Pillow \
-      pyyaml \
-      recommonmark \
-      scikit-learn \
-      scipy \
-      sentencepiece \
-      sphinx \
-      sphinx_rtd_theme \
-      sympy'
-
-RUN python3 -m pip --no-cache-dir install $pip_dependencies
-RUN python3 -m ipykernel.kernelspec
+COPY docker /docker
 
 # The latest tensorflow requires CUDA 10 compatible nvidia drivers (410.xx).
 # If you are unable to update your drivers, an alternative is to compile
 # tensorflow from source instead of installing from pip.
-# Ensure we install the correct version by uninstalling first.
-RUN python3 -m pip uninstall -y tensorflow tensorflow-gpu tf-nightly tf-nightly-gpu
-RUN python3 -m pip --no-cache-dir install tensorflow tensorflow-datasets \
-  tensorflow-hub tensorflow-text tensorflow-probability
-RUN python3 -m pip --no-cache-dir install --no-deps waymo-open-dataset-tf-2-6-0
-
+# TODO(austinwaters): Remove graph-compression-google-research once
+RUN --mount=type=cache,target=/root/.cache \
+  python3 -m pip install -r /docker/dev.requirements.txt
+RUN python3 -m ipykernel.kernelspec
 RUN jupyter serverextension enable --py jupyter_http_over_ws
 
+COPY docker/devel.bashrc /root/devel.bashrc
+RUN echo 'source /root/devel.bashrc' >> /root/.bashrc
+
 # TensorBoard
 EXPOSE 6006
 

docker/dev.requirements.txt

@@ -0,0 +1,219 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+# Specifies the python environment for dev.Dockerfile.
+#
+# Some notes:
+#  - We are freezing these package versions to the last known working set here
+#    for a couple reasons:
+#      (1) our prior setup did not pin package versions nor specified all
+#          transitive dependencies, and so was more fragile and vulnurable to
+#          external updates breaking Lingvo.
+#      (2) future updates to dependencies can be explicitly tracked.
+#  - We used to include waymo-open-dataset-tf-2-6-0; this was removed because it
+#    is incompatible with the working set we have here.
+absl-py==1.1.0
+alabaster==0.7.12
+apache-beam==2.39.0
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+asttokens==2.0.5
+astunparse==1.6.3
+attrs==21.4.0
+Babel==2.10.2
+backcall==0.2.0
+beautifulsoup4==4.11.1
+bleach==5.0.0
+cachetools==4.2.4
+certifi==2022.5.18.1
+cffi==1.15.0
+charset-normalizer==2.0.12
+cloudpickle==2.1.0
+commonmark==0.9.1
+contextlib2==21.6.0
+crcmod==1.7
+cycler==0.11.0
+dataclasses==0.6
+debugpy==1.6.0
+decorator==5.1.1
+defusedxml==0.7.1
+dill==0.3.1.1
+dm-tree==0.1.7
+docopt==0.6.2
+docutils==0.17.1
+entrypoints==0.4
+etils==0.6.0
+executing==0.8.3
+fastavro==1.5.1
+fasteners==0.17.3
+fastjsonschema==2.15.3
+flatbuffers==1.12
+fonttools==4.33.3
+gast==0.4.0
+google-api-core==1.31.6
+google-api-python-client==2.51.0
+google-apitools==0.5.31
+google-auth==1.35.0
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.6
+google-cloud-bigquery==2.34.4
+google-cloud-bigquery-storage==2.13.2
+google-cloud-bigtable==1.7.2
+google-cloud-core==1.7.2
+google-cloud-datastore==1.15.5
+google-cloud-dlp==3.7.1
+google-cloud-language==1.3.2
+google-cloud-pubsub==2.13.0
+google-cloud-pubsublite==1.4.2
+google-cloud-recommendations-ai==0.2.0
+google-cloud-spanner==1.19.3
+google-cloud-videointelligence==1.16.3
+google-cloud-vision==1.0.2
+google-crc32c==1.3.0
+google-pasta==0.2.0
+google-resumable-media==2.3.3
+googleapis-common-protos==1.56.2
+graph-compression-google-research==0.0.4
+grpc-google-iam-v1==0.12.4
+grpcio==1.46.3
+grpcio-gcp==0.2.2
+grpcio-status==1.46.3
+h5py==3.7.0
+hdfs==2.7.0
+httplib2==0.19.1
+idna==3.3
+imagesize==1.3.0
+importlib-metadata==4.11.4
+importlib-resources==5.7.1
+ipykernel==6.14.0
+ipython==8.4.0
+ipython-genutils==0.2.0
+ipywidgets==7.7.0
+jedi==0.18.1
+Jinja2==3.1.2
+joblib==1.1.0
+jsonschema==4.6.0
+jupyter==1.0.0
+jupyter-client==7.3.4
+jupyter-console==6.4.3
+jupyter-core==4.10.0
+jupyter-http-over-ws==0.0.8
+jupyterlab-pygments==0.2.2
+jupyterlab-widgets==1.1.0
+keras==2.9.0
+Keras-Preprocessing==1.1.2
+kiwisolver==1.4.3
+libclang==14.0.1
+Markdown==3.3.7
+MarkupSafe==2.1.1
+matplotlib==3.5.2
+matplotlib-inline==0.1.3
+mistune==0.8.4
+mock==4.0.3
+model-pruning-google-research==0.0.5
+mpmath==1.2.1
+nbclient==0.6.4
+nbconvert==6.5.0
+nbformat==5.4.0
+nest-asyncio==1.5.5
+notebook==6.4.12
+numpy==1.22.4
+oauth2client==4.1.3
+oauthlib==3.2.0
+opt-einsum==3.3.0
+orjson==3.7.2
+overrides==6.1.0
+packaging==21.3
+pandas==1.4.2
+pandocfilters==1.5.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.1.1
+prometheus-client==0.14.1
+promise==2.3
+prompt-toolkit==3.0.29
+proto-plus==1.20.6
+protobuf==3.19.4
+psutil==5.9.1
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==7.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.21
+pydot==1.4.2
+Pygments==2.12.0
+PyGObject==3.26.1
+pymongo==3.12.3
+pyparsing==2.4.7
+pyrsistent==0.18.1
+python-apt==1.6.5+ubuntu0.7
+python-dateutil==2.8.2
+pytz==2022.1
+PyYAML==6.0
+pyzmq==23.1.0
+qtconsole==5.3.1
+QtPy==2.1.0
+recommonmark==0.7.1
+requests==2.28.0
+requests-oauthlib==1.3.1
+rsa==4.8
+scikit-learn==1.1.1
+scipy==1.8.1
+Send2Trash==1.8.0
+sentencepiece==0.1.96
+six==1.16.0
+snowballstemmer==2.2.0
+soupsieve==2.3.2.post1
+Sphinx==5.0.1
+sphinx-rtd-theme==1.0.0
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+stack-data==0.2.0
+sympy==1.10.1
+tensorboard==2.9.1
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorflow==2.9.1
+tensorflow-datasets==4.6.0
+tensorflow-estimator==2.9.0
+tensorflow-hub==0.12.0
+tensorflow-io-gcs-filesystem==0.26.0
+tensorflow-metadata==1.8.0
+tensorflow-probability==0.17.0
+tensorflow-text==2.9.0
+termcolor==1.1.0
+terminado==0.15.0
+threadpoolctl==3.1.0
+tinycss2==1.1.1
+toml==0.10.2
+tornado==6.1
+tqdm==4.64.0
+traitlets==5.2.2.post1
+typing-utils==0.1.0
+typing_extensions==4.2.0
+uritemplate==4.1.1
+urllib3==1.26.9
+wcwidth==0.2.5
+webencodings==0.5.1
+Werkzeug==2.1.2
+widgetsnbextension==3.6.0
+wrapt==1.14.1
+zipp==3.8.0

docker/devel.bashrc

@@ -0,0 +1,31 @@
+# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ==============================================================================
+
+
+## Similar to
+## https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/tf_sig_build_dockerfiles
+
+# Do not print anything if this is not being used interactively
+[ -z "$PS1" ] && return
+
+# Set up attractive prompt
+export PS1="\[\e[31m\]lingvo-dev\[\e[m\] \[\e[33m\]\w\[\e[m\] > "
+export TERM=xterm-256color
+alias grep="grep --color=auto"
+alias ls="ls --color=auto"
+alias l="ls -l"
+# Fix nvidia-docker
+ldconfig