[ML-5672] simplify docker/travis setup

mengxr · web-flow · commit fa19f43dd007 · 2018-11-16T09:34:55.000-08:00
Similar to databricks/spark-deep-learning@846bfc0 we use docker/docker-compose to describe the test env and hence simplify the content in .travis.yml. A major difference between this PR and databricks/spark-deep-learning@846bfc0 is that we use conda to activate the env instead of updating env variables directly. This provides better compatibility. But we need to use interactive shell to load the profile (bash -i). We also mount "~/.ivy2" to avoid unnecessary download in sbt build.
diff --git a/.travis.yml b/.travis.yml
@@ -1,97 +1,35 @@
-dist: trusty
-
-language: java
+sudo: required
 
-jdk: oraclejdk8
+dist: trusty
 
-sudo: required
+language: minimal
 
 services:
   - docker
 
 cache:
   directories:
   - $HOME/.ivy2/
-  - $HOME/.sbt/launchers/
-  - $HOME/.cache/spark-versions/
-  - $HOME/.sbt/boot/scala-2.11.8/
 
 env:
+  global:
+    - DOCKER_COMPOSE_VERSION=1.22.0
   matrix:
-  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.4.0 SPARK_BUILD="spark-2.4.0-bin-hadoop2.7"
-      SPARK_BUILD_URL="https://dist.apache.org/repos/dist/release/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz"
-      PYTHON_VERSION=2.7.13
-  - SCALA_BINARY_VERSION=2.11.8 SPARK_VERSION=2.4.0 SPARK_BUILD="spark-2.4.0-bin-hadoop2.7"
-      SPARK_BUILD_URL="https://dist.apache.org/repos/dist/release/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz"
-      PYTHON_VERSION=3.6.2
+    - PYTHON_VERSION=3.6
+    - PYTHON_VERSION=2.7
 
 before_install:
-  - ./bin/download_travis_dependencies.sh
-  - if [[ "$PYTHON_VERSION" == 2.* ]]; then
-        export CONDA_URL="repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh"
-        export PYSPARK_PYTHON=python2;
-      else
-        export CONDA_URL="repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh";
-        export PYSPARK_PYTHON=python3;
-      fi
-  - docker run -e "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
-               -e SPARK_VERSION
-               -e SPARK_BUILD
-               -e SCALA_BINARY_VERSION
-               -e PYTHON_VERSION
-               -e PYSPARK_PYTHON
-               -e CONDA_URL
-               -d --name ubuntu-test -v $HOME ubuntu:16.04 tail -f /dev/null
-  - docker cp `pwd` ubuntu-test:$HOME/
-  - docker cp $HOME/.cache ubuntu-test:$HOME/
-  - docker ps
-
-# See this page: http://conda.pydata.org/docs/travis.html
-install:
-  # install needed ubuntu packages
-  - docker exec -t ubuntu-test bash -c "apt-get update && apt-get upgrade -y"
-  - docker exec -t ubuntu-test bash -c "apt-get install -y curl bzip2 openjdk-8-jdk unzip"
-  # download and set up protoc
-  - docker exec -t ubuntu-test bash -c "
-      curl -OL https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip;
-      unzip protoc-3.6.1-linux-x86_64.zip -d /usr/local;"
-  # download and set up miniconda
-  - docker exec -t ubuntu-test bash -c "
-      curl https://$CONDA_URL >> $HOME/miniconda.sh;
-      bash $HOME/miniconda.sh -b -p $HOME/miniconda;
-      bash $HOME/miniconda.sh -b -p $HOME/miniconda;
-      $HOME/miniconda/bin/conda config --set always_yes yes --set changeps1 no;
-      $HOME/miniconda/bin/conda update -q conda;
-      $HOME/miniconda/bin/conda info -a;
-      $HOME/miniconda/bin/conda create -q -n test-environment python=$PYTHON_VERSION"
+  # update docker compose to the specified version, https://docs.travis-ci.com/user/docker/#using-docker-compose
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
 
-  # Activate conda environment ad install required packages
-  - docker exec -t ubuntu-test bash -c "
-      source $HOME/miniconda/bin/activate test-environment;
-      python --version;
-      pip --version;
-      pip install --user -r $HOME/tensorframes/python/requirements.txt;"
+install :
+  - docker-compose build --build-arg PYTHON_VERSION=$PYTHON_VERSION
+  - docker-compose up -d
+  - docker-compose exec master bash -i -c "build/sbt tfs_testing/assembly"
 
 script:
- # Run the scala unit tests first
- - docker exec -t ubuntu-test bash -c "
-     source $HOME/miniconda/bin/activate test-environment;
-     cd $HOME/tensorframes;
-     ./build/sbt -Dspark.version=$SPARK_VERSION
-                 -Dpython.version=$PYSPARK_PYTHON
-                 -Dscala.version=$SCALA_BINARY_VERSION
-                 tfs_testing/test"
-
- # Build the assembly
- - docker exec -t ubuntu-test bash -c "
-     source $HOME/miniconda/bin/activate test-environment;
-     cd $HOME/tensorframes;
-     ./build/sbt -Dspark.version=$SPARK_VERSION
-                 -Dscala.version=$SCALA_BINARY_VERSION
-                 tfs_testing/assembly"
-
- # Run python tests
- - docker exec -t ubuntu-test bash -c "
-     source $HOME/miniconda/bin/activate test-environment;
-     cd $HOME/tensorframes;
-     SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD ./python/run-tests.sh"
+  - docker-compose exec master bash -i -c "build/sbt tfs_testing/test"
+  - docker-compose exec master bash -i -c "python/run-tests.sh"
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,53 @@
+# Use docker-compose to build and run this container.
+# $ docker-compose build [--build-arg PYTHON_VERSION=2.7]
+# $ docker-compose up -d
+# $ docker-compose exec master bash -i -c "..."
+# $ docker-compose down
+
+FROM ubuntu:16.04
+
+ARG PYTHON_VERSION=3.6
+
+RUN apt-get update && \
+    apt-get install -y wget bzip2 openjdk-8-jdk unzip && \
+    apt-get clean
+
+# Install protoc.
+RUN wget --quiet https://github.com/google/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip -O /tmp/protoc.zip && \
+    unzip /tmp/protoc.zip -d /usr/local && \
+    rm /tmp/protoc.zip
+
+# Install Miniconda.
+# Reference: https://hub.docker.com/r/continuumio/miniconda/~/dockerfile/
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc
+
+# Create tensorframes conda env.
+ENV PYTHON_VERSION $PYTHON_VERSION
+COPY ./environment.yml /tmp/environment.yml
+RUN /opt/conda/bin/conda create -n tensorframes python=$PYTHON_VERSION && \
+    /opt/conda/bin/conda env update -n tensorframes -f /tmp/environment.yml && \
+    echo "conda activate tensorframes" >> ~/.bashrc
+
+# Install Spark and update env variables.
+ENV SCALA_BINARY_VERSION 2.11.8
+ENV SPARK_VERSION 2.4.0
+ENV SPARK_BUILD "spark-${SPARK_VERSION}-bin-hadoop2.7"
+ENV SPARK_BUILD_URL "https://dist.apache.org/repos/dist/release/spark/spark-2.4.0/${SPARK_BUILD}.tgz"
+RUN wget --quiet $SPARK_BUILD_URL -O /tmp/spark.tgz && \
+    tar -C /opt -xf /tmp/spark.tgz && \
+    mv /opt/$SPARK_BUILD /opt/spark && \
+    rm /tmp/spark.tgz
+ENV SPARK_HOME /opt/spark
+ENV PATH $SPARK_HOME/bin:$PATH
+ENV PYTHONPATH /opt/spark/python/lib/py4j-0.10.7-src.zip:/opt/spark/python/lib/pyspark.zip:$PYTHONPATH
+ENV PYSPARK_PYTHON python
+
+# The tensorframes dir will be mounted here.
+VOLUME /mnt/tensorframes
+WORKDIR /mnt/tensorframes
+
+CMD /bin/bash
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,11 @@
+version: '3'
+services:
+  master:
+    build: .
+    hostname: master
+    entrypoint: tail -f /dev/null
+    ports:
+    - "4040:4040" # driver UI
+    volumes:
+    - .:/mnt/tensorframes
+    - ~/.ivy2:/root/.ivy2
diff --git a/environment.yml b/environment.yml
@@ -0,0 +1,12 @@
+# To choose a Python version, first create the env with:
+# conda create -n tensorframes python=3.6
+# and then update it
+name: tensorframes
+channels:
+- https://repo.anaconda.com/pkgs/main/linux-64/
+dependencies:
+- pandas=0.23.4
+- nomkl
+- tensorflow=1.12.0
+# test
+- nose=1.3.7
