Merge remote-tracking branch 'upstream/master' into mp

Author: mrshenli

.circleci/config.yml

@@ -13,13 +13,46 @@ setup_linux_system_environment: &setup_linux_system_environment
   name: Set Up System Environment
   no_output_timeout: "1h"
   command: |
-    set -e
-     # Set up CircleCI GPG keys for apt, if needed
+    set -ex
+
+    # Set up CircleCI GPG keys for apt, if needed
     curl -L https://packagecloud.io/circleci/trusty/gpgkey | sudo apt-key add -
 
+    # Stop background apt updates.  Hypothetically, the kill should not
+    # be necessary, because stop is supposed to send a kill signal to
+    # the process, but we've added it for good luck.  Also
+    # hypothetically, it's supposed to be unnecessary to wait for
+    # the process to block.  We also have that line for good luck.
+    # If you like, try deleting them and seeing if it works.
+    sudo systemctl stop apt-daily.service || true
+    sudo systemctl kill --kill-who=all apt-daily.service || true
+
+    sudo systemctl stop unattended-upgrades.service || true
+    sudo systemctl kill --kill-who=all unattended-upgrades.service || true
+
+    # wait until `apt-get update` has been killed
+    while systemctl is-active --quiet apt-daily.service
+    do
+      sleep 1;
+    done
+    while systemctl is-active --quiet unattended-upgrades.service
+    do
+      sleep 1;
+    done
+
+    # See if we actually were successful
+    systemctl list-units --all | cat
+
+    sudo apt-get purge -y unattended-upgrades
+
+    cat /etc/apt/sources.list
+
+    ps auxfww | grep [a]pt
+    ps auxfww | grep dpkg
+
 pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
   machine:
-    image: default
+    image: ubuntu-1604:201903-01
   steps:
   - checkout
   - run:
@@ -30,28 +63,43 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
       command: |
         set -e
 
+        # Set up NVIDIA docker repo
         curl -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
-        echo "deb https://nvidia.github.io/libnvidia-container/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
-        echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
-        echo "deb https://nvidia.github.io/nvidia-docker/ubuntu14.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
-
-        sudo apt-get -qq update
-        sudo apt-get -qq remove linux-image-generic linux-headers-generic linux-generic
-        sudo apt-get -qq install \
+        echo "deb https://nvidia.github.io/libnvidia-container/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+        echo "deb https://nvidia.github.io/nvidia-container-runtime/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+        echo "deb https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64 /" | sudo tee -a /etc/apt/sources.list.d/nvidia-docker.list
+
+        sudo apt-get -y update
+        sudo apt-get -y remove linux-image-generic linux-headers-generic linux-generic docker-ce
+        # WARNING: Docker version is hardcoded here; you must update the
+        # version number below for docker-ce and nvidia-docker2 to get newer
+        # versions of Docker.  We hardcode these numbers because we kept
+        # getting broken CI when Docker would update their docker version,
+        # and nvidia-docker2 would be out of date for a day until they
+        # released a newer version of their package.
+        #
+        # How to figure out what the correct versions of these packages are?
+        # My preferred method is to start a Docker instance of the correct
+        # Ubuntu version (e.g., docker run -it ubuntu:16.04) and then ask
+        # apt what the packages you need are.  Note that the CircleCI image
+        # comes with Docker.
+        sudo apt-get -y install \
           linux-headers-$(uname -r) \
           linux-image-generic \
           moreutils \
-          nvidia-docker2 \
-          expect-dev \
-          docker-ce=18.06.2~ce~3-0~ubuntu
+          docker-ce=5:18.09.4~3-0~ubuntu-xenial \
+          nvidia-container-runtime=2.0.0+docker18.09.4-1 \
+          nvidia-docker2=2.0.3+docker18.09.4-1 \
+          expect-dev
 
         sudo pkill -SIGHUP dockerd
 
         sudo pip -q install awscli==1.16.35
 
         if [ -n "${CUDA_VERSION}" ]; then
-          wget 'https://s3.amazonaws.com/ossci-linux/nvidia_driver/NVIDIA-Linux-x86_64-396.26.run'
-          sudo /bin/bash ./NVIDIA-Linux-x86_64-396.26.run -s --no-drm
+          DRIVER_FN="NVIDIA-Linux-x86_64-410.104.run"
+          wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
+          sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
           nvidia-smi
         fi