diff --git a/.travis.yml b/.travis.yml
index a38e90bbce8ba..81cd461dd2c87 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,6 +1,15 @@
 language: python
 python: 3.7
 
+addons:
+  apt:
+    update: true
+    packages:
+      - xvfb
+
+services:
+  - xvfb
+
 # To turn off cached cython files and compiler cache
 # set NOCACHE-true
 # To delete caches go to https://travis-ci.org/OWNER/REPOSITORY/caches or run
@@ -10,11 +19,9 @@ cache:
   ccache: true
   directories:
     - $HOME/.cache # cython cache
-    - $HOME/.ccache # compiler cache
 
 env:
   global:
-    # Variable for test workers
     - PYTEST_WORKERS="auto"
     # create a github personal access token
     # cd pandas-dev/pandas
@@ -22,18 +29,17 @@ env:
     - secure: "EkWLZhbrp/mXJOx38CHjs7BnjXafsqHtwxPQrqWy457VDFWhIY1DMnIR/lOWG+a20Qv52sCsFtiZEmMfUjf0pLGXOqurdxbYBGJ7/ikFLk9yV2rDwiArUlVM9bWFnFxHvdz9zewBH55WurrY4ShZWyV+x2dWjjceWG5VpWeI6sA="
 
 git:
-  # for cloning
   depth: false
 
 matrix:
   fast_finish: true
 
   include:
-    # In allowed failures
     - dist: bionic
       python: 3.9-dev
       env:
         - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
+
     - env:
         - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
@@ -42,7 +48,7 @@ matrix:
 
     - arch: arm64
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
+        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
     - env:
         - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
@@ -71,12 +77,6 @@ before_install:
   - uname -a
   - git --version
   - ./ci/check_git_tags.sh
-  # Because travis runs on Google Cloud and has a /etc/boto.cfg,
-  # it breaks moto import, see:
-  # https://github.com/spulec/moto/issues/1771
-  # https://github.com/boto/boto/issues/3741
-  # This overrides travis and tells it to look nowhere.
-  - export BOTO_CONFIG=/dev/null
 
 install:
   - echo "install start"
diff --git a/ci/build39.sh b/ci/build39.sh
index f2ef11d5a71f4..faef2be03c2bb 100755
--- a/ci/build39.sh
+++ b/ci/build39.sh
@@ -1,7 +1,6 @@
 #!/bin/bash -e
 # Special build for python3.9 until numpy puts its own wheels up
 
-sudo apt-get install build-essential gcc xvfb
 pip install --no-deps -U pip wheel setuptools
 pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis
 
diff --git a/ci/setup_env.sh b/ci/setup_env.sh
index 961433204cfbb..247f809c5fe63 100755
--- a/ci/setup_env.sh
+++ b/ci/setup_env.sh
@@ -42,9 +42,7 @@ else
 fi
 
 if [ "${TRAVIS_CPU_ARCH}" == "arm64" ]; then
-  sudo apt-get update
-  sudo apt-get -y install xvfb
-  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-0/Miniforge3-4.8.5-0-Linux-aarch64.sh"
+  CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.8.5-1/Miniforge3-4.8.5-1-Linux-aarch64.sh"
 else
   CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-$CONDA_OS.sh"
 fi
@@ -100,8 +98,6 @@ echo "conda list (root environment)"
 conda list
 
 # Clean up any left-over from a previous build
-# (note workaround for https://github.com/conda/conda/issues/2679:
-#  `conda env remove` issue)
 conda remove --all -q -y -n pandas-dev
 
 echo
@@ -142,12 +138,6 @@ conda list pandas
 echo "[Build extensions]"
 python setup.py build_ext -q -i -j2
 
-# TODO: Some of our environments end up with old versions of pip (10.x)
-# Adding a new enough version of pip to the requirements explodes the
-# solve time. Just using pip to update itself.
-# - py35_macos
-# - py35_compat
-# - py36_32bit
 echo "[Updating pip]"
 python -m pip install --no-deps -U pip wheel setuptools