diff --git a/docs/core-plugins/babel.md b/docs/core-plugins/babel.md
index d5b5742f8a..70eea46f79 100644
--- a/docs/core-plugins/babel.md
+++ b/docs/core-plugins/babel.md
@@ -20,7 +20,7 @@ module.exports = {
 
 ## Caching
 
-[cache-loader](https://github.com/webpack-contrib/cache-loader) is enabled by default and cache is stored in `<projectRoot>/node_modules/.cache/babel-loader`.
+Cache options of [babel-loader](https://github.com/babel/babel-loader#options) is enabled by default and cache is stored in `<projectRoot>/node_modules/.cache/babel-loader`.
 
 ## Parallelization
 
@@ -38,4 +38,3 @@ vue add babel
 
 - `config.rule('js')`
 - `config.rule('js').use('babel-loader')`
-- `config.rule('js').use('cache-loader')`
diff --git a/packages/@vue/cli-plugin-babel/README.md b/packages/@vue/cli-plugin-babel/README.md
index d5b5742f8a..70eea46f79 100644
--- a/packages/@vue/cli-plugin-babel/README.md
+++ b/packages/@vue/cli-plugin-babel/README.md
@@ -20,7 +20,7 @@ module.exports = {
 
 ## Caching
 
-[cache-loader](https://github.com/webpack-contrib/cache-loader) is enabled by default and cache is stored in `<projectRoot>/node_modules/.cache/babel-loader`.
+Cache options of [babel-loader](https://github.com/babel/babel-loader#options) is enabled by default and cache is stored in `<projectRoot>/node_modules/.cache/babel-loader`.
 
 ## Parallelization
 
@@ -38,4 +38,3 @@ vue add babel
 
 - `config.rule('js')`
 - `config.rule('js').use('babel-loader')`
-- `config.rule('js').use('cache-loader')`
diff --git a/packages/@vue/cli-plugin-babel/index.js b/packages/@vue/cli-plugin-babel/index.js
index e1e3f8e76f..30b561717a 100644
--- a/packages/@vue/cli-plugin-babel/index.js
+++ b/packages/@vue/cli-plugin-babel/index.js
@@ -16,6 +16,7 @@ function genTranspileDepRegex (transpileDependencies) {
   return deps.length ? new RegExp(deps.join('|')) : null
 }
 
+/** @type {import('@vue/cli-service').ServicePlugin} */
 module.exports = (api, options) => {
   const useThreads = process.env.NODE_ENV === 'production' && !!options.parallel
   const cliServicePath = path.dirname(require.resolve('@vue/cli-service'))
@@ -61,19 +62,6 @@ module.exports = (api, options) => {
             return /node_modules/.test(filepath)
           })
           .end()
-        .use('cache-loader')
-          .loader(require.resolve('cache-loader'))
-          .options(api.genCacheConfig('babel-loader', {
-            '@babel/core': require('@babel/core/package.json').version,
-            '@vue/babel-preset-app': require('@vue/babel-preset-app/package.json').version,
-            'babel-loader': require('babel-loader/package.json').version,
-            modern: !!process.env.VUE_CLI_MODERN_BUILD,
-            browserslist: api.service.pkg.browserslist
-          }, [
-            'babel.config.js',
-            '.browserslistrc'
-          ]))
-          .end()
 
     if (useThreads) {
       const threadLoaderConfig = jsRule
@@ -88,5 +76,15 @@ module.exports = (api, options) => {
     jsRule
       .use('babel-loader')
         .loader(require.resolve('babel-loader'))
+        .options(api.genCacheConfig('babel-loader', {
+          '@babel/core': require('@babel/core/package.json').version,
+          '@vue/babel-preset-app': require('@vue/babel-preset-app/package.json').version,
+          'babel-loader': require('babel-loader/package.json').version,
+          modern: !!process.env.VUE_CLI_MODERN_BUILD,
+          browserslist: api.service.pkg.browserslist
+        }, [
+          'babel.config.js',
+          '.browserslistrc'
+        ]))
   })
 }
diff --git a/packages/@vue/cli-plugin-babel/package.json b/packages/@vue/cli-plugin-babel/package.json
index d3a831ff47..e5953030f3 100644
--- a/packages/@vue/cli-plugin-babel/package.json
+++ b/packages/@vue/cli-plugin-babel/package.json
@@ -24,7 +24,6 @@
     "@vue/babel-preset-app": "^4.5.8",
     "@vue/cli-shared-utils": "^4.5.8",
     "babel-loader": "^8.2.2",
-    "cache-loader": "^4.1.0",
     "thread-loader": "^3.0.0",
     "webpack": "^5.10.0"
   },
diff --git a/packages/@vue/cli-service/lib/PluginAPI.js b/packages/@vue/cli-service/lib/PluginAPI.js
index fb20fc6393..d34827f711 100644
--- a/packages/@vue/cli-service/lib/PluginAPI.js
+++ b/packages/@vue/cli-service/lib/PluginAPI.js
@@ -163,7 +163,6 @@ class PluginAPI {
     const variables = {
       partialIdentifier,
       'cli-service': require('../package.json').version,
-      'cache-loader': require('cache-loader/package.json').version,
       env: process.env.NODE_ENV,
       test: !!process.env.VUE_CLI_TEST,
       config: [
@@ -172,6 +171,12 @@ class PluginAPI {
       ]
     }
 
+    try {
+      variables['cache-loader'] = require('cache-loader/package.json').version
+    } catch (e) {
+      // cache-loader is only intended to be used for webpack 4
+    }
+
     if (!Array.isArray(configFiles)) {
       configFiles = [configFiles]
     }