diff --git a/src/lib/get-settings/get-settings.test.js b/src/lib/get-settings/get-settings.test.js
index c0ea6c97..4a80931f 100644
--- a/src/lib/get-settings/get-settings.test.js
+++ b/src/lib/get-settings/get-settings.test.js
@@ -1,8 +1,10 @@
 import getSettings from './index.js';
 
 describe('replacements', () => {
-  it('should return nothing with no settings set', () => {
-    expect(getSettings()).toEqual(undefined);
+  it('should return the default config with no settings set', () => {
+    const derivedSettings = getSettings();
+    expect(derivedSettings.extends).toEqual('lighthouse:default');
+    expect(derivedSettings.settings).toEqual({});
   });
 
   it('should return a template config with preset set to desktop', () => {
@@ -34,6 +36,11 @@ describe('replacements', () => {
     expect(derivedSettings.settings.locale).toEqual('es');
   });
 
+  it('should skip is-crawlable audit when using deployUrl', () => {
+    const derivedSettings = getSettings({}, true);
+    expect(derivedSettings.settings.skipAudits).toEqual(['is-crawlable']);
+  });
+
   it('should error with incorrect syntax for process.env.SETTINGS', () => {
     process.env.SETTINGS = 'not json';
     expect(getSettings).toThrow(/Invalid JSON/);
diff --git a/src/lib/get-settings/index.js b/src/lib/get-settings/index.js
index b9d48ecc..8c7fcc9a 100644
--- a/src/lib/get-settings/index.js
+++ b/src/lib/get-settings/index.js
@@ -20,9 +20,8 @@ const mergeSettingsSources = (inputSettings = {}) => {
   return Object.assign({}, envSettings, inputSettings);
 };
 
-const getSettings = (inputSettings) => {
+const getSettings = (inputSettings, isUsingDeployUrl) => {
   const settings = mergeSettingsSources(inputSettings);
-  if (Object.keys(settings).length === 0) return;
 
   // Set a base-level config based on the preset input value
   // (desktop is currently the only supported option)
@@ -35,6 +34,12 @@ const getSettings = (inputSettings) => {
     derivedSettings.settings.locale = settings.locale;
   }
 
+  // If we are running against the Netlify deploy URL, the injected x-robots-tag will always cause the audit to fail,
+  // likely producing a false positive, so we skip in this case
+  if (isUsingDeployUrl) {
+    derivedSettings.settings.skipAudits = ['is-crawlable'];
+  }
+
   return derivedSettings;
 };
 
diff --git a/src/lib/run-event/index.js b/src/lib/run-event/index.js
index 348f910d..0409e403 100644
--- a/src/lib/run-event/index.js
+++ b/src/lib/run-event/index.js
@@ -47,7 +47,7 @@ const runEvent = async ({
   let errorMetadata = [];
 
   try {
-    const settings = getSettings(inputs?.settings);
+    const settings = getSettings(inputs?.settings, isOnSuccess);
 
     const allErrors = [];
     const data = [];