diff --git a/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/RetryHttpRequestInitializer.java b/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/RetryHttpRequestInitializer.java
index 756dce0a99..d98f72e216 100644
--- a/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/RetryHttpRequestInitializer.java
+++ b/sdk/src/main/java/com/google/cloud/dataflow/sdk/util/RetryHttpRequestInitializer.java
@@ -237,9 +237,10 @@ public void initialize(HttpRequest request) throws IOException {
                                             .setMultiplier(2).build(),
             sleeper, ignoredResponseCodes));
 
-    // Retry immediately on IOExceptions.
+    // Similarly back off on IOExceptions.
     LoggingHttpBackOffIOExceptionHandler loggingBackoffHandler =
-        new LoggingHttpBackOffIOExceptionHandler(BackOff.ZERO_BACKOFF);
+        new LoggingHttpBackOffIOExceptionHandler(
+            new ExponentialBackOff.Builder().setNanoClock(nanoClock).setMultiplier(2).build());
     request.setIOExceptionHandler(loggingBackoffHandler);
 
     // Set response initializer