diff --git a/lib/ex_doc/formatter/html.ex b/lib/ex_doc/formatter/html.ex
index 2c1ef3cbd..d02a06b38 100644
--- a/lib/ex_doc/formatter/html.ex
+++ b/lib/ex_doc/formatter/html.ex
@@ -165,6 +165,8 @@ defmodule ExDoc.Formatter.HTML do
           "ExDoc is outputting to an existing directory. " <>
             "Beware documentation output may be mixed with other entries"
         )
+
+        create.(root)
     end
   end
 
diff --git a/test/ex_doc/formatter/epub_io_test.exs b/test/ex_doc/formatter/epub_io_test.exs
new file mode 100644
index 000000000..086374790
--- /dev/null
+++ b/test/ex_doc/formatter/epub_io_test.exs
@@ -0,0 +1,38 @@
+defmodule ExDoc.Formatter.EPUBIOTest do
+  use ExUnit.Case, async: false
+
+  @moduletag :tmp_dir
+
+  test "succeeds if trying to write into an empty existing directory", %{tmp_dir: tmp_dir} do
+    File.mkdir!("#{tmp_dir}/doc")
+
+    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+             generate_docs(tmp_dir)
+           end) == ""
+  end
+
+  test "warns if trying to write into existing directory with files", %{tmp_dir: tmp_dir} do
+    File.mkdir!("#{tmp_dir}/doc")
+    File.touch!("#{tmp_dir}/doc/foo.txt")
+
+    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+             generate_docs(tmp_dir)
+           end) =~ "ExDoc is outputting to an existing directory"
+
+    # Warn only once
+    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+             generate_docs(tmp_dir)
+           end) == ""
+  end
+
+  defp generate_docs(tmp_dir) do
+    config = [
+      app: :foo,
+      formatter: "epub",
+      output: "#{tmp_dir}/doc",
+      source_beam: "#{tmp_dir}/ebin"
+    ]
+
+    ExDoc.generate_docs("Foo", "1.0.0", config)
+  end
+end
diff --git a/test/ex_doc/formatter/epub_test.exs b/test/ex_doc/formatter/epub_test.exs
index 58c8c330c..c1f02225c 100644
--- a/test/ex_doc/formatter/epub_test.exs
+++ b/test/ex_doc/formatter/epub_test.exs
@@ -103,33 +103,6 @@ defmodule ExDoc.Formatter.EPUBTest do
     assert File.regular?(tmp_dir <> "/epub/another_dir/#{doc_config(context)[:project]}.epub")
   end
 
-  test "succeeds if trying to write into an empty existing directory", context do
-    config = doc_config(context)
-
-    new_output = config[:output] <> "/new-dir"
-    File.mkdir_p!(new_output)
-
-    new_config = Keyword.put(config, :output, new_output)
-
-    refute ExUnit.CaptureIO.capture_io(:stderr, fn ->
-             generate_docs(new_config)
-           end) =~ "ExDoc is outputting to an existing directory"
-  end
-
-  test "warns if trying to write into existing directory with files", context do
-    config = doc_config(context)
-    new_output = config[:output] <> "/new-dir"
-
-    File.mkdir_p!(new_output)
-    File.touch!(Path.join(new_output, "dummy-file"))
-
-    new_config = Keyword.put(config, :output, new_output)
-
-    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
-             generate_docs(new_config)
-           end) =~ "ExDoc is outputting to an existing directory"
-  end
-
   test "generates an EPUB file with a standardized structure", %{tmp_dir: tmp_dir} = context do
     generate_docs_and_unzip(context, doc_config(context))
 
diff --git a/test/ex_doc/formatter/html_io_test.exs b/test/ex_doc/formatter/html_io_test.exs
new file mode 100644
index 000000000..8da38c2b2
--- /dev/null
+++ b/test/ex_doc/formatter/html_io_test.exs
@@ -0,0 +1,38 @@
+defmodule ExDoc.Formatter.HtmlIOTest do
+  use ExUnit.Case, async: false
+
+  @moduletag :tmp_dir
+
+  test "succeeds if trying to write into an empty existing directory", %{tmp_dir: tmp_dir} do
+    File.mkdir!("#{tmp_dir}/doc")
+
+    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+             generate_docs(tmp_dir)
+           end) == ""
+  end
+
+  test "warns if trying to write into existing directory with files", %{tmp_dir: tmp_dir} do
+    File.mkdir!("#{tmp_dir}/doc")
+    File.touch!("#{tmp_dir}/doc/foo.txt")
+
+    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+             generate_docs(tmp_dir)
+           end) =~ "ExDoc is outputting to an existing directory"
+
+    # Warn only once
+    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
+             generate_docs(tmp_dir)
+           end) == ""
+  end
+
+  defp generate_docs(tmp_dir) do
+    config = [
+      app: :foo,
+      formatter: "html",
+      output: "#{tmp_dir}/doc",
+      source_beam: "#{tmp_dir}/ebin"
+    ]
+
+    ExDoc.generate_docs("Foo", "1.0.0", config)
+  end
+end
diff --git a/test/ex_doc/formatter/html_test.exs b/test/ex_doc/formatter/html_test.exs
index ee8489c94..9cde053da 100644
--- a/test/ex_doc/formatter/html_test.exs
+++ b/test/ex_doc/formatter/html_test.exs
@@ -180,33 +180,6 @@ defmodule ExDoc.Formatter.HTMLTest do
     refute content_module =~ re[:index][:refresh]
   end
 
-  test "succeeds if trying to write into an empty existing directory", context do
-    config = doc_config(context)
-
-    new_output = config[:output] <> "/new-dir"
-    File.mkdir_p!(new_output)
-
-    new_config = Keyword.put(config, :output, new_output)
-
-    refute ExUnit.CaptureIO.capture_io(:stderr, fn ->
-             generate_docs(new_config)
-           end) =~ "ExDoc is outputting to an existing directory"
-  end
-
-  test "warns if trying to write into existing directory with files", context do
-    config = doc_config(context)
-    new_output = config[:output] <> "/new-dir"
-
-    File.mkdir_p!(new_output)
-    File.touch!(Path.join(new_output, "dummy-file"))
-
-    new_config = Keyword.put(config, :output, new_output)
-
-    assert ExUnit.CaptureIO.capture_io(:stderr, fn ->
-             generate_docs(new_config)
-           end) =~ "ExDoc is outputting to an existing directory"
-  end
-
   test "allows to set the authors of the document", %{tmp_dir: tmp_dir} = context do
     generate_docs(doc_config(context, authors: ["John Doe", "Jane Doe"]))
     content_index = File.read!(tmp_dir <> "/html/api-reference.html")