diff --git a/lua/nvim-tree/actions/fs/remove-file.lua b/lua/nvim-tree/actions/fs/remove-file.lua
index 76b40b2455d..4053a8424e5 100644
--- a/lua/nvim-tree/actions/fs/remove-file.lua
+++ b/lua/nvim-tree/actions/fs/remove-file.lua
@@ -9,7 +9,10 @@ local M = {
 }
 
 local function close_windows(windows)
-  if view.View.float.enable and #vim.api.nvim_list_wins() == 1 then
+  -- Prevent from closing when the win count equals 1 or 2,
+  -- where the win to remove could be the last opened.
+  -- For details see #2503.
+  if view.View.float.enable and #vim.api.nvim_list_wins() < 3 then
     return
   end
 
@@ -24,15 +27,15 @@ local function clear_buffer(absolute_path)
   local bufs = vim.fn.getbufinfo { bufloaded = 1, buflisted = 1 }
   for _, buf in pairs(bufs) do
     if buf.name == absolute_path then
+      local tree_winnr = vim.api.nvim_get_current_win()
       if buf.hidden == 0 and (#bufs > 1 or view.View.float.enable) then
-        local winnr = vim.api.nvim_get_current_win()
         vim.api.nvim_set_current_win(buf.windows[1])
         vim.cmd ":bn"
-        if not view.View.float.enable then
-          vim.api.nvim_set_current_win(winnr)
-        end
       end
       vim.api.nvim_buf_delete(buf.bufnr, { force = true })
+      if not view.View.float.quit_on_focus_loss then
+        vim.api.nvim_set_current_win(tree_winnr)
+      end
       if M.config.actions.remove_file.close_window then
         close_windows(buf.windows)
       end