Support VT switching for fbdev backend

The fbdev backend supports framebuffer rendering on a specifiend
VT. To enhancce its compatibility in environments with multiple
TTYs, the following improvements ensure that each framebuffer
device operates independently on its own VT without affecting
others.

- Implement signal handlers to manage virtual terminal switching.
- Pause screen rendering when switching to another VT.
- Resume screen rendering when switching back to the original VT.

The improvement has been validated on Ubuntu 24.04 running in a
virtual machine and on a Raspberry Pi 3B.

Author: huaxinliao

backend/fbdev.c

@@ -30,6 +30,8 @@ typedef struct {
 
     /* Linux virtual terminal (VT) */
     int vt_fd;
+    bool vt_active;
+    struct vt_mode old_vtm;
 
     /* Linux framebuffer */
     int fb_fd;
@@ -38,6 +40,7 @@ typedef struct {
     uint16_t cmap[3][256];
     uint8_t *fb_base;
     size_t fb_len;
+
 } twin_fbdev_t;
 
 /* color conversion */
@@ -86,22 +89,6 @@ static void twin_fbdev_get_screen_size(twin_fbdev_t *tx,
     *height = info.yres;
 }
 
-static void twin_fbdev_damage(twin_screen_t *screen, twin_fbdev_t *tx)
-{
-    int width, height;
-    twin_fbdev_get_screen_size(tx, &width, &height);
-    twin_screen_damage(tx->screen, 0, 0, width, height);
-}
-
-static bool twin_fbdev_work(void *closure)
-{
-    twin_screen_t *screen = SCREEN(closure);
-
-    if (twin_screen_damaged(screen))
-        twin_screen_update(screen);
-    return true;
-}
-
 static inline bool twin_fbdev_is_rgb565(twin_fbdev_t *tx)
 {
     return tx->fb_var.red.offset == 11 && tx->fb_var.red.length == 5 &&
@@ -192,6 +179,50 @@ static bool twin_fbdev_apply_config(twin_fbdev_t *tx)
     return true;
 }
 
+static void twin_fbdev_damage(twin_fbdev_t *tx)
+{
+    int width, height;
+    twin_fbdev_get_screen_size(tx, &width, &height);
+    twin_screen_damage(tx->screen, 0, 0, width, height);
+}
+
+static bool twin_fbdev_damaged(void *closure)
+{
+    twin_fbdev_t *tx = PRIV(closure);
+    twin_screen_t *screen = SCREEN(closure);
+
+    if (!tx->vt_active && twin_screen_damaged(screen))
+        twin_screen_update(screen);
+
+    return true;
+}
+
+static bool twin_fbdev_work(void *closure)
+{
+    twin_fbdev_t *tx = PRIV(closure);
+    twin_screen_t *screen = SCREEN(closure);
+
+    if (tx->vt_active && (tx->fb_base != MAP_FAILED)) {
+        /* Unmap the fbdev */
+        munmap(tx->fb_base, tx->fb_len);
+        tx->fb_base = MAP_FAILED;
+    }
+
+    if (!tx->vt_active && (tx->fb_base == MAP_FAILED)) {
+        /* Restore the fbdev settings */
+        if (!twin_fbdev_apply_config(tx))
+            log_error("Failed to apply configurations to the fbdev");
+
+        /* Mark entire screen for refresh */
+        twin_screen_damage(screen, 0, 0, screen->width, screen->height);
+    }
+
+    if (!tx->vt_active && twin_screen_damaged(screen))
+        twin_screen_update(screen);
+
+    return true;
+}
+
 twin_context_t *twin_fbdev_init(int width, int height)
 {
     char *fbdev_path = getenv(FBDEV_NAME);
@@ -204,6 +235,7 @@ twin_context_t *twin_fbdev_init(int width, int height)
     twin_context_t *ctx = calloc(1, sizeof(twin_context_t));
     if (!ctx)
         return NULL;
+
     ctx->priv = calloc(1, sizeof(twin_fbdev_t));
     if (!ctx->priv)
         return NULL;
@@ -218,10 +250,13 @@ twin_context_t *twin_fbdev_init(int width, int height)
     }
 
     /* Set up virtual terminal environment */
-    if (!twin_vt_setup(&tx->vt_fd)) {
+    if (!twin_vt_setup(&tx->vt_fd, &tx->old_vtm, &tx->vt_active)) {
         goto bail_fb_fd;
     }
 
+    /* Set up signal handlers for switching TTYs */
+    twin_vt_setup_signal_handler();
+
     /* Apply configurations to the framebuffer device */
     if (!twin_fbdev_apply_config(tx)) {
         log_error("Failed to apply configurations to the framebuffer device");
@@ -254,6 +289,9 @@ twin_context_t *twin_fbdev_init(int width, int height)
     /* Setup file handler and work functions */
     twin_set_work(twin_fbdev_work, TWIN_WORK_REDISPLAY, ctx);
 
+    /* Register a callback function to handle damaged rendering */
+    twin_screen_register_damaged(ctx->screen, twin_fbdev_damaged, ctx);
+
     return ctx;
 
 bail_screen:
@@ -292,7 +330,6 @@ static void twin_fbdev_exit(twin_context_t *ctx)
 }
 
 /* Register the Linux framebuffer backend */
-
 const twin_backend_t g_twin_backend = {
     .init = twin_fbdev_init,
     .configure = twin_fbdev_configure,

backend/linux_vt.h

@@ -10,11 +10,19 @@
 #include <fcntl.h>
 #include <linux/kd.h>
 #include <linux/vt.h>
+#include <signal.h>
 #include <stdlib.h>
 #include <sys/ioctl.h>
 #include "twin_private.h"
 
 #define VT_DEV_TTY_MAX 11
+#define SIG_SWITCH_FROM SIGUSR1
+#define SIG_SWITCH_TO SIGUSR2
+
+static int *vt_fd;
+
+static bool *is_vt_active;
+
 static inline int twin_vt_open(int vt_num)
 {
     int fd;
@@ -35,7 +43,42 @@ static inline int twin_vt_mode(int fd, int mode)
     return ioctl(fd, KDSETMODE, mode);
 }
 
-static inline bool twin_vt_setup(int *fd_ptr)
+static inline void twin_vt_process_switch_signal(int signum)
+{
+    if (signum == SIG_SWITCH_FROM) {
+        /* Notify kernel to release current virtual terminal */
+        ioctl(*vt_fd, VT_RELDISP, 1);
+
+        /* Set the virtual terminal display in text mode */
+        ioctl(*vt_fd, KDSETMODE, KD_TEXT);
+
+        *is_vt_active = true;
+    } else if (signum == SIG_SWITCH_TO) {
+        /* Switch complete */
+        ioctl(*vt_fd, VT_RELDISP, VT_ACKACQ);
+
+        /* Restore the virtual terminal display in graphics mode */
+        ioctl(*vt_fd, KDSETMODE, KD_GRAPHICS);
+
+        *is_vt_active = false;
+    }
+}
+
+static inline void twin_vt_setup_signal_handler()
+{
+    struct sigaction tty_sig = {.sa_handler = twin_vt_process_switch_signal};
+    sigfillset(&tty_sig.sa_mask);
+
+    /* Set the signal handler for leaving the current TTY */
+    sigaction(SIG_SWITCH_FROM, &tty_sig, NULL);
+
+    /* Set the signal handler for returning to the previous TTY */
+    sigaction(SIG_SWITCH_TO, &tty_sig, NULL);
+}
+
+static inline bool twin_vt_setup(int *fd_ptr,
+                                 struct vt_mode *old_vtm,
+                                 bool *vt_active)
 {
     /* Open VT0 to inquire information */
     if ((*fd_ptr = twin_vt_open(0)) < -1) {
@@ -57,12 +100,43 @@ static inline bool twin_vt_setup(int *fd_ptr)
         return false;
     }
 
-    /* Set VT to graphics mode to inhibit command-line text */
+    /* Attempt to activate the specified virtual terminal by its number */
+    if (ioctl(*fd_ptr, VT_ACTIVATE, vt_num) < 0) {
+        log_error("Failed to activate VT %d", vt_num);
+        return false;
+    }
+
+    /* Wait until the specified virtual terminal becomes the active VT */
+    if (ioctl(*fd_ptr, VT_WAITACTIVE, vt_num) < 0) {
+        log_error("Failed to activate VT %d", vt_num);
+        return false;
+    }
+
+    /* Save the virtual terminal mode */
+    if (ioctl(*fd_ptr, VT_GETMODE, &old_vtm) < 0) {
+        log_error("Failed to get VT mode");
+        return false;
+    }
+
+    /* Set the virtual terminal mode */
+    struct vt_mode vtm = {
+        .mode = VT_PROCESS, .relsig = SIGUSR1, .acqsig = SIGUSR2};
+
+    if (ioctl(*fd_ptr, VT_SETMODE, &vtm) < 0) {
+        log_error("Failed to set VT mode");
+        return false;
+    }
+
+    /* Set the virtual terminal display mode to graphics mode */
     if (twin_vt_mode(*fd_ptr, KD_GRAPHICS) < 0) {
         log_error("Failed to set KD_GRAPHICS mode");
+        twin_vt_mode(*fd_ptr, KD_TEXT);
         return false;
     }
 
+    vt_fd = fd_ptr;
+    is_vt_active = vt_active;
+
     return true;
 }
 #endif