avdevice/gdigrab: process window in a separate thread

Move window creation and event processing to a dedicated thread.

GetMessage only processes events from the calling thread's message
queue. Because gdigrab_read_header and gdigrab_read_packet don't run on
the same thread, the message queue was not being drained.

Fixes: #11539

libavdevice/gdigrab.c

@@ -64,7 +64,10 @@ struct gdigrab {
     void      *buffer;      /**< The buffer containing the bitmap image data */
     RECT       clip_rect;   /**< The subarea of the screen or window to clip */
 
-    HWND       region_hwnd; /**< Handle of the region border window */
+    HANDLE        region_thread;    /**< Region window thread */
+    DWORD         region_thread_id; /**< Region window thread ID */
+    HANDLE        region_init;      /**< Region window ready signal */
+    volatile LONG region_result;    /**< Region window status, 0 success */
 
     int cursor_error_printed;
 };
@@ -114,19 +117,21 @@ gdigrab_region_wnd_proc(HWND hwnd, UINT msg, WPARAM wparam, LPARAM lparam)
 }
 
 /**
- * Initialize the region outline window.
+ * Run the region outline window loop.
  *
- * @param s1 The format context.
- * @param gdigrab gdigrab context.
- * @return 0 success, !0 failure
+ * @param opaque gdigrab context.
+ * @return 0
  */
-static int
-gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
+static DWORD WINAPI
+gdigrab_region_wnd_task(LPVOID opaque)
 {
+    struct gdigrab *gdigrab = opaque;
+
     HWND hwnd;
     RECT rect = gdigrab->clip_rect;
     HRGN region = NULL;
     HRGN region_interior = NULL;
+    MSG msg;
 
     DWORD style = WS_POPUP | WS_VISIBLE;
     DWORD ex = WS_EX_TOOLWINDOW | WS_EX_TOPMOST | WS_EX_TRANSPARENT;
@@ -142,7 +147,8 @@ gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
             rect.right - rect.left, rect.bottom - rect.top,
             NULL, NULL, NULL, NULL);
     if (!hwnd) {
-        WIN32_API_ERROR("Could not create region display window");
+        av_log(NULL, AV_LOG_ERROR, "Could not create region display window "
+            "(error %li)\n", GetLastError());
         goto error;
     }
 
@@ -155,7 +161,8 @@ gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
             rect.bottom - rect.top - REGION_WND_BORDER);
     CombineRgn(region, region, region_interior, RGN_DIFF);
     if (!SetWindowRgn(hwnd, region, FALSE)) {
-        WIN32_API_ERROR("Could not set window region");
+        av_log(NULL, AV_LOG_ERROR, "Could not set window region "
+            "(error %li)\n", GetLastError());
         goto error;
     }
     // The "region" memory is now owned by the window
@@ -166,8 +173,14 @@ gdigrab_region_wnd_init(AVFormatContext *s1, struct gdigrab *gdigrab)
 
     ShowWindow(hwnd, SW_SHOW);
 
-    gdigrab->region_hwnd = hwnd;
+    InterlockedExchange(&gdigrab->region_result, 0);
+    SetEvent(gdigrab->region_init);
 
+    while (GetMessage(&msg, NULL, 0, 0)) {
+        DispatchMessage(&msg);
+    }
+
+    DestroyWindow(hwnd);
     return 0;
 
 error:
@@ -177,7 +190,44 @@ error:
         DeleteObject(region_interior);
     if (hwnd)
         DestroyWindow(hwnd);
-    return 1;
+    InterlockedExchange(&gdigrab->region_result, AVERROR(EIO));
+    SetEvent(gdigrab->region_init);
+    return 0;
+}
+
+/**
+ * Start the region outline window thread.
+ *
+ * @param gdigrab gdigrab context.
+ * @return 0 success, !0 failure
+ */
+static int
+gdigrab_region_wnd_init(struct gdigrab *gdigrab)
+{
+    int result = AVERROR(EIO);
+
+    /* Start the thread and wait for it to try creating a window */
+    gdigrab->region_result = 0;
+    gdigrab->region_init = CreateEvent(NULL, TRUE, FALSE, NULL);
+    if (!gdigrab->region_init)
+        goto end;
+    gdigrab->region_thread = CreateThread(NULL, 0, gdigrab_region_wnd_task,
+        gdigrab, 0, &gdigrab->region_thread_id);
+    if (!gdigrab->region_thread)
+        goto end;
+
+    WaitForSingleObject(gdigrab->region_init, INFINITE);
+    result = gdigrab->region_result;
+
+ end:
+    if (gdigrab->region_thread && result < 0) {
+        WaitForSingleObject(gdigrab->region_thread, INFINITE);
+        CloseHandle(gdigrab->region_thread);
+    }
+    if (gdigrab->region_init) {
+        CloseHandle(gdigrab->region_init);
+    }
+    return result;
 }
 
 /**
@@ -189,30 +239,9 @@ error:
 static void
 gdigrab_region_wnd_destroy(AVFormatContext *s1, struct gdigrab *gdigrab)
 {
-    if (gdigrab->region_hwnd)
-        DestroyWindow(gdigrab->region_hwnd);
-    gdigrab->region_hwnd = NULL;
-}
-
-/**
- * Process the Windows message queue.
- *
- * This is important to prevent Windows from thinking the window has become
- * unresponsive. As well, things like WM_PAINT (to actually draw the window
- * contents) are handled from the message queue context.
- *
- * @param s1 The format context.
- * @param gdigrab gdigrab context.
- */
-static void
-gdigrab_region_wnd_update(AVFormatContext *s1, struct gdigrab *gdigrab)
-{
-    HWND hwnd = gdigrab->region_hwnd;
-    MSG msg;
-
-    while (PeekMessage(&msg, hwnd, 0, 0, PM_REMOVE)) {
-        DispatchMessage(&msg);
-    }
+    PostThreadMessage(gdigrab->region_thread_id, WM_QUIT, 0, 0);
+    WaitForSingleObject(gdigrab->region_thread, INFINITE);
+    CloseHandle(gdigrab->region_thread);
 }
 
 /**
@@ -435,7 +464,7 @@ gdigrab_read_header(AVFormatContext *s1)
     gdigrab->cursor_error_printed = 0;
 
     if (gdigrab->show_region) {
-        if (gdigrab_region_wnd_init(s1, gdigrab)) {
+        if (gdigrab_region_wnd_init(gdigrab)) {
             ret = AVERROR(EIO);
             goto error;
         }
@@ -573,10 +602,6 @@ static int gdigrab_read_packet(AVFormatContext *s1, AVPacket *pkt)
     /* Calculate the time of the next frame */
     time_frame += INT64_C(1000000);
 
-    /* Run Window message processing queue */
-    if (gdigrab->show_region)
-        gdigrab_region_wnd_update(s1, gdigrab);
-
     /* wait based on the frame rate */
     for (;;) {
         curtime = av_gettime_relative();