sys/: Fix up the XShm call testing so that we catch errors, and don't cause new ones by attempting to detach from a s...

Original commit message from CVS:
* sys/ximage/ximagesink.c: (gst_ximagesink_check_xshm_calls):
* sys/xvimage/xvimagesink.c: (gst_xvimagesink_check_xshm_calls):
Fix up the XShm call testing so that we catch errors, and don't
cause new ones by attempting to detach from a segment we failed
to attach to. Fixes #312439.

ChangeLog

@@ -1,3 +1,11 @@
+2006-02-12  Jan Schmidt  <thaytan@mad.scientist.com>
+
+	* sys/ximage/ximagesink.c: (gst_ximagesink_check_xshm_calls):
+	* sys/xvimage/xvimagesink.c: (gst_xvimagesink_check_xshm_calls):
+	Fix up the XShm call testing so that we catch errors, and don't
+	cause new ones by attempting to detach from a segment we failed
+	to attach to. Fixes #312439.
+
 2006-02-10  Edward Hervey  <edward@fluendo.com>
 
 	* gst/typefind/gsttypefindfunctions.c: (plugin_init):

sys/ximage/ximagesink.c

@@ -304,9 +304,17 @@ gst_ximagesink_check_xshm_calls (GstXImageSink * ximagesink,
   size_t size;
   int (*handler) (Display *, XErrorEvent *);
   gboolean result = FALSE;
+  gboolean did_attach = FALSE;
 
   g_return_val_if_fail (xcontext != NULL, FALSE);
 
+  /* Sync to ensure any older errors are already processed */
+  XSync (xcontext->disp, FALSE);
+
+  /* Set defaults so we don't free these later unnecessarily */
+  SHMInfo.shmaddr = ((void *) -1);
+  SHMInfo.shmid = -1;
+
   /* Setting an error handler to catch failure */
   error_caught = FALSE;
   handler = XSetErrorHandler (gst_ximagesink_handle_xerror);
@@ -316,7 +324,10 @@ gst_ximagesink_check_xshm_calls (GstXImageSink * ximagesink,
 
   ximage = XShmCreateImage (xcontext->disp, xcontext->visual,
       xcontext->depth, ZPixmap, NULL, &SHMInfo, 1, 1);
-  if (!ximage) {
+
+  /* Might cause an error, sync to ensure it is noticed */
+  XSync (xcontext->disp, FALSE);
+  if (!ximage || error_caught) {
     GST_WARNING ("could not XShmCreateImage a 1x1 image");
     goto beach;
   }
@@ -342,27 +353,31 @@ gst_ximagesink_check_xshm_calls (GstXImageSink * ximagesink,
     goto beach;
   }
 
-  XSync (xcontext->disp, 0);
+  /* Sync to ensure we see any errors we caused */
+  XSync (xcontext->disp, FALSE);
 
-  /* Destroy the image */
+  if (!error_caught) {
-  if (SHMInfo.shmaddr != ((void *) -1)) {
+    did_attach = TRUE;
-    XShmDetach (xcontext->disp, &SHMInfo);
+    /* store whether we succeeded in result */
-    XSync (xcontext->disp, 0);
+    result = TRUE;
-    shmdt (SHMInfo.shmaddr);
   }
+
+beach:
+  /* Sync to ensure we swallow any errors we caused and reset error_caught */
+  XSync (xcontext->disp, FALSE);
+  error_caught = FALSE;
+  XSetErrorHandler (handler);
+
+  if (did_attach) {
+    XShmDetach (xcontext->disp, &SHMInfo);
+    XSync (xcontext->disp, FALSE);
+  }
+  if (SHMInfo.shmaddr != ((void *) -1))
+    shmdt (SHMInfo.shmaddr);
   if (SHMInfo.shmid > 0)
     shmctl (SHMInfo.shmid, IPC_RMID, 0);
   if (ximage)
     XDestroyImage (ximage);
-
-  /* store whether we succeeded in result and reset error_caught */
-  result = !error_caught;
-  error_caught = FALSE;
-
-beach:
-  XSetErrorHandler (handler);
-
-  XSync (xcontext->disp, FALSE);
   return result;
 }
 #endif /* HAVE_XSHM */

sys/xvimage/xvimagesink.c

@@ -382,9 +382,17 @@ gst_xvimagesink_check_xshm_calls (GstXContext * xcontext)
   gint size;
   int (*handler) (Display *, XErrorEvent *);
   gboolean result = FALSE;
+  gboolean did_attach = FALSE;
 
   g_return_val_if_fail (xcontext != NULL, FALSE);
 
+  /* Sync to ensure any older errors are already processed */
+  XSync (xcontext->disp, FALSE);
+
+  /* Set defaults so we don't free these later unnecessarily */
+  SHMInfo.shmaddr = ((void *) -1);
+  SHMInfo.shmid = -1;
+
   /* Setting an error handler to catch failure */
   error_caught = FALSE;
   handler = XSetErrorHandler (gst_xvimagesink_handle_xerror);
@@ -393,7 +401,10 @@ gst_xvimagesink_check_xshm_calls (GstXContext * xcontext)
   GST_DEBUG ("XvShmCreateImage of 1x1");
   xvimage = XvShmCreateImage (xcontext->disp, xcontext->xv_port_id,
       xcontext->im_format, NULL, 1, 1, &SHMInfo);
-  if (!xvimage) {
+
+  /* Might cause an error, sync to ensure it is noticed */
+  XSync (xcontext->disp, FALSE);
+  if (!xvimage || error_caught) {
     GST_WARNING ("could not XvShmCreateImage a 1x1 image");
     goto beach;
   }
@@ -419,19 +430,28 @@ gst_xvimagesink_check_xshm_calls (GstXContext * xcontext)
     goto beach;
   }
 
-  /* store whether we succeeded in result and reset error_caught */
+  /* Sync to ensure we see any errors we caused */
-  result = !error_caught;
+  XSync (xcontext->disp, FALSE);
-  error_caught = FALSE;
+
+  if (!error_caught) {
+    did_attach = TRUE;
+    /* store whether we succeeded in result */
+    result = TRUE;
+  }
 
 beach:
+  /* Sync to ensure we swallow any errors we caused and reset error_caught */
+  XSync (xcontext->disp, FALSE);
+
+  error_caught = FALSE;
   XSetErrorHandler (handler);
 
-  XSync (xcontext->disp, FALSE);
+  if (did_attach) {
-  if (SHMInfo.shmaddr != ((void *) -1)) {
     XShmDetach (xcontext->disp, &SHMInfo);
     XSync (xcontext->disp, FALSE);
-    shmdt (SHMInfo.shmaddr);
   }
+  if (SHMInfo.shmaddr != ((void *) -1))
+    shmdt (SHMInfo.shmaddr);
   if (SHMInfo.shmid > 0)
     shmctl (SHMInfo.shmid, IPC_RMID, 0);
   if (xvimage)