vaapipostproc: add rotation support

Adds vpp rotation support to vaapipostproc.  Uses
property video-direction. Default is identity (no
rotation).

Closes #104

gst-libs/gst/vaapi/gstvaapifilter.c

@@ -1392,7 +1392,7 @@ gst_vaapi_filter_process_unlocked (GstVaapiFilter * filter,
   guint i, num_filters = 0;
   VAStatus va_status;
   VARectangle src_rect, dst_rect;
-  guint va_mirror = from_GstVideoOrientationMethod (filter->video_direction);
+  guint va_mirror = 0, va_rotation = 0;
 
   if (!ensure_operations (filter))
     return GST_VAAPI_FILTER_STATUS_ERROR_ALLOCATION_FAILED;
@@ -1475,8 +1475,12 @@ gst_vaapi_filter_process_unlocked (GstVaapiFilter * filter,
   pipeline_param->filters = filters;
   pipeline_param->num_filters = num_filters;
 
+  from_GstVideoOrientationMethod (filter->video_direction, &va_mirror,
+      &va_rotation);
+
 #if VA_CHECK_VERSION(1,1,0)
   pipeline_param->mirror_state = va_mirror;
+  pipeline_param->rotation_state = va_rotation;
 #endif
 
   // Reference frames for advanced deinterlacing
@@ -1893,21 +1897,39 @@ gst_vaapi_filter_set_video_direction (GstVaapiFilter * filter,
       break;
     case GST_VIDEO_ORIENTATION_HORIZ:
     case GST_VIDEO_ORIENTATION_VERT:
+    case GST_VIDEO_ORIENTATION_90R:
+    case GST_VIDEO_ORIENTATION_180:
+    case GST_VIDEO_ORIENTATION_90L:
+    case GST_VIDEO_ORIENTATION_UL_LR:
+    case GST_VIDEO_ORIENTATION_UR_LL:
     {
 #if VA_CHECK_VERSION(1,1,0)
       VAProcPipelineCaps pipeline_caps;
-      guint va_mirror = from_GstVideoOrientationMethod (method);
+      guint va_mirror = VA_MIRROR_NONE;
+      guint va_rotation = VA_ROTATION_NONE;
 
       VAStatus va_status = vaQueryVideoProcPipelineCaps (filter->va_display,
           filter->va_context, NULL, 0, &pipeline_caps);
       if (!vaapi_check_status (va_status, "vaQueryVideoProcPipelineCaps()"))
         return FALSE;
 
+      from_GstVideoOrientationMethod (method, &va_mirror, &va_rotation);
+
+      if (va_mirror != VA_MIRROR_NONE) {
         if (!(pipeline_caps.mirror_flags & va_mirror)) {
           GST_WARNING ("%s video-direction unsupported",
               gst_vaapi_get_video_direction_nick (method));
           return TRUE;
         }
+      }
+
+      if (va_rotation != VA_ROTATION_NONE) {
+        if (!(pipeline_caps.rotation_flags & (1 << va_rotation))) {
+          GST_WARNING ("%s video-direction unsupported",
+              gst_vaapi_get_video_direction_nick (method));
+          return TRUE;
+        }
+      }
 #else
       GST_WARNING ("%s video-direction unsupported",
           gst_vaapi_get_video_direction_nick (method));

gst-libs/gst/vaapi/gstvaapiutils.c

@@ -821,26 +821,50 @@ to_GstVaapiScaleMethod (guint flags)
   return method;
 }
 
-/* VPP: translate GstVideoOrientationMethod into VA mirror flags */
+/* VPP: translate GstVideoOrientationMethod into VA mirror/rotation flags */
-guint
+void
-from_GstVideoOrientationMethod (guint value)
+from_GstVideoOrientationMethod (guint value, guint * va_mirror,
+    guint * va_rotation)
 {
-  guint va_flags = 0;
+  *va_mirror = 0;
+  *va_rotation = 0;
 
   switch (value) {
 #if VA_CHECK_VERSION(1,1,0)
     case GST_VIDEO_ORIENTATION_IDENTITY:
-      va_flags = VA_MIRROR_NONE;
+      *va_mirror = VA_MIRROR_NONE;
+      *va_rotation = VA_ROTATION_NONE;
       break;
     case GST_VIDEO_ORIENTATION_HORIZ:
-      va_flags = VA_MIRROR_HORIZONTAL;
+      *va_mirror = VA_MIRROR_HORIZONTAL;
+      *va_rotation = VA_ROTATION_NONE;
       break;
     case GST_VIDEO_ORIENTATION_VERT:
-      va_flags = VA_MIRROR_VERTICAL;
+      *va_mirror = VA_MIRROR_VERTICAL;
+      *va_rotation = VA_ROTATION_NONE;
+      break;
+    case GST_VIDEO_ORIENTATION_90R:
+      *va_mirror = VA_MIRROR_NONE;
+      *va_rotation = VA_ROTATION_90;
+      break;
+    case GST_VIDEO_ORIENTATION_180:
+      *va_mirror = VA_MIRROR_NONE;
+      *va_rotation = VA_ROTATION_180;
+      break;
+    case GST_VIDEO_ORIENTATION_90L:
+      *va_mirror = VA_MIRROR_NONE;
+      *va_rotation = VA_ROTATION_270;
+      break;
+    case GST_VIDEO_ORIENTATION_UL_LR:
+      *va_mirror = VA_MIRROR_HORIZONTAL;
+      *va_rotation = VA_ROTATION_90;
+      break;
+    case GST_VIDEO_ORIENTATION_UR_LL:
+      *va_mirror = VA_MIRROR_VERTICAL;
+      *va_rotation = VA_ROTATION_90;
       break;
 #endif
     default:
       break;
   }
-  return va_flags;
 }

gst-libs/gst/vaapi/gstvaapiutils.h

@@ -148,7 +148,8 @@ guint
 to_GstVaapiScaleMethod (guint flags);
 
 G_GNUC_INTERNAL
-guint
+void
-from_GstVideoOrientationMethod (guint value);
+from_GstVideoOrientationMethod (guint value, guint * va_mirror,
+    guint * va_rotation);
 
 #endif /* GST_VAAPI_UTILS_H */

gst/vaapi/gstvaapipostprocutil.c

@@ -177,6 +177,17 @@ _fixate_frame_size (GstVaapiPostproc * postproc, GstVideoInfo * vinfo,
     from_w = GST_VIDEO_INFO_WIDTH (vinfo);
     from_h = GST_VIDEO_INFO_HEIGHT (vinfo);
 
+    /* compensate for rotation if needed */
+    switch (postproc->video_direction) {
+      case GST_VIDEO_ORIENTATION_90R:
+      case GST_VIDEO_ORIENTATION_90L:
+      case GST_VIDEO_ORIENTATION_UL_LR:
+      case GST_VIDEO_ORIENTATION_UR_LL:
+        G_PRIMITIVE_SWAP (gint, from_w, from_h);
+      default:
+        break;
+    }
+
     gst_structure_get_int (outs, "width", &w);
     gst_structure_get_int (outs, "height", &h);