VAAPI colormatrix and fullrange support

VAAPI: color matrix and fullrange support attempt #1.
Also, guard against requesting/setting unsupported properties.
And fix video eq handling like in vo_xv.

This is based on my earlier vo_xv extension and some unused part
of vo_vaapi itself.

The ffmpeg vaapi wrapper seems to ignore va_context.colorspace.
If that information is at all present, it will be in AVContext
and vo_frame.flags as usual, so there might be a chance for this to
work.
--- xine-lib-1.2/src/video_out/video_out_vaapi.c	2014-04-19 14:47:30.000000000 +0200
+++ xine-lib-1.2/src/video_out/video_out_vaapi.c	2014-04-25 01:21:00.000000000 +0200
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2012 Edgar Hucek <gimli|@dark-green.com>
- * Copyright (C) 2012-2013 xine developers
+ * Copyright (C) 2012-2014 xine developers
  *
  * This file is part of xine, a free video player.
  *
@@ -268,8 +268,15 @@ struct vaapi_driver_s {
   unsigned int        scaling_level;
   va_property_t       props[VO_NUM_PROPERTIES];
   unsigned int        swap_uv_planes;
+
+  /* color matrix and fullrange emulation */
+  int                 cm_state, color_matrix, update_csc, vaapi_cm_flags;
 };
 
+/* import common color matrix stuff */
+#define CM_DRIVER_T vaapi_driver_t
+#include "color_matrix.c"
+
 ff_vaapi_surface_t  *va_render_surfaces   = NULL;
 VASurfaceID         *va_surface_ids       = NULL;
 VASurfaceID         *va_soft_surface_ids  = NULL;
@@ -1600,36 +1607,85 @@ error:
   return VA_STATUS_ERROR_UNKNOWN;
 }
 
-static inline int vaapi_get_colorspace_flags(vo_driver_t *this_gen)
-{
-  vaapi_driver_t      *this = (vaapi_driver_t *) this_gen;
-  ff_vaapi_context_t  *va_context = this->va_context;
+static void vaapi_update_csc (vaapi_driver_t *that, vaapi_frame_t *frame) {
+  int color_matrix;
 
-  if(!va_context)
-    return 0;
+  color_matrix = cm_from_frame (&frame->vo_frame);
 
-  int colorspace = 0;
-#if USE_VAAPI_COLORSPACE
-  switch (va_context->va_colorspace) {
-    case 0:
-      colorspace = ((va_context->sw_width >= 1280 || va_context->sw_height > 576) ?
-               VA_SRC_BT709 : VA_SRC_BT601);
-      break;
-    case 1:
-      colorspace = VA_SRC_BT601;
-      break;
-    case 2:
-      colorspace = VA_SRC_BT709;
-      break;
-    case 3:
-      colorspace = VA_SRC_SMPTE_240;
-      break;
-    default:
-      colorspace = VA_SRC_BT601;
-      break;
-  }
+  if (that->update_csc || that->color_matrix != color_matrix) {
+    int brightness = that->props[VO_PROP_BRIGHTNESS].value;
+    int contrast   = that->props[VO_PROP_CONTRAST].value;
+    int saturation = that->props[VO_PROP_SATURATION].value;
+    int hue        = that->props[VO_PROP_HUE].value;
+    VADisplayAttribute attr;
+
+#if defined(VA_SRC_BT601)
+    switch (color_matrix >> 1) {
+#if defined(VA_SRC_BT709)
+      case 1:  that->vaapi_cm_flags = VA_SRC_BT709;     break;
+#elif defined(VA_SRC_SMPTE_240)
+      case 1:  that->vaapi_cm_flags = VA_SRC_SMPTE_240; break;
+#endif
+#if defined(VA_SRC_SMPTE_240)
+      case 7:  that->vaapi_cm_flags = VA_SRC_SMPTE_240; break;
+#elif defined(VA_SRC_BT709)
+      case 7:  that->vaapi_cm_flags = VA_SRC_BT709;     break;
 #endif
-    return colorspace;
+      default: that->vaapi_cm_flags = VA_SRC_BT601;
+    }
+#endif
+
+    if (color_matrix & 1) {
+      int a, b;
+      /* fullrange mode. XXX assuming TV set style bcs controls 0% - 200% */
+      saturation -= that->props[VO_PROP_SATURATION].min;
+      saturation  = (saturation * (112 * 255) + (127 * 219 / 2)) / (127 * 219);
+      saturation += that->props[VO_PROP_SATURATION].min;
+      if (saturation > that->props[VO_PROP_SATURATION].max)
+        saturation = that->props[VO_PROP_SATURATION].max;
+
+      contrast -= that->props[VO_PROP_CONTRAST].min;
+      contrast  = (contrast * 219 + 127) / 255;
+      a         = contrast * (that->props[VO_PROP_BRIGHTNESS].max - that->props[VO_PROP_BRIGHTNESS].min);
+      contrast += that->props[VO_PROP_CONTRAST].min;
+      b         = 256 * (that->props[VO_PROP_CONTRAST].max - that->props[VO_PROP_CONTRAST].min);
+
+      brightness += (16 * a + b / 2) / b;
+      if (brightness > that->props[VO_PROP_BRIGHTNESS].max)
+        brightness = that->props[VO_PROP_BRIGHTNESS].max;
+    }
+
+    if (that->props[VO_PROP_BRIGHTNESS].atom) {
+      attr.type  = that->props[VO_PROP_BRIGHTNESS].type;
+      attr.value = brightness;
+      vaSetDisplayAttributes (that->va_context->va_display, &attr, 1);
+    }
+
+    if (that->props[VO_PROP_CONTRAST].atom) {
+      attr.type  = that->props[VO_PROP_CONTRAST].type;
+      attr.value = contrast;
+      vaSetDisplayAttributes (that->va_context->va_display, &attr, 1);
+    }
+
+    if (that->props[VO_PROP_SATURATION].atom) {
+      attr.type  = that->props[VO_PROP_SATURATION].type;
+      attr.value = saturation;
+      vaSetDisplayAttributes (that->va_context->va_display, &attr, 1);
+    }
+
+    if (that->props[VO_PROP_HUE].atom) {
+      attr.type  = that->props[VO_PROP_HUE].type;
+      attr.value = hue;
+      vaSetDisplayAttributes (that->va_context->va_display, &attr, 1);
+    }
+
+    that->color_matrix = color_matrix;
+    that->update_csc = 0;
+
+    xprintf (that->xine, XINE_VERBOSITY_LOG,"video_out_vaapi: %s b %d c %d s %d h %d [%s]\n",
+      color_matrix & 1 ? "modified" : "",
+      brightness, contrast, saturation, hue, cm_names[color_matrix]);
+  }
 }
 
 static void vaapi_property_callback (void *property_gen, xine_cfg_entry_t *entry) {
@@ -2615,6 +2671,9 @@ static int vaapi_redraw_needed (vo_drive
     ret = 1;
   }
 
+  if (this->update_csc)
+    ret = 1;
+
   return ret;
 }
 
@@ -3229,7 +3288,8 @@ static VAStatus vaapi_hardware_render_fr
   for(i = 0; i <= !!((deint > 1) && interlaced_frame); i++) {
     unsigned int flags = (deint && (interlaced_frame) ? (((!!(top_field_first)) ^ i) == 0 ? VA_BOTTOM_FIELD : VA_TOP_FIELD) : VA_FRAME_PICTURE);
 
-    //flags |= vaapi_get_colorspace_flags(this_gen);
+    vaapi_update_csc (this, frame);
+    flags |= this->vaapi_cm_flags;
 
     flags |= VA_CLEAR_DRAWABLE;
     flags |= this->scaling_level;
@@ -3524,6 +3584,8 @@ static void vaapi_display_frame (vo_driv
 static int vaapi_get_property (vo_driver_t *this_gen, int property) {
   vaapi_driver_t *this = (vaapi_driver_t *) this_gen;
 
+  if ((property < 0) || (property >= VO_NUM_PROPERTIES)) return 0;
+
   switch (property) {
     case VO_PROP_WINDOW_WIDTH:
       this->props[property].value = this->sc.gui_width;
@@ -3563,6 +3625,20 @@ static int vaapi_set_property (vo_driver
 
   lprintf("vaapi_set_property property=%d, value=%d\n", property, value );
 
+  if ((property < 0) || (property >= VO_NUM_PROPERTIES)) return 0;
+
+  if ((property == VO_PROP_BRIGHTNESS)
+    || (property == VO_PROP_CONTRAST)
+    || (property == VO_PROP_SATURATION)
+    || (property == VO_PROP_HUE)) {
+    /* defer these to vaapi_update_csc () */
+    if((value < this->props[property].min) || (value > this->props[property].max))
+      value = (this->props[property].min + this->props[property].max) >> 1;
+    this->props[property].value = value;
+    this->update_csc = 1;
+    return value;
+  }
+
   if(this->props[property].atom) {
     VADisplayAttribute attr;
 
@@ -3723,6 +3799,8 @@ static void vaapi_dispose_locked (vo_dri
   pthread_mutex_unlock(&this->vaapi_lock);
   pthread_mutex_destroy(&this->vaapi_lock);
 
+  cm_close (this);
+
   free (this);
 }
 
@@ -3981,6 +4059,9 @@ static vo_driver_t *vaapi_open_plugin (v
     this->props[i].this  = this;
   }
 
+  cm_init (this);
+  this->update_csc = 1;
+
   this->sc.user_ratio                        =
     this->props[VO_PROP_ASPECT_RATIO].value  = XINE_VO_ASPECT_AUTO;
   this->props[VO_PROP_ZOOM_X].value          = 100;
@@ -3999,6 +4080,12 @@ static vo_driver_t *vaapi_open_plugin (v
 
   pthread_mutex_unlock(&this->vaapi_lock);
 
+#if (defined VA_SRC_BT601) && ((defined VA_SRC_BT709) || (defined VA_SRC_SMPTE_240))
+  this->capabilities |= VO_CAP_COLOR_MATRIX;
+#endif
+  if ((this->capabilities & (VO_CAP_BRIGHTNESS | VO_CAP_CONTRAST)) == (VO_CAP_BRIGHTNESS | VO_CAP_CONTRAST))
+    this->capabilities |= VO_CAP_FULLRANGE;
+
   xprintf(this->xine, XINE_VERBOSITY_LOG, LOG_MODULE " vaapi_open: Deinterlace : %d\n", this->deinterlace);
   xprintf(this->xine, XINE_VERBOSITY_LOG, LOG_MODULE " vaapi_open: Render surfaces : %d\n", RENDER_SURFACES);
   xprintf(this->xine, XINE_VERBOSITY_LOG, LOG_MODULE " vaapi_open: Opengl render : %d\n", this->opengl_render);