drm/vmwgfx: Properly check display/scanout surface size

The scanout surface size is the smaller of max texture size and
max STDU size.
Signed-off-by: Sinclair Yeh <syeh@vmware.com>
Reviewed-by: Thomas Hellstrom <thellstrom@vmware.com>

drivers/gpu/drm/vmwgfx/vmwgfx_kms.c

@@ -2182,7 +2182,10 @@ int vmw_du_connector_fill_modes(struct drm_connector *connector,
 
 	if (dev_priv->active_display_unit == vmw_du_screen_target) {
 		max_width  = min(max_width,  dev_priv->stdu_max_width);
+		max_width  = min(max_width,  dev_priv->texture_max_width);
+
 		max_height = min(max_height, dev_priv->stdu_max_height);
+		max_height = min(max_height, dev_priv->texture_max_height);
 	}
 
 	/* Add preferred mode */

drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c

@@ -1601,7 +1601,21 @@ int vmw_kms_stdu_init_display(struct vmw_private *dev_priv)
 
 	dev_priv->active_display_unit = vmw_du_screen_target;
 
-	if (!(dev_priv->capabilities & SVGA_CAP_3D)) {
+	if (dev_priv->capabilities & SVGA_CAP_3D) {
+		/*
+		 * For 3D VMs, display (scanout) buffer size is the smaller of
+		 * max texture and max STDU
+		 */
+		uint32_t max_width, max_height;
+
+		max_width = min(dev_priv->texture_max_width,
+				dev_priv->stdu_max_width);
+		max_height = min(dev_priv->texture_max_height,
+				 dev_priv->stdu_max_height);
+
+		dev->mode_config.max_width = max_width;
+		dev->mode_config.max_height = max_height;
+	} else {
 		/*
 		 * Given various display aspect ratios, there's no way to
 		 * estimate these using prim_bb_mem.  So just set these to

drivers/gpu/drm/vmwgfx/vmwgfx_surface.c

@@ -1479,10 +1479,24 @@ int vmw_surface_gb_priv_define(struct drm_device *dev,
 	*srf_out = NULL;
 
 	if (for_scanout) {
+		uint32_t max_width, max_height;
+
 		if (!svga3dsurface_is_screen_target_format(format)) {
 			DRM_ERROR("Invalid Screen Target surface format.");
 			return -EINVAL;
 		}
+
+		max_width = min(dev_priv->texture_max_width,
+				dev_priv->stdu_max_width);
+		max_height = min(dev_priv->texture_max_height,
+				 dev_priv->stdu_max_height);
+
+		if (size.width > max_width || size.height > max_height) {
+			DRM_ERROR("%ux%u\n, exeeds max surface size %ux%u",
+				  size.width, size.height,
+				  max_width, max_height);
+			return -EINVAL;
+		}
 	} else {
 		const struct svga3d_surface_desc *desc;