drm/vmwgfx: Prune modes based on available VRAM size

This needs to be reviewed once we support screen objects and don't rely
on VRAM for the frame-buffer.

Also fix some integer overflow issues pointed out by Michel Daenzer.
Signed-off-by: Thomas Hellstrom <thellstrom@vmware.com>
Signed-off-by: Dave Airlie <airlied@redhat.com>

drivers/gpu/drm/vmwgfx/vmwgfx_drv.h

@@ -522,6 +522,9 @@ void vmw_kms_write_svga(struct vmw_private *vmw_priv,
 int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file_priv);
 void vmw_kms_idle_workqueues(struct vmw_master *vmaster);
+bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv,
+				uint32_t pitch,
+				uint32_t height);
 u32 vmw_get_vblank_counter(struct drm_device *dev, int crtc);
 
 /**

drivers/gpu/drm/vmwgfx/vmwgfx_fb.c

@@ -144,6 +144,13 @@ static int vmw_fb_check_var(struct fb_var_screeninfo *var,
 		return -EINVAL;
 	}
 
+	if (!vmw_kms_validate_mode_vram(vmw_priv,
+					info->fix.line_length,
+					var->yoffset + var->yres)) {
+		DRM_ERROR("Requested geom can not fit in framebuffer\n");
+		return -EINVAL;
+	}
+
 	return 0;
 }
 

drivers/gpu/drm/vmwgfx/vmwgfx_kms.c

@@ -838,7 +838,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	struct vmw_framebuffer *vfb = NULL;
 	struct vmw_surface *surface = NULL;
 	struct vmw_dma_buffer *bo = NULL;
-	unsigned int required_size;
+	u64 required_size;
 	int ret;
 
 	/**
@@ -848,7 +848,7 @@ static struct drm_framebuffer *vmw_kms_fb_create(struct drm_device *dev,
 	 */
 
 	required_size = mode_cmd->pitch * mode_cmd->height;
-	if (unlikely(required_size > dev_priv->vram_size)) {
+	if (unlikely(required_size > (u64) dev_priv->vram_size)) {
 		DRM_ERROR("VRAM size is too small for requested mode.\n");
 		return NULL;
 	}
@@ -1133,6 +1133,13 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 	return ret;
 }
 
+bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv,
+				uint32_t pitch,
+				uint32_t height)
+{
+	return ((u64) pitch * (u64) height) < (u64) dev_priv->vram_size;
+}
+
 u32 vmw_get_vblank_counter(struct drm_device *dev, int crtc)
 {
 	return 0;

drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c

@@ -427,7 +427,9 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
 {
 	struct vmw_legacy_display_unit *ldu = vmw_connector_to_ldu(connector);
 	struct drm_device *dev = connector->dev;
+	struct vmw_private *dev_priv = vmw_priv(dev);
 	struct drm_display_mode *mode = NULL;
+	struct drm_display_mode *bmode;
 	struct drm_display_mode prefmode = { DRM_MODE("preferred",
 		DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
 		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -443,22 +445,30 @@ static int vmw_ldu_connector_fill_modes(struct drm_connector *connector,
 		mode->hdisplay = ldu->pref_width;
 		mode->vdisplay = ldu->pref_height;
 		mode->vrefresh = drm_mode_vrefresh(mode);
-		drm_mode_probed_add(connector, mode);
+		if (vmw_kms_validate_mode_vram(dev_priv, mode->hdisplay * 2,
+					       mode->vdisplay)) {
+			drm_mode_probed_add(connector, mode);
 
-		if (ldu->pref_mode) {
-			list_del_init(&ldu->pref_mode->head);
-			drm_mode_destroy(dev, ldu->pref_mode);
-		}
+			if (ldu->pref_mode) {
+				list_del_init(&ldu->pref_mode->head);
+				drm_mode_destroy(dev, ldu->pref_mode);
+			}
 
-		ldu->pref_mode = mode;
+			ldu->pref_mode = mode;
+		}
 	}
 
 	for (i = 0; vmw_ldu_connector_builtin[i].type != 0; i++) {
-		if (vmw_ldu_connector_builtin[i].hdisplay > max_width ||
-		    vmw_ldu_connector_builtin[i].vdisplay > max_height)
+		bmode = &vmw_ldu_connector_builtin[i];
+		if (bmode->hdisplay > max_width ||
+		    bmode->vdisplay > max_height)
+			continue;
+
+		if (!vmw_kms_validate_mode_vram(dev_priv, bmode->hdisplay * 2,
+						bmode->vdisplay))
 			continue;
 
-		mode = drm_mode_duplicate(dev, &vmw_ldu_connector_builtin[i]);
+		mode = drm_mode_duplicate(dev, bmode);
 		if (!mode)
 			return 0;
 		mode->vrefresh = drm_mode_vrefresh(mode);