radeonfb: accelerate imageblit and other improvements

Implement support for HW color expansion of 1bpp images, along with some
improvements to the FIFO handling and other accel operations.

The offset fixup code is now unnecessary as the fbcon core will call our
set_par upon switch back from KD_GRAPHICS before anything else happens.  I
removed it as it would slow down accel operations.

The fifo wait has been improved to avoid hitting the HW register as often,
and the various accel ops are now performing better caching of register
values.

Overall, this improve accel performances.  The imageblit acceleration does
result in a small overall regression in performances on some machines (on
the order of 5% on some x86), probably becaus the SW path provides a
better bus utilisation, but I decided to ingnore that as the performances
is still very good, and on the other hand, some machines such as some
sparc64 get a 3 fold performance improvement.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: David S. Miller <davem@davemloft.net>
Cc: Krzysztof Halasa <khc@pm.waw.pl>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

drivers/video/aty/radeon_backlight.c

@@ -66,7 +66,7 @@ static int radeon_bl_update_status(struct backlight_device *bd)
 		level = bd->props.brightness;
 
 	del_timer_sync(&rinfo->lvds_timer);
-	radeon_engine_idle();
+	radeon_engine_idle(rinfo);
 
 	lvds_gen_cntl = INREG(LVDS_GEN_CNTL);
 	if (level > 0) {

drivers/video/aty/radeon_base.c

@@ -852,7 +852,6 @@ static int radeonfb_pan_display (struct fb_var_screeninfo *var,
         if (rinfo->asleep)
         	return 0;
 
-	radeon_fifo_wait(2);
         OUTREG(CRTC_OFFSET, ((var->yoffset * var->xres_virtual + var->xoffset)
 			     * var->bits_per_pixel / 8) & ~7);
         return 0;
@@ -882,7 +881,6 @@ static int radeonfb_ioctl (struct fb_info *info, unsigned int cmd,
 			if (rc)
 				return rc;
 
-			radeon_fifo_wait(2);
 			if (value & 0x01) {
 				tmp = INREG(LVDS_GEN_CNTL);
 
@@ -940,7 +938,7 @@ int radeon_screen_blank(struct radeonfb_info *rinfo, int blank, int mode_switch)
 	if (rinfo->lock_blank)
 		return 0;
 
-	radeon_engine_idle();
+	radeon_engine_idle(rinfo);
 
 	val = INREG(CRTC_EXT_CNTL);
         val &= ~(CRTC_DISPLAY_DIS | CRTC_HSYNC_DIS |
@@ -1048,7 +1046,7 @@ static int radeonfb_blank (int blank, struct fb_info *info)
 
 	if (rinfo->asleep)
 		return 0;
-		
+
 	return radeon_screen_blank(rinfo, blank, 0);
 }
 
@@ -1074,8 +1072,6 @@ static int radeon_setcolreg (unsigned regno, unsigned red, unsigned green,
         pindex = regno;
 
         if (!rinfo->asleep) {
-		radeon_fifo_wait(9);
-
 		if (rinfo->bpp == 16) {
 			pindex = regno * 8;
 
@@ -1244,8 +1240,6 @@ static void radeon_write_pll_regs(struct radeonfb_info *rinfo, struct radeon_reg
 {
 	int i;
 
-	radeon_fifo_wait(20);
-
 	/* Workaround from XFree */
 	if (rinfo->is_mobility) {
 	        /* A temporal workaround for the occational blanking on certain laptop
@@ -1341,7 +1335,7 @@ static void radeon_lvds_timer_func(unsigned long data)
 {
 	struct radeonfb_info *rinfo = (struct radeonfb_info *)data;
 
-	radeon_engine_idle();
+	radeon_engine_idle(rinfo);
 
 	OUTREG(LVDS_GEN_CNTL, rinfo->pending_lvds_gen_cntl);
 }
@@ -1359,10 +1353,11 @@ void radeon_write_mode (struct radeonfb_info *rinfo, struct radeon_regs *mode,
 	if (nomodeset)
 		return;
 
+	radeon_engine_idle(rinfo);
+
 	if (!regs_only)
 		radeon_screen_blank(rinfo, FB_BLANK_NORMAL, 0);
 
-	radeon_fifo_wait(31);
 	for (i=0; i<10; i++)
 		OUTREG(common_regs[i].reg, common_regs[i].val);
 
@@ -1390,7 +1385,6 @@ void radeon_write_mode (struct radeonfb_info *rinfo, struct radeon_regs *mode,
 	radeon_write_pll_regs(rinfo, mode);
 
 	if ((primary_mon == MT_DFP) || (primary_mon == MT_LCD)) {
-		radeon_fifo_wait(10);
 		OUTREG(FP_CRTC_H_TOTAL_DISP, mode->fp_crtc_h_total_disp);
 		OUTREG(FP_CRTC_V_TOTAL_DISP, mode->fp_crtc_v_total_disp);
 		OUTREG(FP_H_SYNC_STRT_WID, mode->fp_h_sync_strt_wid);
@@ -1405,7 +1399,6 @@ void radeon_write_mode (struct radeonfb_info *rinfo, struct radeon_regs *mode,
 	if (!regs_only)
 		radeon_screen_blank(rinfo, FB_BLANK_UNBLANK, 0);
 
-	radeon_fifo_wait(2);
 	OUTPLL(VCLK_ECP_CNTL, mode->vclk_ecp_cntl);
 	
 	return;
@@ -1556,7 +1549,7 @@ static int radeonfb_set_par(struct fb_info *info)
 	/* We always want engine to be idle on a mode switch, even
 	 * if we won't actually change the mode
 	 */
-	radeon_engine_idle();
+	radeon_engine_idle(rinfo);
 
 	hSyncStart = mode->xres + mode->right_margin;
 	hSyncEnd = hSyncStart + mode->hsync_len;
@@ -1851,7 +1844,6 @@ static int radeonfb_set_par(struct fb_info *info)
 	return 0;
 }
 
-
 static struct fb_ops radeonfb_ops = {
 	.owner			= THIS_MODULE,
 	.fb_check_var		= radeonfb_check_var,
@@ -1875,6 +1867,7 @@ static int __devinit radeon_set_fbinfo (struct radeonfb_info *rinfo)
 	info->par = rinfo;
 	info->pseudo_palette = rinfo->pseudo_palette;
 	info->flags = FBINFO_DEFAULT
+		    | FBINFO_HWACCEL_IMAGEBLIT
 		    | FBINFO_HWACCEL_COPYAREA
 		    | FBINFO_HWACCEL_FILLRECT
 		    | FBINFO_HWACCEL_XPAN
@@ -2006,7 +1999,6 @@ static void radeon_identify_vram(struct radeonfb_info *rinfo)
           u32 tom = INREG(NB_TOM);
           tmp = ((((tom >> 16) - (tom & 0xffff) + 1) << 6) * 1024);
 
- 		radeon_fifo_wait(6);
           OUTREG(MC_FB_LOCATION, tom);
           OUTREG(DISPLAY_BASE_ADDR, (tom & 0xffff) << 16);
           OUTREG(CRTC2_DISPLAY_BASE_ADDR, (tom & 0xffff) << 16);

drivers/video/aty/radeon_pm.c

@@ -2653,9 +2653,9 @@ int radeonfb_pci_suspend(struct pci_dev *pdev, pm_message_t mesg)
 
 	if (!(info->flags & FBINFO_HWACCEL_DISABLED)) {
 		/* Make sure engine is reset */
-		radeon_engine_idle();
+		radeon_engine_idle(rinfo);
 		radeonfb_engine_reset(rinfo);
-		radeon_engine_idle();
+		radeon_engine_idle(rinfo);
 	}
 
 	/* Blank display and LCD */
@@ -2767,7 +2767,7 @@ int radeonfb_pci_resume(struct pci_dev *pdev)
 
 		rinfo->asleep = 0;
 	} else
-		radeon_engine_idle();
+		radeon_engine_idle(rinfo);
 
 	/* Restore display & engine */
 	radeon_write_mode (rinfo, &rinfo->state, 1);

drivers/video/aty/radeonfb.h

@@ -336,7 +336,15 @@ struct radeonfb_info {
 	int			mon2_type;
 	u8		        *mon2_EDID;
 
-	u32			dp_gui_master_cntl;
+	/* accel bits */
+	u32			dp_gui_mc_base;
+	u32			dp_gui_mc_cache;
+	u32			dp_cntl_cache;
+	u32			dp_brush_fg_cache;
+	u32			dp_brush_bg_cache;
+	u32			dp_src_fg_cache;
+	u32			dp_src_bg_cache;
+	u32			fifo_free;
 
 	struct pll_info		pll;
 
@@ -348,6 +356,7 @@ struct radeonfb_info {
 	int			lock_blank;
 	int			dynclk;
 	int			no_schedule;
+	int 			gfx_mode;
 	enum radeon_pm_mode	pm_mode;
 	reinit_function_ptr     reinit_func;
 
@@ -392,8 +401,14 @@ static inline void _radeon_msleep(struct radeonfb_info *rinfo, unsigned long ms)
 #define OUTREG8(addr,val)	writeb(val, (rinfo->mmio_base)+addr)
 #define INREG16(addr)		readw((rinfo->mmio_base)+addr)
 #define OUTREG16(addr,val)	writew(val, (rinfo->mmio_base)+addr)
+
+#ifdef CONFIG_PPC
+#define INREG(addr)	     	({ eieio(); ld_le32(rinfo->mmio_base+(addr)); })
+#define OUTREG(addr,val)	do { eieio(); st_le32(rinfo->mmio_base+(addr),(val)); } while(0)
+#else
 #define INREG(addr)		readl((rinfo->mmio_base)+addr)
 #define OUTREG(addr,val)	writel(val, (rinfo->mmio_base)+addr)
+#endif
 
 static inline void _OUTREGP(struct radeonfb_info *rinfo, u32 addr,
 		       u32 val, u32 mask)
@@ -535,17 +550,7 @@ static inline u32 radeon_get_dstbpp(u16 depth)
  * 2D Engine helper routines
  */
 
-static inline void _radeon_fifo_wait(struct radeonfb_info *rinfo, int entries)
-{
-	int i;
-
-	for (i=0; i<2000000; i++) {
-		if ((INREG(RBBM_STATUS) & 0x7f) >= entries)
-			return;
-		udelay(1);
-	}
-	printk(KERN_ERR "radeonfb: FIFO Timeout !\n");
-}
+extern void radeon_fifo_update_and_wait(struct radeonfb_info *rinfo, int entries);
 
 static inline void radeon_engine_flush (struct radeonfb_info *rinfo)
 {
@@ -558,7 +563,7 @@ static inline void radeon_engine_flush (struct radeonfb_info *rinfo)
 	/* Ensure FIFO is empty, ie, make sure the flush commands
 	 * has reached the cache
 	 */
-	_radeon_fifo_wait (rinfo, 64);
+	radeon_fifo_update_and_wait(rinfo, 64);
 
 	/* Wait for the flush to complete */
 	for (i=0; i < 2000000; i++) {
@@ -570,12 +575,12 @@ static inline void radeon_engine_flush (struct radeonfb_info *rinfo)
 }
 
 
-static inline void _radeon_engine_idle(struct radeonfb_info *rinfo)
+static inline void radeon_engine_idle(struct radeonfb_info *rinfo)
 {
 	int i;
 
 	/* ensure FIFO is empty before waiting for idle */
-	_radeon_fifo_wait (rinfo, 64);
+	radeon_fifo_update_and_wait (rinfo, 64);
 
 	for (i=0; i<2000000; i++) {
 		if (((INREG(RBBM_STATUS) & GUI_ACTIVE)) == 0) {
@@ -588,8 +593,6 @@ static inline void _radeon_engine_idle(struct radeonfb_info *rinfo)
 }
 
 
-#define radeon_engine_idle()		_radeon_engine_idle(rinfo)
-#define radeon_fifo_wait(entries)	_radeon_fifo_wait(rinfo,entries)
 #define radeon_msleep(ms)		_radeon_msleep(rinfo,ms)
 
 
@@ -619,6 +622,7 @@ extern void radeonfb_imageblit(struct fb_info *p, const struct fb_image *image);
 extern int radeonfb_sync(struct fb_info *info);
 extern void radeonfb_engine_init (struct radeonfb_info *rinfo);
 extern void radeonfb_engine_reset(struct radeonfb_info *rinfo);
+extern void radeon_fixup_mem_offset(struct radeonfb_info *rinfo);
 
 /* Other functions */
 extern int radeon_screen_blank(struct radeonfb_info *rinfo, int blank, int mode_switch);

include/video/radeon.h

@@ -525,6 +525,9 @@
 #define CRTC_DISPLAY_DIS			   (1 << 10)
 #define CRTC_CRT_ON				   (1 << 15)
 
+/* DSTCACHE_MODE bits constants */
+#define RB2D_DC_AUTOFLUSH_ENABLE                   (1 << 8)
+#define RB2D_DC_DC_DISABLE_IGNORE_PE               (1 << 17)
 
 /* DSTCACHE_CTLSTAT bit constants */
 #define RB2D_DC_FLUSH_2D			   (1 << 0)
@@ -866,15 +869,10 @@
 #define GMC_DST_16BPP_YVYU422                      0x00000c00
 #define GMC_DST_32BPP_AYUV444                      0x00000e00
 #define GMC_DST_16BPP_ARGB4444                     0x00000f00
-#define GMC_SRC_MONO                               0x00000000
-#define GMC_SRC_MONO_LBKGD                         0x00001000
-#define GMC_SRC_DSTCOLOR                           0x00003000
 #define GMC_BYTE_ORDER_MSB_TO_LSB                  0x00000000
 #define GMC_BYTE_ORDER_LSB_TO_MSB                  0x00004000
 #define GMC_DP_CONVERSION_TEMP_9300                0x00008000
 #define GMC_DP_CONVERSION_TEMP_6500                0x00000000
-#define GMC_DP_SRC_RECT                            0x02000000
-#define GMC_DP_SRC_HOST                            0x03000000
 #define GMC_DP_SRC_HOST_BYTEALIGN                  0x04000000
 #define GMC_3D_FCN_EN_CLR                          0x00000000
 #define GMC_3D_FCN_EN_SET                          0x08000000
@@ -885,6 +883,9 @@
 #define GMC_WRITE_MASK_LEAVE                       0x00000000
 #define GMC_WRITE_MASK_SET                         0x40000000
 #define GMC_CLR_CMP_CNTL_DIS      		   (1 << 28)
+#define GMC_SRC_DATATYPE_MASK			   (3 << 12)
+#define GMC_SRC_DATATYPE_MONO_FG_BG		   (0 << 12)
+#define GMC_SRC_DATATYPE_MONO_FG_LA		   (1 << 12)
 #define GMC_SRC_DATATYPE_COLOR			   (3 << 12)
 #define ROP3_S                			   0x00cc0000
 #define ROP3_SRCCOPY				   0x00cc0000
@@ -893,6 +894,7 @@
 #define DP_SRC_SOURCE_MASK        		   (7    << 24)
 #define GMC_BRUSH_NONE            		   (15   <<  4)
 #define DP_SRC_SOURCE_MEMORY			   (2    << 24)
+#define DP_SRC_SOURCE_HOST_DATA			   (3    << 24)
 #define GMC_BRUSH_SOLIDCOLOR			   0x000000d0
 
 /* DP_MIX bit constants */
@@ -978,6 +980,12 @@
 #define DISP_PWR_MAN_TV_ENABLE_RST                 (1 << 25)
 #define DISP_PWR_MAN_AUTO_PWRUP_EN                 (1 << 26)
 
+/* RBBM_GUICNTL constants */
+#define RBBM_GUICNTL_HOST_DATA_SWAP_NONE	   (0 << 0)
+#define RBBM_GUICNTL_HOST_DATA_SWAP_16BIT          (1 << 0)
+#define RBBM_GUICNTL_HOST_DATA_SWAP_32BIT	   (2 << 0)
+#define RBBM_GUICNTL_HOST_DATA_SWAP_HDW		   (3 << 0)
+
 /* masks */
 
 #define CONFIG_MEMSIZE_MASK		0x1f000000