drm/amdgpu: workaround the TMR MC address issue (v2)

With the 2-level gart page table,  vram is squeezed into gart aperture
and FB aperture is disabled. Therefore all VRAM virtual addresses are
 in the GART aperture. However currently PSP requires TMR addresses
in FB aperture. So we need some design change at PSP FW level to support
this 2-level gart table driver change. Right now this PSP FW support
doesn't exist. To workaround this issue temporarily, FB aperture is
added back and the gart aperture address is converted back to FB aperture
for this PSP TMR address.

Will revert it after we get a fix from PSP FW.

v2: squash in tmr fix for other asics (Kevin)
Signed-off-by: Oak Zeng <Oak.Zeng@amd.com>
Reviewed-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h

@@ -208,6 +208,15 @@ struct amdgpu_gmc {
 	 */
 	u64			fb_start;
 	u64			fb_end;
+	/* In the case of use GART table for vmid0 FB access, [fb_start, fb_end]
+	 * will be squeezed to GART aperture. But we have a PSP FW issue to fix
+	 * for now. To temporarily workaround the PSP FW issue, added below two
+	 * variables to remember the original fb_start/end to re-enable FB
+	 * aperture to workaround the PSP FW issue. Will delete it after we
+	 * get a proper PSP FW fix.
+	 */
+	u64			fb_start_original;
+	u64			fb_end_original;
 	unsigned		vram_width;
 	u64			real_vram_size;
 	int			vram_mtrr;

drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

@@ -407,6 +407,16 @@ static int psp_tmr_init(struct psp_context *psp)
 				      AMDGPU_GEM_DOMAIN_VRAM,
 				      &psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 
+	/* workaround the tmr_mc_addr:
+	 * PSP requires an address in FB aperture. Right now driver produce
+	 * tmr_mc_addr in the GART aperture. Convert it back to FB aperture
+	 * for PSP. Will revert it after we get a fix from PSP FW.
+	 */
+	if (psp->adev->asic_type == CHIP_ALDEBARAN) {
+		psp->tmr_mc_addr -= psp->adev->gmc.fb_start;
+		psp->tmr_mc_addr += psp->adev->gmc.fb_start_original;
+	}
+
 	return ret;
 }
 

drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c

@@ -141,12 +141,21 @@ static void gfxhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
 	 * FB aperture and AGP aperture. Disable them.
 	 */
 	if (adev->gmc.pdb0_bo) {
-		WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_TOP, 0);
-		WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_BASE, 0x00FFFFFF);
-		WREG32_SOC15(GC, 0, mmMC_VM_AGP_TOP, 0);
-		WREG32_SOC15(GC, 0, mmMC_VM_AGP_BOT, 0xFFFFFF);
-		WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, 0x3FFFFFFF);
-		WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0);
+		if (adev->asic_type == CHIP_ALDEBARAN) {
+			WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_TOP, adev->gmc.fb_end_original >> 24);
+			WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_BASE, adev->gmc.fb_start_original >> 24);
+			WREG32_SOC15(GC, 0, mmMC_VM_AGP_TOP, 0);
+			WREG32_SOC15(GC, 0, mmMC_VM_AGP_BOT, 0xFFFFFF);
+			WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, adev->gmc.fb_start_original >> 18);
+			WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, adev->gmc.fb_end_original >> 18);
+		} else {
+			WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_TOP, 0);
+			WREG32_SOC15(GC, 0, mmMC_VM_FB_LOCATION_BASE, 0x00FFFFFF);
+			WREG32_SOC15(GC, 0, mmMC_VM_AGP_TOP, 0);
+			WREG32_SOC15(GC, 0, mmMC_VM_AGP_BOT, 0xFFFFFF);
+			WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR, 0x3FFFFFFF);
+			WREG32_SOC15(GC, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0);
+		}
 	}
 }
 

drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c

@@ -47,6 +47,8 @@ static u64 mmhub_v1_7_get_fb_location(struct amdgpu_device *adev)
 
 	adev->gmc.fb_start = base;
 	adev->gmc.fb_end = top;
+	adev->gmc.fb_start_original = base;
+	adev->gmc.fb_end_original = top;
 
 	return base;
 }
@@ -124,10 +126,10 @@ static void mmhub_v1_7_init_system_aperture_regs(struct amdgpu_device *adev)
 	if (adev->gmc.pdb0_bo) {
 		WREG32_SOC15(MMHUB, 0, regMC_VM_AGP_BOT, 0xFFFFFF);
 		WREG32_SOC15(MMHUB, 0, regMC_VM_AGP_TOP, 0);
-		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP, 0);
-		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE, 0x00FFFFFF);
-		WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 0x3FFFFFFF);
-		WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP, adev->gmc.fb_end_original >> 24);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE, adev->gmc.fb_start_original >> 24);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, adev->gmc.fb_start_original >> 18);
+		WREG32_SOC15(MMHUB, 0, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, adev->gmc.fb_end_original >> 18);
 	}
 	if (amdgpu_sriov_vf(adev))
 		return;