Lines Matching refs:rdev
127 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
128 extern void r600_ih_ring_fini(struct radeon_device *rdev);
129 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
130 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
131 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
132 extern void sumo_rlc_fini(struct radeon_device *rdev);
133 extern int sumo_rlc_init(struct radeon_device *rdev);
134 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
135 extern void si_rlc_reset(struct radeon_device *rdev);
136 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
137 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
138 extern int cik_sdma_resume(struct radeon_device *rdev);
139 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
140 extern void cik_sdma_fini(struct radeon_device *rdev);
141 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
142 static void cik_rlc_stop(struct radeon_device *rdev);
143 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
144 static void cik_program_aspm(struct radeon_device *rdev);
145 static void cik_init_pg(struct radeon_device *rdev);
146 static void cik_init_cg(struct radeon_device *rdev);
147 static void cik_fini_pg(struct radeon_device *rdev);
148 static void cik_fini_cg(struct radeon_device *rdev);
149 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
162 int cik_get_allowed_info_register(struct radeon_device *rdev, in cik_get_allowed_info_register() argument
188 u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg) in cik_didt_rreg() argument
193 spin_lock_irqsave(&rdev->didt_idx_lock, flags); in cik_didt_rreg()
196 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); in cik_didt_rreg()
200 void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v) in cik_didt_wreg() argument
204 spin_lock_irqsave(&rdev->didt_idx_lock, flags); in cik_didt_wreg()
207 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags); in cik_didt_wreg()
211 int ci_get_temp(struct radeon_device *rdev) in ci_get_temp() argument
228 int kv_get_temp(struct radeon_device *rdev) in kv_get_temp() argument
246 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg) in cik_pciep_rreg() argument
251 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); in cik_pciep_rreg()
255 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); in cik_pciep_rreg()
259 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v) in cik_pciep_wreg() argument
263 spin_lock_irqsave(&rdev->pciep_idx_lock, flags); in cik_pciep_wreg()
268 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags); in cik_pciep_wreg()
1628 static void cik_init_golden_registers(struct radeon_device *rdev) in cik_init_golden_registers() argument
1630 switch (rdev->family) { in cik_init_golden_registers()
1632 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1635 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1638 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1641 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1646 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1649 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1652 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1655 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1660 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1663 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1666 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1669 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1674 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1677 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1680 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1683 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1688 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1691 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1694 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1697 radeon_program_register_sequence(rdev, in cik_init_golden_registers()
1714 u32 cik_get_xclk(struct radeon_device *rdev) in cik_get_xclk() argument
1716 u32 reference_clock = rdev->clock.spll.reference_freq; in cik_get_xclk()
1718 if (rdev->flags & RADEON_IS_IGP) { in cik_get_xclk()
1737 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index) in cik_mm_rdoorbell() argument
1739 if (index < rdev->doorbell.num_doorbells) { in cik_mm_rdoorbell()
1740 return readl(rdev->doorbell.ptr + index); in cik_mm_rdoorbell()
1757 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v) in cik_mm_wdoorbell() argument
1759 if (index < rdev->doorbell.num_doorbells) { in cik_mm_wdoorbell()
1760 writel(v, rdev->doorbell.ptr + index); in cik_mm_wdoorbell()
1850 static void cik_srbm_select(struct radeon_device *rdev, in cik_srbm_select() argument
1869 int ci_mc_load_microcode(struct radeon_device *rdev) in ci_mc_load_microcode() argument
1878 if (!rdev->mc_fw) in ci_mc_load_microcode()
1881 if (rdev->new_fw) { in ci_mc_load_microcode()
1883 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data; in ci_mc_load_microcode()
1889 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes)); in ci_mc_load_microcode()
1892 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in ci_mc_load_microcode()
1894 ucode_size = rdev->mc_fw->size / 4; in ci_mc_load_microcode()
1896 switch (rdev->family) { in ci_mc_load_microcode()
1908 fw_data = (const __be32 *)rdev->mc_fw->data; in ci_mc_load_microcode()
1920 if (rdev->new_fw) { in ci_mc_load_microcode()
1930 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) { in ci_mc_load_microcode()
1939 if (rdev->new_fw) in ci_mc_load_microcode()
1951 for (i = 0; i < rdev->usec_timeout; i++) { in ci_mc_load_microcode()
1956 for (i = 0; i < rdev->usec_timeout; i++) { in ci_mc_load_microcode()
1975 static int cik_init_microcode(struct radeon_device *rdev) in cik_init_microcode() argument
1990 switch (rdev->family) { in cik_init_microcode()
1993 if ((rdev->pdev->revision == 0x80) || in cik_init_microcode()
1994 (rdev->pdev->revision == 0x81) || in cik_init_microcode()
1995 (rdev->pdev->device == 0x665f)) in cik_init_microcode()
2011 if (rdev->pdev->revision == 0x80) in cik_init_microcode()
2064 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); in cik_init_microcode()
2067 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev); in cik_init_microcode()
2070 if (rdev->pfp_fw->size != pfp_req_size) { in cik_init_microcode()
2072 rdev->pfp_fw->size, fw_name); in cik_init_microcode()
2077 err = radeon_ucode_validate(rdev->pfp_fw); in cik_init_microcode()
2088 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); in cik_init_microcode()
2091 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev); in cik_init_microcode()
2094 if (rdev->me_fw->size != me_req_size) { in cik_init_microcode()
2096 rdev->me_fw->size, fw_name); in cik_init_microcode()
2100 err = radeon_ucode_validate(rdev->me_fw); in cik_init_microcode()
2111 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); in cik_init_microcode()
2114 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev); in cik_init_microcode()
2117 if (rdev->ce_fw->size != ce_req_size) { in cik_init_microcode()
2119 rdev->ce_fw->size, fw_name); in cik_init_microcode()
2123 err = radeon_ucode_validate(rdev->ce_fw); in cik_init_microcode()
2134 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); in cik_init_microcode()
2137 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev); in cik_init_microcode()
2140 if (rdev->mec_fw->size != mec_req_size) { in cik_init_microcode()
2142 rdev->mec_fw->size, fw_name); in cik_init_microcode()
2146 err = radeon_ucode_validate(rdev->mec_fw); in cik_init_microcode()
2156 if (rdev->family == CHIP_KAVERI) { in cik_init_microcode()
2158 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev); in cik_init_microcode()
2162 err = radeon_ucode_validate(rdev->mec2_fw); in cik_init_microcode()
2172 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); in cik_init_microcode()
2175 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev); in cik_init_microcode()
2178 if (rdev->rlc_fw->size != rlc_req_size) { in cik_init_microcode()
2180 rdev->rlc_fw->size, fw_name); in cik_init_microcode()
2184 err = radeon_ucode_validate(rdev->rlc_fw); in cik_init_microcode()
2195 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); in cik_init_microcode()
2198 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev); in cik_init_microcode()
2201 if (rdev->sdma_fw->size != sdma_req_size) { in cik_init_microcode()
2203 rdev->sdma_fw->size, fw_name); in cik_init_microcode()
2207 err = radeon_ucode_validate(rdev->sdma_fw); in cik_init_microcode()
2218 if (!(rdev->flags & RADEON_IS_IGP)) { in cik_init_microcode()
2220 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in cik_init_microcode()
2223 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in cik_init_microcode()
2226 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev); in cik_init_microcode()
2230 if ((rdev->mc_fw->size != mc_req_size) && in cik_init_microcode()
2231 (rdev->mc_fw->size != mc2_req_size)){ in cik_init_microcode()
2233 rdev->mc_fw->size, fw_name); in cik_init_microcode()
2236 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size); in cik_init_microcode()
2238 err = radeon_ucode_validate(rdev->mc_fw); in cik_init_microcode()
2252 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); in cik_init_microcode()
2255 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev); in cik_init_microcode()
2259 release_firmware(rdev->smc_fw); in cik_init_microcode()
2260 rdev->smc_fw = NULL; in cik_init_microcode()
2262 } else if (rdev->smc_fw->size != smc_req_size) { in cik_init_microcode()
2264 rdev->smc_fw->size, fw_name); in cik_init_microcode()
2268 err = radeon_ucode_validate(rdev->smc_fw); in cik_init_microcode()
2280 rdev->new_fw = false; in cik_init_microcode()
2285 rdev->new_fw = true; in cik_init_microcode()
2293 release_firmware(rdev->pfp_fw); in cik_init_microcode()
2294 rdev->pfp_fw = NULL; in cik_init_microcode()
2295 release_firmware(rdev->me_fw); in cik_init_microcode()
2296 rdev->me_fw = NULL; in cik_init_microcode()
2297 release_firmware(rdev->ce_fw); in cik_init_microcode()
2298 rdev->ce_fw = NULL; in cik_init_microcode()
2299 release_firmware(rdev->mec_fw); in cik_init_microcode()
2300 rdev->mec_fw = NULL; in cik_init_microcode()
2301 release_firmware(rdev->mec2_fw); in cik_init_microcode()
2302 rdev->mec2_fw = NULL; in cik_init_microcode()
2303 release_firmware(rdev->rlc_fw); in cik_init_microcode()
2304 rdev->rlc_fw = NULL; in cik_init_microcode()
2305 release_firmware(rdev->sdma_fw); in cik_init_microcode()
2306 rdev->sdma_fw = NULL; in cik_init_microcode()
2307 release_firmware(rdev->mc_fw); in cik_init_microcode()
2308 rdev->mc_fw = NULL; in cik_init_microcode()
2309 release_firmware(rdev->smc_fw); in cik_init_microcode()
2310 rdev->smc_fw = NULL; in cik_init_microcode()
2329 static void cik_tiling_mode_table_init(struct radeon_device *rdev) in cik_tiling_mode_table_init() argument
2331 u32 *tile = rdev->config.cik.tile_mode_array; in cik_tiling_mode_table_init()
2332 u32 *macrotile = rdev->config.cik.macrotile_mode_array; in cik_tiling_mode_table_init()
2334 ARRAY_SIZE(rdev->config.cik.tile_mode_array); in cik_tiling_mode_table_init()
2336 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array); in cik_tiling_mode_table_init()
2339 u32 num_rbs = rdev->config.cik.max_backends_per_se * in cik_tiling_mode_table_init()
2340 rdev->config.cik.max_shader_engines; in cik_tiling_mode_table_init()
2342 switch (rdev->config.cik.mem_row_size_in_kb) { in cik_tiling_mode_table_init()
2355 num_pipe_configs = rdev->config.cik.max_tile_pipes; in cik_tiling_mode_table_init()
3035 static void cik_select_se_sh(struct radeon_device *rdev, in cik_select_se_sh() argument
3081 static u32 cik_get_rb_disabled(struct radeon_device *rdev, in cik_get_rb_disabled() argument
3111 static void cik_setup_rb(struct radeon_device *rdev, in cik_setup_rb() argument
3122 cik_select_se_sh(rdev, i, j); in cik_setup_rb()
3123 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se); in cik_setup_rb()
3124 if (rdev->family == CHIP_HAWAII) in cik_setup_rb()
3130 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_setup_rb()
3139 rdev->config.cik.backend_enable_mask = enabled_rbs; in cik_setup_rb()
3142 cik_select_se_sh(rdev, i, 0xffffffff); in cik_setup_rb()
3167 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_setup_rb()
3178 static void cik_gpu_init(struct radeon_device *rdev) in cik_gpu_init() argument
3186 switch (rdev->family) { in cik_gpu_init()
3188 rdev->config.cik.max_shader_engines = 2; in cik_gpu_init()
3189 rdev->config.cik.max_tile_pipes = 4; in cik_gpu_init()
3190 rdev->config.cik.max_cu_per_sh = 7; in cik_gpu_init()
3191 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3192 rdev->config.cik.max_backends_per_se = 2; in cik_gpu_init()
3193 rdev->config.cik.max_texture_channel_caches = 4; in cik_gpu_init()
3194 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3195 rdev->config.cik.max_gs_threads = 32; in cik_gpu_init()
3196 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3198 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3199 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3200 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3201 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3205 rdev->config.cik.max_shader_engines = 4; in cik_gpu_init()
3206 rdev->config.cik.max_tile_pipes = 16; in cik_gpu_init()
3207 rdev->config.cik.max_cu_per_sh = 11; in cik_gpu_init()
3208 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3209 rdev->config.cik.max_backends_per_se = 4; in cik_gpu_init()
3210 rdev->config.cik.max_texture_channel_caches = 16; in cik_gpu_init()
3211 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3212 rdev->config.cik.max_gs_threads = 32; in cik_gpu_init()
3213 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3215 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3216 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3217 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3218 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3222 rdev->config.cik.max_shader_engines = 1; in cik_gpu_init()
3223 rdev->config.cik.max_tile_pipes = 4; in cik_gpu_init()
3224 rdev->config.cik.max_cu_per_sh = 8; in cik_gpu_init()
3225 rdev->config.cik.max_backends_per_se = 2; in cik_gpu_init()
3226 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3227 rdev->config.cik.max_texture_channel_caches = 4; in cik_gpu_init()
3228 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3229 rdev->config.cik.max_gs_threads = 16; in cik_gpu_init()
3230 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3232 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3233 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3234 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3235 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3241 rdev->config.cik.max_shader_engines = 1; in cik_gpu_init()
3242 rdev->config.cik.max_tile_pipes = 2; in cik_gpu_init()
3243 rdev->config.cik.max_cu_per_sh = 2; in cik_gpu_init()
3244 rdev->config.cik.max_sh_per_se = 1; in cik_gpu_init()
3245 rdev->config.cik.max_backends_per_se = 1; in cik_gpu_init()
3246 rdev->config.cik.max_texture_channel_caches = 2; in cik_gpu_init()
3247 rdev->config.cik.max_gprs = 256; in cik_gpu_init()
3248 rdev->config.cik.max_gs_threads = 16; in cik_gpu_init()
3249 rdev->config.cik.max_hw_contexts = 8; in cik_gpu_init()
3251 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20; in cik_gpu_init()
3252 rdev->config.cik.sc_prim_fifo_size_backend = 0x100; in cik_gpu_init()
3253 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30; in cik_gpu_init()
3254 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130; in cik_gpu_init()
3277 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes; in cik_gpu_init()
3278 rdev->config.cik.mem_max_burst_length_bytes = 256; in cik_gpu_init()
3280 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; in cik_gpu_init()
3281 if (rdev->config.cik.mem_row_size_in_kb > 4) in cik_gpu_init()
3282 rdev->config.cik.mem_row_size_in_kb = 4; in cik_gpu_init()
3284 rdev->config.cik.shader_engine_tile_size = 32; in cik_gpu_init()
3285 rdev->config.cik.num_gpus = 1; in cik_gpu_init()
3286 rdev->config.cik.multi_gpu_tile_size = 64; in cik_gpu_init()
3290 switch (rdev->config.cik.mem_row_size_in_kb) { in cik_gpu_init()
3310 rdev->config.cik.tile_config = 0; in cik_gpu_init()
3311 switch (rdev->config.cik.num_tile_pipes) { in cik_gpu_init()
3313 rdev->config.cik.tile_config |= (0 << 0); in cik_gpu_init()
3316 rdev->config.cik.tile_config |= (1 << 0); in cik_gpu_init()
3319 rdev->config.cik.tile_config |= (2 << 0); in cik_gpu_init()
3324 rdev->config.cik.tile_config |= (3 << 0); in cik_gpu_init()
3327 rdev->config.cik.tile_config |= in cik_gpu_init()
3329 rdev->config.cik.tile_config |= in cik_gpu_init()
3331 rdev->config.cik.tile_config |= in cik_gpu_init()
3343 cik_tiling_mode_table_init(rdev); in cik_gpu_init()
3345 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines, in cik_gpu_init()
3346 rdev->config.cik.max_sh_per_se, in cik_gpu_init()
3347 rdev->config.cik.max_backends_per_se); in cik_gpu_init()
3349 rdev->config.cik.active_cus = 0; in cik_gpu_init()
3350 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { in cik_gpu_init()
3351 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { in cik_gpu_init()
3352 rdev->config.cik.active_cus += in cik_gpu_init()
3353 hweight32(cik_get_cu_active_bitmap(rdev, i, j)); in cik_gpu_init()
3386 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) | in cik_gpu_init()
3387 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) | in cik_gpu_init()
3388 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) | in cik_gpu_init()
3389 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size))); in cik_gpu_init()
3432 static void cik_scratch_init(struct radeon_device *rdev) in cik_scratch_init() argument
3436 rdev->scratch.num_reg = 7; in cik_scratch_init()
3437 rdev->scratch.reg_base = SCRATCH_REG0; in cik_scratch_init()
3438 for (i = 0; i < rdev->scratch.num_reg; i++) { in cik_scratch_init()
3439 rdev->scratch.free[i] = true; in cik_scratch_init()
3440 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4); in cik_scratch_init()
3455 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring) in cik_ring_test() argument
3462 r = radeon_scratch_get(rdev, &scratch); in cik_ring_test()
3468 r = radeon_ring_lock(rdev, ring, 3); in cik_ring_test()
3471 radeon_scratch_free(rdev, scratch); in cik_ring_test()
3477 radeon_ring_unlock_commit(rdev, ring, false); in cik_ring_test()
3479 for (i = 0; i < rdev->usec_timeout; i++) { in cik_ring_test()
3485 if (i < rdev->usec_timeout) { in cik_ring_test()
3492 radeon_scratch_free(rdev, scratch); in cik_ring_test()
3504 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev, in cik_hdp_flush_cp_ring_emit() argument
3507 struct radeon_ring *ring = &rdev->ring[ridx]; in cik_hdp_flush_cp_ring_emit()
3550 void cik_fence_gfx_ring_emit(struct radeon_device *rdev, in cik_fence_gfx_ring_emit() argument
3553 struct radeon_ring *ring = &rdev->ring[fence->ring]; in cik_fence_gfx_ring_emit()
3554 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; in cik_fence_gfx_ring_emit()
3591 void cik_fence_compute_ring_emit(struct radeon_device *rdev, in cik_fence_compute_ring_emit() argument
3594 struct radeon_ring *ring = &rdev->ring[fence->ring]; in cik_fence_compute_ring_emit()
3595 u64 addr = rdev->fence_drv[fence->ring].gpu_addr; in cik_fence_compute_ring_emit()
3621 bool cik_semaphore_ring_emit(struct radeon_device *rdev, in cik_semaphore_ring_emit() argument
3655 struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev, in cik_copy_cpdma() argument
3662 int ring_index = rdev->asic->copy.blit_ring_index; in cik_copy_cpdma()
3663 struct radeon_ring *ring = &rdev->ring[ring_index]; in cik_copy_cpdma()
3672 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18); in cik_copy_cpdma()
3675 radeon_sync_free(rdev, &sync, NULL); in cik_copy_cpdma()
3679 radeon_sync_resv(rdev, &sync, resv, false); in cik_copy_cpdma()
3680 radeon_sync_rings(rdev, &sync, ring->idx); in cik_copy_cpdma()
3701 r = radeon_fence_emit(rdev, &fence, ring->idx); in cik_copy_cpdma()
3703 radeon_ring_unlock_undo(rdev, ring); in cik_copy_cpdma()
3704 radeon_sync_free(rdev, &sync, NULL); in cik_copy_cpdma()
3708 radeon_ring_unlock_commit(rdev, ring, false); in cik_copy_cpdma()
3709 radeon_sync_free(rdev, &sync, fence); in cik_copy_cpdma()
3729 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib) in cik_ring_ib_execute() argument
3731 struct radeon_ring *ring = &rdev->ring[ib->ring]; in cik_ring_ib_execute()
3749 } else if (rdev->wb.enabled) { in cik_ring_ib_execute()
3779 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring) in cik_ib_test() argument
3787 r = radeon_scratch_get(rdev, &scratch); in cik_ib_test()
3793 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256); in cik_ib_test()
3796 radeon_scratch_free(rdev, scratch); in cik_ib_test()
3803 r = radeon_ib_schedule(rdev, &ib, NULL, false); in cik_ib_test()
3805 radeon_scratch_free(rdev, scratch); in cik_ib_test()
3806 radeon_ib_free(rdev, &ib); in cik_ib_test()
3814 radeon_scratch_free(rdev, scratch); in cik_ib_test()
3815 radeon_ib_free(rdev, &ib); in cik_ib_test()
3819 radeon_scratch_free(rdev, scratch); in cik_ib_test()
3820 radeon_ib_free(rdev, &ib); in cik_ib_test()
3824 for (i = 0; i < rdev->usec_timeout; i++) { in cik_ib_test()
3830 if (i < rdev->usec_timeout) { in cik_ib_test()
3837 radeon_scratch_free(rdev, scratch); in cik_ib_test()
3838 radeon_ib_free(rdev, &ib); in cik_ib_test()
3873 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable) in cik_cp_gfx_enable() argument
3878 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) in cik_cp_gfx_enable()
3879 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size); in cik_cp_gfx_enable()
3881 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; in cik_cp_gfx_enable()
3894 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev) in cik_cp_gfx_load_microcode() argument
3898 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw) in cik_cp_gfx_load_microcode()
3901 cik_cp_gfx_enable(rdev, false); in cik_cp_gfx_load_microcode()
3903 if (rdev->new_fw) { in cik_cp_gfx_load_microcode()
3905 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; in cik_cp_gfx_load_microcode()
3907 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; in cik_cp_gfx_load_microcode()
3909 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; in cik_cp_gfx_load_microcode()
3919 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); in cik_cp_gfx_load_microcode()
3928 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); in cik_cp_gfx_load_microcode()
3937 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); in cik_cp_gfx_load_microcode()
3948 fw_data = (const __be32 *)rdev->pfp_fw->data; in cik_cp_gfx_load_microcode()
3955 fw_data = (const __be32 *)rdev->ce_fw->data; in cik_cp_gfx_load_microcode()
3962 fw_data = (const __be32 *)rdev->me_fw->data; in cik_cp_gfx_load_microcode()
3981 static int cik_cp_gfx_start(struct radeon_device *rdev) in cik_cp_gfx_start() argument
3983 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_cp_gfx_start()
3987 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1); in cik_cp_gfx_start()
3991 cik_cp_gfx_enable(rdev, true); in cik_cp_gfx_start()
3993 r = radeon_ring_lock(rdev, ring, cik_default_size + 17); in cik_cp_gfx_start()
4028 radeon_ring_unlock_commit(rdev, ring, false); in cik_cp_gfx_start()
4041 static void cik_cp_gfx_fini(struct radeon_device *rdev) in cik_cp_gfx_fini() argument
4043 cik_cp_gfx_enable(rdev, false); in cik_cp_gfx_fini()
4044 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); in cik_cp_gfx_fini()
4056 static int cik_cp_gfx_resume(struct radeon_device *rdev) in cik_cp_gfx_resume() argument
4065 if (rdev->family != CHIP_HAWAII) in cik_cp_gfx_resume()
4074 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF); in cik_cp_gfx_resume()
4078 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_cp_gfx_resume()
4092 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC); in cik_cp_gfx_resume()
4093 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF); in cik_cp_gfx_resume()
4098 if (!rdev->wb.enabled) in cik_cp_gfx_resume()
4109 cik_cp_gfx_start(rdev); in cik_cp_gfx_resume()
4110 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true; in cik_cp_gfx_resume()
4111 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]); in cik_cp_gfx_resume()
4113 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false; in cik_cp_gfx_resume()
4117 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX) in cik_cp_gfx_resume()
4118 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size); in cik_cp_gfx_resume()
4123 u32 cik_gfx_get_rptr(struct radeon_device *rdev, in cik_gfx_get_rptr() argument
4128 if (rdev->wb.enabled) in cik_gfx_get_rptr()
4129 rptr = rdev->wb.wb[ring->rptr_offs/4]; in cik_gfx_get_rptr()
4136 u32 cik_gfx_get_wptr(struct radeon_device *rdev, in cik_gfx_get_wptr() argument
4142 void cik_gfx_set_wptr(struct radeon_device *rdev, in cik_gfx_set_wptr() argument
4149 u32 cik_compute_get_rptr(struct radeon_device *rdev, in cik_compute_get_rptr() argument
4154 if (rdev->wb.enabled) { in cik_compute_get_rptr()
4155 rptr = rdev->wb.wb[ring->rptr_offs/4]; in cik_compute_get_rptr()
4157 mutex_lock(&rdev->srbm_mutex); in cik_compute_get_rptr()
4158 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); in cik_compute_get_rptr()
4160 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_compute_get_rptr()
4161 mutex_unlock(&rdev->srbm_mutex); in cik_compute_get_rptr()
4167 u32 cik_compute_get_wptr(struct radeon_device *rdev, in cik_compute_get_wptr() argument
4172 if (rdev->wb.enabled) { in cik_compute_get_wptr()
4174 wptr = rdev->wb.wb[ring->wptr_offs/4]; in cik_compute_get_wptr()
4176 mutex_lock(&rdev->srbm_mutex); in cik_compute_get_wptr()
4177 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); in cik_compute_get_wptr()
4179 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_compute_get_wptr()
4180 mutex_unlock(&rdev->srbm_mutex); in cik_compute_get_wptr()
4186 void cik_compute_set_wptr(struct radeon_device *rdev, in cik_compute_set_wptr() argument
4190 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr; in cik_compute_set_wptr()
4194 static void cik_compute_stop(struct radeon_device *rdev, in cik_compute_stop() argument
4199 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0); in cik_compute_stop()
4207 for (j = 0; j < rdev->usec_timeout; j++) { in cik_compute_stop()
4216 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_compute_stop()
4227 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable) in cik_cp_compute_enable() argument
4236 mutex_lock(&rdev->srbm_mutex); in cik_cp_compute_enable()
4237 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]); in cik_cp_compute_enable()
4238 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]); in cik_cp_compute_enable()
4239 mutex_unlock(&rdev->srbm_mutex); in cik_cp_compute_enable()
4242 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false; in cik_cp_compute_enable()
4243 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false; in cik_cp_compute_enable()
4256 static int cik_cp_compute_load_microcode(struct radeon_device *rdev) in cik_cp_compute_load_microcode() argument
4260 if (!rdev->mec_fw) in cik_cp_compute_load_microcode()
4263 cik_cp_compute_enable(rdev, false); in cik_cp_compute_load_microcode()
4265 if (rdev->new_fw) { in cik_cp_compute_load_microcode()
4267 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; in cik_cp_compute_load_microcode()
4275 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); in cik_cp_compute_load_microcode()
4283 if (rdev->family == CHIP_KAVERI) { in cik_cp_compute_load_microcode()
4285 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; in cik_cp_compute_load_microcode()
4288 (rdev->mec2_fw->data + in cik_cp_compute_load_microcode()
4300 fw_data = (const __be32 *)rdev->mec_fw->data; in cik_cp_compute_load_microcode()
4306 if (rdev->family == CHIP_KAVERI) { in cik_cp_compute_load_microcode()
4308 fw_data = (const __be32 *)rdev->mec_fw->data; in cik_cp_compute_load_microcode()
4327 static int cik_cp_compute_start(struct radeon_device *rdev) in cik_cp_compute_start() argument
4329 cik_cp_compute_enable(rdev, true); in cik_cp_compute_start()
4342 static void cik_cp_compute_fini(struct radeon_device *rdev) in cik_cp_compute_fini() argument
4346 cik_cp_compute_enable(rdev, false); in cik_cp_compute_fini()
4354 if (rdev->ring[idx].mqd_obj) { in cik_cp_compute_fini()
4355 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); in cik_cp_compute_fini()
4357 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r); in cik_cp_compute_fini()
4359 radeon_bo_unpin(rdev->ring[idx].mqd_obj); in cik_cp_compute_fini()
4360 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); in cik_cp_compute_fini()
4362 radeon_bo_unref(&rdev->ring[idx].mqd_obj); in cik_cp_compute_fini()
4363 rdev->ring[idx].mqd_obj = NULL; in cik_cp_compute_fini()
4368 static void cik_mec_fini(struct radeon_device *rdev) in cik_mec_fini() argument
4372 if (rdev->mec.hpd_eop_obj) { in cik_mec_fini()
4373 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); in cik_mec_fini()
4375 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r); in cik_mec_fini()
4376 radeon_bo_unpin(rdev->mec.hpd_eop_obj); in cik_mec_fini()
4377 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); in cik_mec_fini()
4379 radeon_bo_unref(&rdev->mec.hpd_eop_obj); in cik_mec_fini()
4380 rdev->mec.hpd_eop_obj = NULL; in cik_mec_fini()
4386 static int cik_mec_init(struct radeon_device *rdev) in cik_mec_init() argument
4395 if (rdev->family == CHIP_KAVERI) in cik_mec_init()
4396 rdev->mec.num_mec = 2; in cik_mec_init()
4398 rdev->mec.num_mec = 1; in cik_mec_init()
4399 rdev->mec.num_pipe = 4; in cik_mec_init()
4400 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8; in cik_mec_init()
4402 if (rdev->mec.hpd_eop_obj == NULL) { in cik_mec_init()
4403 r = radeon_bo_create(rdev, in cik_mec_init()
4404 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2, in cik_mec_init()
4407 &rdev->mec.hpd_eop_obj); in cik_mec_init()
4409 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r); in cik_mec_init()
4414 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false); in cik_mec_init()
4416 cik_mec_fini(rdev); in cik_mec_init()
4419 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT, in cik_mec_init()
4420 &rdev->mec.hpd_eop_gpu_addr); in cik_mec_init()
4422 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r); in cik_mec_init()
4423 cik_mec_fini(rdev); in cik_mec_init()
4426 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd); in cik_mec_init()
4428 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r); in cik_mec_init()
4429 cik_mec_fini(rdev); in cik_mec_init()
4434 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2); in cik_mec_init()
4436 radeon_bo_kunmap(rdev->mec.hpd_eop_obj); in cik_mec_init()
4437 radeon_bo_unreserve(rdev->mec.hpd_eop_obj); in cik_mec_init()
4518 static int cik_cp_compute_resume(struct radeon_device *rdev) in cik_cp_compute_resume() argument
4530 r = cik_cp_compute_start(rdev); in cik_cp_compute_resume()
4540 mutex_lock(&rdev->srbm_mutex); in cik_cp_compute_resume()
4542 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) { in cik_cp_compute_resume()
4546 cik_srbm_select(rdev, me, pipe, 0, 0); in cik_cp_compute_resume()
4548 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ; in cik_cp_compute_resume()
4563 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_cp_compute_resume()
4564 mutex_unlock(&rdev->srbm_mutex); in cik_cp_compute_resume()
4573 if (rdev->ring[idx].mqd_obj == NULL) { in cik_cp_compute_resume()
4574 r = radeon_bo_create(rdev, in cik_cp_compute_resume()
4578 NULL, &rdev->ring[idx].mqd_obj); in cik_cp_compute_resume()
4580 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r); in cik_cp_compute_resume()
4585 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false); in cik_cp_compute_resume()
4587 cik_cp_compute_fini(rdev); in cik_cp_compute_resume()
4590 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT, in cik_cp_compute_resume()
4593 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r); in cik_cp_compute_resume()
4594 cik_cp_compute_fini(rdev); in cik_cp_compute_resume()
4597 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf); in cik_cp_compute_resume()
4599 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r); in cik_cp_compute_resume()
4600 cik_cp_compute_fini(rdev); in cik_cp_compute_resume()
4614 mutex_lock(&rdev->srbm_mutex); in cik_cp_compute_resume()
4615 cik_srbm_select(rdev, rdev->ring[idx].me, in cik_cp_compute_resume()
4616 rdev->ring[idx].pipe, in cik_cp_compute_resume()
4617 rdev->ring[idx].queue, 0); in cik_cp_compute_resume()
4640 for (j = 0; j < rdev->usec_timeout; j++) { in cik_cp_compute_resume()
4661 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8; in cik_cp_compute_resume()
4673 order_base_2(rdev->ring[idx].ring_size / 8); in cik_cp_compute_resume()
4687 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET; in cik_cp_compute_resume()
4689 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET; in cik_cp_compute_resume()
4698 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET; in cik_cp_compute_resume()
4700 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET; in cik_cp_compute_resume()
4715 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index); in cik_cp_compute_resume()
4727 rdev->ring[idx].wptr = 0; in cik_cp_compute_resume()
4728 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr; in cik_cp_compute_resume()
4740 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_cp_compute_resume()
4741 mutex_unlock(&rdev->srbm_mutex); in cik_cp_compute_resume()
4743 radeon_bo_kunmap(rdev->ring[idx].mqd_obj); in cik_cp_compute_resume()
4744 radeon_bo_unreserve(rdev->ring[idx].mqd_obj); in cik_cp_compute_resume()
4746 rdev->ring[idx].ready = true; in cik_cp_compute_resume()
4747 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]); in cik_cp_compute_resume()
4749 rdev->ring[idx].ready = false; in cik_cp_compute_resume()
4755 static void cik_cp_enable(struct radeon_device *rdev, bool enable) in cik_cp_enable() argument
4757 cik_cp_gfx_enable(rdev, enable); in cik_cp_enable()
4758 cik_cp_compute_enable(rdev, enable); in cik_cp_enable()
4761 static int cik_cp_load_microcode(struct radeon_device *rdev) in cik_cp_load_microcode() argument
4765 r = cik_cp_gfx_load_microcode(rdev); in cik_cp_load_microcode()
4768 r = cik_cp_compute_load_microcode(rdev); in cik_cp_load_microcode()
4775 static void cik_cp_fini(struct radeon_device *rdev) in cik_cp_fini() argument
4777 cik_cp_gfx_fini(rdev); in cik_cp_fini()
4778 cik_cp_compute_fini(rdev); in cik_cp_fini()
4781 static int cik_cp_resume(struct radeon_device *rdev) in cik_cp_resume() argument
4785 cik_enable_gui_idle_interrupt(rdev, false); in cik_cp_resume()
4787 r = cik_cp_load_microcode(rdev); in cik_cp_resume()
4791 r = cik_cp_gfx_resume(rdev); in cik_cp_resume()
4794 r = cik_cp_compute_resume(rdev); in cik_cp_resume()
4798 cik_enable_gui_idle_interrupt(rdev, true); in cik_cp_resume()
4803 static void cik_print_gpu_status_regs(struct radeon_device *rdev) in cik_print_gpu_status_regs() argument
4805 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n", in cik_print_gpu_status_regs()
4807 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n", in cik_print_gpu_status_regs()
4809 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n", in cik_print_gpu_status_regs()
4811 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n", in cik_print_gpu_status_regs()
4813 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n", in cik_print_gpu_status_regs()
4815 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n", in cik_print_gpu_status_regs()
4817 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n", in cik_print_gpu_status_regs()
4819 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n", in cik_print_gpu_status_regs()
4821 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n", in cik_print_gpu_status_regs()
4823 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n", in cik_print_gpu_status_regs()
4825 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT)); in cik_print_gpu_status_regs()
4826 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n", in cik_print_gpu_status_regs()
4828 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n", in cik_print_gpu_status_regs()
4830 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n", in cik_print_gpu_status_regs()
4832 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n", in cik_print_gpu_status_regs()
4834 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n", in cik_print_gpu_status_regs()
4836 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS)); in cik_print_gpu_status_regs()
4837 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT)); in cik_print_gpu_status_regs()
4838 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n", in cik_print_gpu_status_regs()
4840 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS)); in cik_print_gpu_status_regs()
4852 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev) in cik_gpu_check_soft_reset() argument
4912 if (evergreen_is_display_hung(rdev)) in cik_gpu_check_soft_reset()
4932 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask) in cik_gpu_soft_reset() argument
4941 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask); in cik_gpu_soft_reset()
4943 cik_print_gpu_status_regs(rdev); in cik_gpu_soft_reset()
4944 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", in cik_gpu_soft_reset()
4946 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", in cik_gpu_soft_reset()
4950 cik_fini_pg(rdev); in cik_gpu_soft_reset()
4951 cik_fini_cg(rdev); in cik_gpu_soft_reset()
4954 cik_rlc_stop(rdev); in cik_gpu_soft_reset()
4975 evergreen_mc_stop(rdev, &save); in cik_gpu_soft_reset()
4976 if (evergreen_mc_wait_for_idle(rdev)) { in cik_gpu_soft_reset()
4977 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in cik_gpu_soft_reset()
5013 if (!(rdev->flags & RADEON_IS_IGP)) { in cik_gpu_soft_reset()
5021 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); in cik_gpu_soft_reset()
5035 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); in cik_gpu_soft_reset()
5049 evergreen_mc_resume(rdev, &save); in cik_gpu_soft_reset()
5052 cik_print_gpu_status_regs(rdev); in cik_gpu_soft_reset()
5061 static void kv_save_regs_for_reset(struct radeon_device *rdev, in kv_save_regs_for_reset() argument
5073 static void kv_restore_regs_for_reset(struct radeon_device *rdev, in kv_restore_regs_for_reset() argument
5146 static void cik_gpu_pci_config_reset(struct radeon_device *rdev) in cik_gpu_pci_config_reset() argument
5152 dev_info(rdev->dev, "GPU pci config reset\n"); in cik_gpu_pci_config_reset()
5157 cik_fini_pg(rdev); in cik_gpu_pci_config_reset()
5158 cik_fini_cg(rdev); in cik_gpu_pci_config_reset()
5177 cik_rlc_stop(rdev); in cik_gpu_pci_config_reset()
5182 evergreen_mc_stop(rdev, &save); in cik_gpu_pci_config_reset()
5183 if (evergreen_mc_wait_for_idle(rdev)) { in cik_gpu_pci_config_reset()
5184 dev_warn(rdev->dev, "Wait for MC idle timed out !\n"); in cik_gpu_pci_config_reset()
5187 if (rdev->flags & RADEON_IS_IGP) in cik_gpu_pci_config_reset()
5188 kv_save_regs_for_reset(rdev, &kv_save); in cik_gpu_pci_config_reset()
5191 pci_clear_master(rdev->pdev); in cik_gpu_pci_config_reset()
5193 radeon_pci_config_reset(rdev); in cik_gpu_pci_config_reset()
5198 for (i = 0; i < rdev->usec_timeout; i++) { in cik_gpu_pci_config_reset()
5205 if (rdev->flags & RADEON_IS_IGP) in cik_gpu_pci_config_reset()
5206 kv_restore_regs_for_reset(rdev, &kv_save); in cik_gpu_pci_config_reset()
5219 int cik_asic_reset(struct radeon_device *rdev, bool hard) in cik_asic_reset() argument
5224 cik_gpu_pci_config_reset(rdev); in cik_asic_reset()
5228 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_asic_reset()
5231 r600_set_bios_scratch_engine_hung(rdev, true); in cik_asic_reset()
5234 cik_gpu_soft_reset(rdev, reset_mask); in cik_asic_reset()
5236 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_asic_reset()
5240 cik_gpu_pci_config_reset(rdev); in cik_asic_reset()
5242 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_asic_reset()
5245 r600_set_bios_scratch_engine_hung(rdev, false); in cik_asic_reset()
5259 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring) in cik_gfx_is_lockup() argument
5261 u32 reset_mask = cik_gpu_check_soft_reset(rdev); in cik_gfx_is_lockup()
5266 radeon_ring_lockup_update(rdev, ring); in cik_gfx_is_lockup()
5269 return radeon_ring_test_lockup(rdev, ring); in cik_gfx_is_lockup()
5281 static void cik_mc_program(struct radeon_device *rdev) in cik_mc_program() argument
5297 evergreen_mc_stop(rdev, &save); in cik_mc_program()
5298 if (radeon_mc_wait_for_idle(rdev)) { in cik_mc_program()
5299 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in cik_mc_program()
5305 rdev->mc.vram_start >> 12); in cik_mc_program()
5307 rdev->mc.vram_end >> 12); in cik_mc_program()
5309 rdev->vram_scratch.gpu_addr >> 12); in cik_mc_program()
5310 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16; in cik_mc_program()
5311 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF); in cik_mc_program()
5314 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8)); in cik_mc_program()
5320 if (radeon_mc_wait_for_idle(rdev)) { in cik_mc_program()
5321 dev_warn(rdev->dev, "Wait for MC idle timedout !\n"); in cik_mc_program()
5323 evergreen_mc_resume(rdev, &save); in cik_mc_program()
5326 rv515_vga_render_disable(rdev); in cik_mc_program()
5338 static int cik_mc_init(struct radeon_device *rdev) in cik_mc_init() argument
5344 rdev->mc.vram_is_ddr = true; in cik_mc_init()
5382 rdev->mc.vram_width = numchan * chansize; in cik_mc_init()
5384 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0); in cik_mc_init()
5385 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0); in cik_mc_init()
5387 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; in cik_mc_init()
5388 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL; in cik_mc_init()
5389 rdev->mc.visible_vram_size = rdev->mc.aper_size; in cik_mc_init()
5390 si_vram_gtt_location(rdev, &rdev->mc); in cik_mc_init()
5391 radeon_update_bandwidth_info(rdev); in cik_mc_init()
5409 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev) in cik_pcie_gart_tlb_flush() argument
5429 static int cik_pcie_gart_enable(struct radeon_device *rdev) in cik_pcie_gart_enable() argument
5433 if (rdev->gart.robj == NULL) { in cik_pcie_gart_enable()
5434 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n"); in cik_pcie_gart_enable()
5437 r = radeon_gart_table_vram_pin(rdev); in cik_pcie_gart_enable()
5460 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12); in cik_pcie_gart_enable()
5461 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12); in cik_pcie_gart_enable()
5462 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12); in cik_pcie_gart_enable()
5464 (u32)(rdev->dummy_page.addr >> 12)); in cik_pcie_gart_enable()
5476 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1); in cik_pcie_gart_enable()
5480 rdev->vm_manager.saved_table_addr[i]); in cik_pcie_gart_enable()
5483 rdev->vm_manager.saved_table_addr[i]); in cik_pcie_gart_enable()
5488 (u32)(rdev->dummy_page.addr >> 12)); in cik_pcie_gart_enable()
5505 if (rdev->family == CHIP_KAVERI) { in cik_pcie_gart_enable()
5513 mutex_lock(&rdev->srbm_mutex); in cik_pcie_gart_enable()
5515 cik_srbm_select(rdev, 0, 0, 0, i); in cik_pcie_gart_enable()
5528 cik_srbm_select(rdev, 0, 0, 0, 0); in cik_pcie_gart_enable()
5529 mutex_unlock(&rdev->srbm_mutex); in cik_pcie_gart_enable()
5531 cik_pcie_gart_tlb_flush(rdev); in cik_pcie_gart_enable()
5533 (unsigned)(rdev->mc.gtt_size >> 20), in cik_pcie_gart_enable()
5534 (unsigned long long)rdev->gart.table_addr); in cik_pcie_gart_enable()
5535 rdev->gart.ready = true; in cik_pcie_gart_enable()
5546 static void cik_pcie_gart_disable(struct radeon_device *rdev) in cik_pcie_gart_disable() argument
5556 rdev->vm_manager.saved_table_addr[i] = RREG32(reg); in cik_pcie_gart_disable()
5575 radeon_gart_table_vram_unpin(rdev); in cik_pcie_gart_disable()
5585 static void cik_pcie_gart_fini(struct radeon_device *rdev) in cik_pcie_gart_fini() argument
5587 cik_pcie_gart_disable(rdev); in cik_pcie_gart_fini()
5588 radeon_gart_table_vram_free(rdev); in cik_pcie_gart_fini()
5589 radeon_gart_fini(rdev); in cik_pcie_gart_fini()
5601 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) in cik_ib_parse() argument
5621 int cik_vm_init(struct radeon_device *rdev) in cik_vm_init() argument
5628 rdev->vm_manager.nvm = 16; in cik_vm_init()
5630 if (rdev->flags & RADEON_IS_IGP) { in cik_vm_init()
5633 rdev->vm_manager.vram_base_offset = tmp; in cik_vm_init()
5635 rdev->vm_manager.vram_base_offset = 0; in cik_vm_init()
5647 void cik_vm_fini(struct radeon_device *rdev) in cik_vm_fini() argument
5660 static void cik_vm_decode_fault(struct radeon_device *rdev, in cik_vm_decode_fault() argument
5669 if (rdev->family == CHIP_HAWAII) in cik_vm_decode_fault()
5688 void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring, in cik_vm_flush() argument
5733 cik_hdp_flush_cp_ring_emit(rdev, ring->idx); in cik_vm_flush()
5768 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev, in cik_enable_gui_idle_interrupt() argument
5780 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable) in cik_enable_lbpw() argument
5792 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev) in cik_wait_for_rlc_serdes() argument
5797 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { in cik_wait_for_rlc_serdes()
5798 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { in cik_wait_for_rlc_serdes()
5799 cik_select_se_sh(rdev, i, j); in cik_wait_for_rlc_serdes()
5800 for (k = 0; k < rdev->usec_timeout; k++) { in cik_wait_for_rlc_serdes()
5807 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_wait_for_rlc_serdes()
5810 for (k = 0; k < rdev->usec_timeout; k++) { in cik_wait_for_rlc_serdes()
5817 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc) in cik_update_rlc() argument
5826 static u32 cik_halt_rlc(struct radeon_device *rdev) in cik_halt_rlc() argument
5838 for (i = 0; i < rdev->usec_timeout; i++) { in cik_halt_rlc()
5844 cik_wait_for_rlc_serdes(rdev); in cik_halt_rlc()
5850 void cik_enter_rlc_safe_mode(struct radeon_device *rdev) in cik_enter_rlc_safe_mode() argument
5858 for (i = 0; i < rdev->usec_timeout; i++) { in cik_enter_rlc_safe_mode()
5864 for (i = 0; i < rdev->usec_timeout; i++) { in cik_enter_rlc_safe_mode()
5871 void cik_exit_rlc_safe_mode(struct radeon_device *rdev) in cik_exit_rlc_safe_mode() argument
5886 static void cik_rlc_stop(struct radeon_device *rdev) in cik_rlc_stop() argument
5890 cik_enable_gui_idle_interrupt(rdev, false); in cik_rlc_stop()
5892 cik_wait_for_rlc_serdes(rdev); in cik_rlc_stop()
5902 static void cik_rlc_start(struct radeon_device *rdev) in cik_rlc_start() argument
5906 cik_enable_gui_idle_interrupt(rdev, true); in cik_rlc_start()
5920 static int cik_rlc_resume(struct radeon_device *rdev) in cik_rlc_resume() argument
5924 if (!rdev->rlc_fw) in cik_rlc_resume()
5927 cik_rlc_stop(rdev); in cik_rlc_resume()
5933 si_rlc_reset(rdev); in cik_rlc_resume()
5935 cik_init_pg(rdev); in cik_rlc_resume()
5937 cik_init_cg(rdev); in cik_rlc_resume()
5942 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_rlc_resume()
5950 if (rdev->new_fw) { in cik_rlc_resume()
5952 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data; in cik_rlc_resume()
5954 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_rlc_resume()
5966 switch (rdev->family) { in cik_rlc_resume()
5983 fw_data = (const __be32 *)rdev->rlc_fw->data; in cik_rlc_resume()
5991 cik_enable_lbpw(rdev, false); in cik_rlc_resume()
5993 if (rdev->family == CHIP_BONAIRE) in cik_rlc_resume()
5996 cik_rlc_start(rdev); in cik_rlc_resume()
6001 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable) in cik_enable_cgcg() argument
6007 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) { in cik_enable_cgcg()
6008 cik_enable_gui_idle_interrupt(rdev, true); in cik_enable_cgcg()
6010 tmp = cik_halt_rlc(rdev); in cik_enable_cgcg()
6012 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_enable_cgcg()
6018 cik_update_rlc(rdev, tmp); in cik_enable_cgcg()
6022 cik_enable_gui_idle_interrupt(rdev, false); in cik_enable_cgcg()
6037 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable) in cik_enable_mgcg() argument
6041 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) { in cik_enable_mgcg()
6042 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) { in cik_enable_mgcg()
6043 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) { in cik_enable_mgcg()
6057 tmp = cik_halt_rlc(rdev); in cik_enable_mgcg()
6059 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_enable_mgcg()
6065 cik_update_rlc(rdev, tmp); in cik_enable_mgcg()
6067 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) { in cik_enable_mgcg()
6073 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) && in cik_enable_mgcg()
6074 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS)) in cik_enable_mgcg()
6105 tmp = cik_halt_rlc(rdev); in cik_enable_mgcg()
6107 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_enable_mgcg()
6113 cik_update_rlc(rdev, tmp); in cik_enable_mgcg()
6130 static void cik_enable_mc_ls(struct radeon_device *rdev, in cik_enable_mc_ls() argument
6138 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS)) in cik_enable_mc_ls()
6147 static void cik_enable_mc_mgcg(struct radeon_device *rdev, in cik_enable_mc_mgcg() argument
6155 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG)) in cik_enable_mc_mgcg()
6164 static void cik_enable_sdma_mgcg(struct radeon_device *rdev, in cik_enable_sdma_mgcg() argument
6169 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) { in cik_enable_sdma_mgcg()
6185 static void cik_enable_sdma_mgls(struct radeon_device *rdev, in cik_enable_sdma_mgls() argument
6190 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) { in cik_enable_sdma_mgls()
6213 static void cik_enable_uvd_mgcg(struct radeon_device *rdev, in cik_enable_uvd_mgcg() argument
6218 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) { in cik_enable_uvd_mgcg()
6239 static void cik_enable_bif_mgls(struct radeon_device *rdev, in cik_enable_bif_mgls() argument
6246 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS)) in cik_enable_bif_mgls()
6257 static void cik_enable_hdp_mgcg(struct radeon_device *rdev, in cik_enable_hdp_mgcg() argument
6264 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG)) in cik_enable_hdp_mgcg()
6273 static void cik_enable_hdp_ls(struct radeon_device *rdev, in cik_enable_hdp_ls() argument
6280 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS)) in cik_enable_hdp_ls()
6289 void cik_update_cg(struct radeon_device *rdev, in cik_update_cg() argument
6294 cik_enable_gui_idle_interrupt(rdev, false); in cik_update_cg()
6297 cik_enable_mgcg(rdev, true); in cik_update_cg()
6298 cik_enable_cgcg(rdev, true); in cik_update_cg()
6300 cik_enable_cgcg(rdev, false); in cik_update_cg()
6301 cik_enable_mgcg(rdev, false); in cik_update_cg()
6303 cik_enable_gui_idle_interrupt(rdev, true); in cik_update_cg()
6307 if (!(rdev->flags & RADEON_IS_IGP)) { in cik_update_cg()
6308 cik_enable_mc_mgcg(rdev, enable); in cik_update_cg()
6309 cik_enable_mc_ls(rdev, enable); in cik_update_cg()
6314 cik_enable_sdma_mgcg(rdev, enable); in cik_update_cg()
6315 cik_enable_sdma_mgls(rdev, enable); in cik_update_cg()
6319 cik_enable_bif_mgls(rdev, enable); in cik_update_cg()
6323 if (rdev->has_uvd) in cik_update_cg()
6324 cik_enable_uvd_mgcg(rdev, enable); in cik_update_cg()
6328 cik_enable_hdp_mgcg(rdev, enable); in cik_update_cg()
6329 cik_enable_hdp_ls(rdev, enable); in cik_update_cg()
6333 vce_v2_0_enable_mgcg(rdev, enable); in cik_update_cg()
6337 static void cik_init_cg(struct radeon_device *rdev) in cik_init_cg() argument
6340 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true); in cik_init_cg()
6342 if (rdev->has_uvd) in cik_init_cg()
6343 si_init_uvd_internal_cg(rdev); in cik_init_cg()
6345 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | in cik_init_cg()
6352 static void cik_fini_cg(struct radeon_device *rdev) in cik_fini_cg() argument
6354 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC | in cik_fini_cg()
6360 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false); in cik_fini_cg()
6363 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev, in cik_enable_sck_slowdown_on_pu() argument
6369 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) in cik_enable_sck_slowdown_on_pu()
6377 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev, in cik_enable_sck_slowdown_on_pd() argument
6383 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS)) in cik_enable_sck_slowdown_on_pd()
6391 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable) in cik_enable_cp_pg() argument
6396 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP)) in cik_enable_cp_pg()
6404 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable) in cik_enable_gds_pg() argument
6409 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS)) in cik_enable_gds_pg()
6421 void cik_init_cp_pg_table(struct radeon_device *rdev) in cik_init_cp_pg_table() argument
6428 if (rdev->family == CHIP_KAVERI) in cik_init_cp_pg_table()
6431 if (rdev->rlc.cp_table_ptr == NULL) in cik_init_cp_pg_table()
6435 dst_ptr = rdev->rlc.cp_table_ptr; in cik_init_cp_pg_table()
6437 if (rdev->new_fw) { in cik_init_cp_pg_table()
6442 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data; in cik_init_cp_pg_table()
6444 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6448 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data; in cik_init_cp_pg_table()
6450 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6454 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data; in cik_init_cp_pg_table()
6456 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6460 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data; in cik_init_cp_pg_table()
6462 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6466 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data; in cik_init_cp_pg_table()
6468 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes)); in cik_init_cp_pg_table()
6483 fw_data = (const __be32 *)rdev->ce_fw->data; in cik_init_cp_pg_table()
6486 fw_data = (const __be32 *)rdev->pfp_fw->data; in cik_init_cp_pg_table()
6489 fw_data = (const __be32 *)rdev->me_fw->data; in cik_init_cp_pg_table()
6492 fw_data = (const __be32 *)rdev->mec_fw->data; in cik_init_cp_pg_table()
6505 static void cik_enable_gfx_cgpg(struct radeon_device *rdev, in cik_enable_gfx_cgpg() argument
6510 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) { in cik_enable_gfx_cgpg()
6535 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh) in cik_get_cu_active_bitmap() argument
6540 cik_select_se_sh(rdev, se, sh); in cik_get_cu_active_bitmap()
6543 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff); in cik_get_cu_active_bitmap()
6550 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) { in cik_get_cu_active_bitmap()
6558 static void cik_init_ao_cu_mask(struct radeon_device *rdev) in cik_init_ao_cu_mask() argument
6564 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) { in cik_init_ao_cu_mask()
6565 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) { in cik_init_ao_cu_mask()
6569 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) { in cik_init_ao_cu_mask()
6570 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) { in cik_init_ao_cu_mask()
6591 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev, in cik_enable_gfx_static_mgpg() argument
6597 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG)) in cik_enable_gfx_static_mgpg()
6605 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev, in cik_enable_gfx_dynamic_mgpg() argument
6611 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG)) in cik_enable_gfx_dynamic_mgpg()
6622 static void cik_init_gfx_cgpg(struct radeon_device *rdev) in cik_init_gfx_cgpg() argument
6627 if (rdev->rlc.cs_data) { in cik_init_gfx_cgpg()
6629 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr)); in cik_init_gfx_cgpg()
6630 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr)); in cik_init_gfx_cgpg()
6631 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size); in cik_init_gfx_cgpg()
6637 if (rdev->rlc.reg_list) { in cik_init_gfx_cgpg()
6639 for (i = 0; i < rdev->rlc.reg_list_size; i++) in cik_init_gfx_cgpg()
6640 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]); in cik_init_gfx_cgpg()
6648 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8); in cik_init_gfx_cgpg()
6649 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8); in cik_init_gfx_cgpg()
6671 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable) in cik_update_gfx_pg() argument
6673 cik_enable_gfx_cgpg(rdev, enable); in cik_update_gfx_pg()
6674 cik_enable_gfx_static_mgpg(rdev, enable); in cik_update_gfx_pg()
6675 cik_enable_gfx_dynamic_mgpg(rdev, enable); in cik_update_gfx_pg()
6678 u32 cik_get_csb_size(struct radeon_device *rdev) in cik_get_csb_size() argument
6684 if (rdev->rlc.cs_data == NULL) in cik_get_csb_size()
6692 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { in cik_get_csb_size()
6710 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer) in cik_get_csb_buffer() argument
6716 if (rdev->rlc.cs_data == NULL) in cik_get_csb_buffer()
6728 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) { in cik_get_csb_buffer()
6744 switch (rdev->family) { in cik_get_csb_buffer()
6775 static void cik_init_pg(struct radeon_device *rdev) in cik_init_pg() argument
6777 if (rdev->pg_flags) { in cik_init_pg()
6778 cik_enable_sck_slowdown_on_pu(rdev, true); in cik_init_pg()
6779 cik_enable_sck_slowdown_on_pd(rdev, true); in cik_init_pg()
6780 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { in cik_init_pg()
6781 cik_init_gfx_cgpg(rdev); in cik_init_pg()
6782 cik_enable_cp_pg(rdev, true); in cik_init_pg()
6783 cik_enable_gds_pg(rdev, true); in cik_init_pg()
6785 cik_init_ao_cu_mask(rdev); in cik_init_pg()
6786 cik_update_gfx_pg(rdev, true); in cik_init_pg()
6790 static void cik_fini_pg(struct radeon_device *rdev) in cik_fini_pg() argument
6792 if (rdev->pg_flags) { in cik_fini_pg()
6793 cik_update_gfx_pg(rdev, false); in cik_fini_pg()
6794 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) { in cik_fini_pg()
6795 cik_enable_cp_pg(rdev, false); in cik_fini_pg()
6796 cik_enable_gds_pg(rdev, false); in cik_fini_pg()
6823 static void cik_enable_interrupts(struct radeon_device *rdev) in cik_enable_interrupts() argument
6832 rdev->ih.enabled = true; in cik_enable_interrupts()
6842 static void cik_disable_interrupts(struct radeon_device *rdev) in cik_disable_interrupts() argument
6854 rdev->ih.enabled = false; in cik_disable_interrupts()
6855 rdev->ih.rptr = 0; in cik_disable_interrupts()
6865 static void cik_disable_interrupt_state(struct radeon_device *rdev) in cik_disable_interrupt_state() argument
6894 if (rdev->num_crtc >= 4) { in cik_disable_interrupt_state()
6898 if (rdev->num_crtc >= 6) { in cik_disable_interrupt_state()
6903 if (rdev->num_crtc >= 2) { in cik_disable_interrupt_state()
6907 if (rdev->num_crtc >= 4) { in cik_disable_interrupt_state()
6911 if (rdev->num_crtc >= 6) { in cik_disable_interrupt_state()
6946 static int cik_irq_init(struct radeon_device *rdev) in cik_irq_init() argument
6953 ret = r600_ih_ring_alloc(rdev); in cik_irq_init()
6958 cik_disable_interrupts(rdev); in cik_irq_init()
6961 ret = cik_rlc_resume(rdev); in cik_irq_init()
6963 r600_ih_ring_fini(rdev); in cik_irq_init()
6969 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8); in cik_irq_init()
6979 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8); in cik_irq_init()
6980 rb_bufsz = order_base_2(rdev->ih.ring_size / 4); in cik_irq_init()
6986 if (rdev->wb.enabled) in cik_irq_init()
6990 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC); in cik_irq_init()
6991 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF); in cik_irq_init()
7002 if (rdev->msi_enabled) in cik_irq_init()
7007 cik_disable_interrupt_state(rdev); in cik_irq_init()
7009 pci_set_master(rdev->pdev); in cik_irq_init()
7012 cik_enable_interrupts(rdev); in cik_irq_init()
7026 int cik_irq_set(struct radeon_device *rdev) in cik_irq_set() argument
7036 if (!rdev->irq.installed) { in cik_irq_set()
7041 if (!rdev->ih.enabled) { in cik_irq_set()
7042 cik_disable_interrupts(rdev); in cik_irq_set()
7044 cik_disable_interrupt_state(rdev); in cik_irq_set()
7072 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) { in cik_irq_set()
7076 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) { in cik_irq_set()
7077 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_irq_set()
7119 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) { in cik_irq_set()
7120 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_irq_set()
7163 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) { in cik_irq_set()
7168 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) { in cik_irq_set()
7173 if (rdev->irq.crtc_vblank_int[0] || in cik_irq_set()
7174 atomic_read(&rdev->irq.pflip[0])) { in cik_irq_set()
7178 if (rdev->irq.crtc_vblank_int[1] || in cik_irq_set()
7179 atomic_read(&rdev->irq.pflip[1])) { in cik_irq_set()
7183 if (rdev->irq.crtc_vblank_int[2] || in cik_irq_set()
7184 atomic_read(&rdev->irq.pflip[2])) { in cik_irq_set()
7188 if (rdev->irq.crtc_vblank_int[3] || in cik_irq_set()
7189 atomic_read(&rdev->irq.pflip[3])) { in cik_irq_set()
7193 if (rdev->irq.crtc_vblank_int[4] || in cik_irq_set()
7194 atomic_read(&rdev->irq.pflip[4])) { in cik_irq_set()
7198 if (rdev->irq.crtc_vblank_int[5] || in cik_irq_set()
7199 atomic_read(&rdev->irq.pflip[5])) { in cik_irq_set()
7203 if (rdev->irq.hpd[0]) { in cik_irq_set()
7207 if (rdev->irq.hpd[1]) { in cik_irq_set()
7211 if (rdev->irq.hpd[2]) { in cik_irq_set()
7215 if (rdev->irq.hpd[3]) { in cik_irq_set()
7219 if (rdev->irq.hpd[4]) { in cik_irq_set()
7223 if (rdev->irq.hpd[5]) { in cik_irq_set()
7246 if (rdev->num_crtc >= 4) { in cik_irq_set()
7250 if (rdev->num_crtc >= 6) { in cik_irq_set()
7255 if (rdev->num_crtc >= 2) { in cik_irq_set()
7261 if (rdev->num_crtc >= 4) { in cik_irq_set()
7267 if (rdev->num_crtc >= 6) { in cik_irq_set()
7296 static inline void cik_irq_ack(struct radeon_device *rdev) in cik_irq_ack() argument
7300 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS); in cik_irq_ack()
7301 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE); in cik_irq_ack()
7302 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2); in cik_irq_ack()
7303 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3); in cik_irq_ack()
7304 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4); in cik_irq_ack()
7305 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5); in cik_irq_ack()
7306 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6); in cik_irq_ack()
7308 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7310 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7312 if (rdev->num_crtc >= 4) { in cik_irq_ack()
7313 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7315 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7318 if (rdev->num_crtc >= 6) { in cik_irq_ack()
7319 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7321 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS + in cik_irq_ack()
7325 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7328 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7331 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) in cik_irq_ack()
7333 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) in cik_irq_ack()
7335 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) in cik_irq_ack()
7337 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) in cik_irq_ack()
7340 if (rdev->num_crtc >= 4) { in cik_irq_ack()
7341 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7344 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7347 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) in cik_irq_ack()
7349 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) in cik_irq_ack()
7351 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) in cik_irq_ack()
7353 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) in cik_irq_ack()
7357 if (rdev->num_crtc >= 6) { in cik_irq_ack()
7358 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7361 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED) in cik_irq_ack()
7364 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) in cik_irq_ack()
7366 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) in cik_irq_ack()
7368 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) in cik_irq_ack()
7370 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) in cik_irq_ack()
7374 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) { in cik_irq_ack()
7379 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) { in cik_irq_ack()
7384 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) { in cik_irq_ack()
7389 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) { in cik_irq_ack()
7394 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) { in cik_irq_ack()
7399 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) { in cik_irq_ack()
7404 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) { in cik_irq_ack()
7409 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) { in cik_irq_ack()
7414 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) { in cik_irq_ack()
7419 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) { in cik_irq_ack()
7424 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) { in cik_irq_ack()
7429 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) { in cik_irq_ack()
7443 static void cik_irq_disable(struct radeon_device *rdev) in cik_irq_disable() argument
7445 cik_disable_interrupts(rdev); in cik_irq_disable()
7448 cik_irq_ack(rdev); in cik_irq_disable()
7449 cik_disable_interrupt_state(rdev); in cik_irq_disable()
7460 static void cik_irq_suspend(struct radeon_device *rdev) in cik_irq_suspend() argument
7462 cik_irq_disable(rdev); in cik_irq_suspend()
7463 cik_rlc_stop(rdev); in cik_irq_suspend()
7475 static void cik_irq_fini(struct radeon_device *rdev) in cik_irq_fini() argument
7477 cik_irq_suspend(rdev); in cik_irq_fini()
7478 r600_ih_ring_fini(rdev); in cik_irq_fini()
7492 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev) in cik_get_ih_wptr() argument
7496 if (rdev->wb.enabled) in cik_get_ih_wptr()
7497 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]); in cik_get_ih_wptr()
7507 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", in cik_get_ih_wptr()
7508 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask); in cik_get_ih_wptr()
7509 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask; in cik_get_ih_wptr()
7514 return (wptr & rdev->ih.ptr_mask); in cik_get_ih_wptr()
7549 int cik_irq_process(struct radeon_device *rdev) in cik_irq_process() argument
7551 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_irq_process()
7552 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_irq_process()
7564 if (!rdev->ih.enabled || rdev->shutdown) in cik_irq_process()
7567 wptr = cik_get_ih_wptr(rdev); in cik_irq_process()
7571 if (atomic_xchg(&rdev->ih.lock, 1)) in cik_irq_process()
7574 rptr = rdev->ih.rptr; in cik_irq_process()
7581 cik_irq_ack(rdev); in cik_irq_process()
7587 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff; in cik_irq_process()
7588 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff; in cik_irq_process()
7589 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff; in cik_irq_process()
7595 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)) in cik_irq_process()
7598 if (rdev->irq.crtc_vblank_int[0]) { in cik_irq_process()
7599 drm_handle_vblank(rdev->ddev, 0); in cik_irq_process()
7600 rdev->pm.vblank_sync = true; in cik_irq_process()
7601 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7603 if (atomic_read(&rdev->irq.pflip[0])) in cik_irq_process()
7604 radeon_crtc_handle_vblank(rdev, 0); in cik_irq_process()
7605 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT; in cik_irq_process()
7610 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)) in cik_irq_process()
7613 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT; in cik_irq_process()
7625 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)) in cik_irq_process()
7628 if (rdev->irq.crtc_vblank_int[1]) { in cik_irq_process()
7629 drm_handle_vblank(rdev->ddev, 1); in cik_irq_process()
7630 rdev->pm.vblank_sync = true; in cik_irq_process()
7631 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7633 if (atomic_read(&rdev->irq.pflip[1])) in cik_irq_process()
7634 radeon_crtc_handle_vblank(rdev, 1); in cik_irq_process()
7635 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT; in cik_irq_process()
7640 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)) in cik_irq_process()
7643 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT; in cik_irq_process()
7655 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)) in cik_irq_process()
7658 if (rdev->irq.crtc_vblank_int[2]) { in cik_irq_process()
7659 drm_handle_vblank(rdev->ddev, 2); in cik_irq_process()
7660 rdev->pm.vblank_sync = true; in cik_irq_process()
7661 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7663 if (atomic_read(&rdev->irq.pflip[2])) in cik_irq_process()
7664 radeon_crtc_handle_vblank(rdev, 2); in cik_irq_process()
7665 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT; in cik_irq_process()
7670 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)) in cik_irq_process()
7673 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT; in cik_irq_process()
7685 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)) in cik_irq_process()
7688 if (rdev->irq.crtc_vblank_int[3]) { in cik_irq_process()
7689 drm_handle_vblank(rdev->ddev, 3); in cik_irq_process()
7690 rdev->pm.vblank_sync = true; in cik_irq_process()
7691 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7693 if (atomic_read(&rdev->irq.pflip[3])) in cik_irq_process()
7694 radeon_crtc_handle_vblank(rdev, 3); in cik_irq_process()
7695 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT; in cik_irq_process()
7700 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)) in cik_irq_process()
7703 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT; in cik_irq_process()
7715 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)) in cik_irq_process()
7718 if (rdev->irq.crtc_vblank_int[4]) { in cik_irq_process()
7719 drm_handle_vblank(rdev->ddev, 4); in cik_irq_process()
7720 rdev->pm.vblank_sync = true; in cik_irq_process()
7721 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7723 if (atomic_read(&rdev->irq.pflip[4])) in cik_irq_process()
7724 radeon_crtc_handle_vblank(rdev, 4); in cik_irq_process()
7725 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT; in cik_irq_process()
7730 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)) in cik_irq_process()
7733 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT; in cik_irq_process()
7745 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)) in cik_irq_process()
7748 if (rdev->irq.crtc_vblank_int[5]) { in cik_irq_process()
7749 drm_handle_vblank(rdev->ddev, 5); in cik_irq_process()
7750 rdev->pm.vblank_sync = true; in cik_irq_process()
7751 wake_up(&rdev->irq.vblank_queue); in cik_irq_process()
7753 if (atomic_read(&rdev->irq.pflip[5])) in cik_irq_process()
7754 radeon_crtc_handle_vblank(rdev, 5); in cik_irq_process()
7755 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT; in cik_irq_process()
7760 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)) in cik_irq_process()
7763 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT; in cik_irq_process()
7780 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1); in cik_irq_process()
7785 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT)) in cik_irq_process()
7788 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT; in cik_irq_process()
7794 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT)) in cik_irq_process()
7797 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT; in cik_irq_process()
7803 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT)) in cik_irq_process()
7806 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT; in cik_irq_process()
7812 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT)) in cik_irq_process()
7815 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT; in cik_irq_process()
7821 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT)) in cik_irq_process()
7824 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT; in cik_irq_process()
7830 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT)) in cik_irq_process()
7833 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT; in cik_irq_process()
7839 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT)) in cik_irq_process()
7842 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT; in cik_irq_process()
7848 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT)) in cik_irq_process()
7851 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT; in cik_irq_process()
7857 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT)) in cik_irq_process()
7860 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT; in cik_irq_process()
7866 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT)) in cik_irq_process()
7869 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT; in cik_irq_process()
7875 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT)) in cik_irq_process()
7878 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT; in cik_irq_process()
7884 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT)) in cik_irq_process()
7887 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT; in cik_irq_process()
7903 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX); in cik_irq_process()
7914 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data); in cik_irq_process()
7915 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n", in cik_irq_process()
7917 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n", in cik_irq_process()
7919 cik_vm_decode_fault(rdev, status, addr, mc_client); in cik_irq_process()
7925 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX); in cik_irq_process()
7928 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX); in cik_irq_process()
7937 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); in cik_irq_process()
7947 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX); in cik_irq_process()
7952 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX); in cik_irq_process()
7954 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX); in cik_irq_process()
8013 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX); in cik_irq_process()
8026 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); in cik_irq_process()
8040 rdev->pm.dpm.thermal.high_to_low = false; in cik_irq_process()
8045 rdev->pm.dpm.thermal.high_to_low = true; in cik_irq_process()
8097 rptr &= rdev->ih.ptr_mask; in cik_irq_process()
8101 schedule_work(&rdev->dp_work); in cik_irq_process()
8103 schedule_delayed_work(&rdev->hotplug_work, 0); in cik_irq_process()
8105 rdev->needs_reset = true; in cik_irq_process()
8106 wake_up_all(&rdev->fence_queue); in cik_irq_process()
8109 schedule_work(&rdev->pm.dpm.thermal.work); in cik_irq_process()
8110 rdev->ih.rptr = rptr; in cik_irq_process()
8111 atomic_set(&rdev->ih.lock, 0); in cik_irq_process()
8114 wptr = cik_get_ih_wptr(rdev); in cik_irq_process()
8124 static void cik_uvd_init(struct radeon_device *rdev) in cik_uvd_init() argument
8128 if (!rdev->has_uvd) in cik_uvd_init()
8131 r = radeon_uvd_init(rdev); in cik_uvd_init()
8133 dev_err(rdev->dev, "failed UVD (%d) init.\n", r); in cik_uvd_init()
8140 rdev->has_uvd = false; in cik_uvd_init()
8143 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL; in cik_uvd_init()
8144 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096); in cik_uvd_init()
8147 static void cik_uvd_start(struct radeon_device *rdev) in cik_uvd_start() argument
8151 if (!rdev->has_uvd) in cik_uvd_start()
8154 r = radeon_uvd_resume(rdev); in cik_uvd_start()
8156 dev_err(rdev->dev, "failed UVD resume (%d).\n", r); in cik_uvd_start()
8159 r = uvd_v4_2_resume(rdev); in cik_uvd_start()
8161 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r); in cik_uvd_start()
8164 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX); in cik_uvd_start()
8166 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r); in cik_uvd_start()
8172 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0; in cik_uvd_start()
8175 static void cik_uvd_resume(struct radeon_device *rdev) in cik_uvd_resume() argument
8180 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size) in cik_uvd_resume()
8183 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX]; in cik_uvd_resume()
8184 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0)); in cik_uvd_resume()
8186 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r); in cik_uvd_resume()
8189 r = uvd_v1_0_init(rdev); in cik_uvd_resume()
8191 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r); in cik_uvd_resume()
8196 static void cik_vce_init(struct radeon_device *rdev) in cik_vce_init() argument
8200 if (!rdev->has_vce) in cik_vce_init()
8203 r = radeon_vce_init(rdev); in cik_vce_init()
8205 dev_err(rdev->dev, "failed VCE (%d) init.\n", r); in cik_vce_init()
8212 rdev->has_vce = false; in cik_vce_init()
8215 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL; in cik_vce_init()
8216 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096); in cik_vce_init()
8217 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL; in cik_vce_init()
8218 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096); in cik_vce_init()
8221 static void cik_vce_start(struct radeon_device *rdev) in cik_vce_start() argument
8225 if (!rdev->has_vce) in cik_vce_start()
8228 r = radeon_vce_resume(rdev); in cik_vce_start()
8230 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); in cik_vce_start()
8233 r = vce_v2_0_resume(rdev); in cik_vce_start()
8235 dev_err(rdev->dev, "failed VCE resume (%d).\n", r); in cik_vce_start()
8238 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX); in cik_vce_start()
8240 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r); in cik_vce_start()
8243 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX); in cik_vce_start()
8245 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r); in cik_vce_start()
8251 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0; in cik_vce_start()
8252 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0; in cik_vce_start()
8255 static void cik_vce_resume(struct radeon_device *rdev) in cik_vce_resume() argument
8260 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size) in cik_vce_resume()
8263 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX]; in cik_vce_resume()
8264 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); in cik_vce_resume()
8266 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); in cik_vce_resume()
8269 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX]; in cik_vce_resume()
8270 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP); in cik_vce_resume()
8272 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r); in cik_vce_resume()
8275 r = vce_v1_0_init(rdev); in cik_vce_resume()
8277 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r); in cik_vce_resume()
8291 static int cik_startup(struct radeon_device *rdev) in cik_startup() argument
8298 cik_pcie_gen3_enable(rdev); in cik_startup()
8300 cik_program_aspm(rdev); in cik_startup()
8303 r = r600_vram_scratch_init(rdev); in cik_startup()
8307 cik_mc_program(rdev); in cik_startup()
8309 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) { in cik_startup()
8310 r = ci_mc_load_microcode(rdev); in cik_startup()
8317 r = cik_pcie_gart_enable(rdev); in cik_startup()
8320 cik_gpu_init(rdev); in cik_startup()
8323 if (rdev->flags & RADEON_IS_IGP) { in cik_startup()
8324 if (rdev->family == CHIP_KAVERI) { in cik_startup()
8325 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list; in cik_startup()
8326 rdev->rlc.reg_list_size = in cik_startup()
8329 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list; in cik_startup()
8330 rdev->rlc.reg_list_size = in cik_startup()
8334 rdev->rlc.cs_data = ci_cs_data; in cik_startup()
8335 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */ in cik_startup()
8336 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */ in cik_startup()
8337 r = sumo_rlc_init(rdev); in cik_startup()
8344 r = radeon_wb_init(rdev); in cik_startup()
8349 r = cik_mec_init(rdev); in cik_startup()
8355 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX); in cik_startup()
8357 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in cik_startup()
8361 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX); in cik_startup()
8363 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in cik_startup()
8367 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX); in cik_startup()
8369 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r); in cik_startup()
8373 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX); in cik_startup()
8375 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); in cik_startup()
8379 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX); in cik_startup()
8381 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r); in cik_startup()
8385 cik_uvd_start(rdev); in cik_startup()
8386 cik_vce_start(rdev); in cik_startup()
8389 if (!rdev->irq.installed) { in cik_startup()
8390 r = radeon_irq_kms_init(rdev); in cik_startup()
8395 r = cik_irq_init(rdev); in cik_startup()
8398 radeon_irq_kms_fini(rdev); in cik_startup()
8401 cik_irq_set(rdev); in cik_startup()
8403 if (rdev->family == CHIP_HAWAII) { in cik_startup()
8404 if (rdev->new_fw) in cik_startup()
8412 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_startup()
8413 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET, in cik_startup()
8420 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_startup()
8421 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET, in cik_startup()
8431 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_startup()
8432 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET, in cik_startup()
8442 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; in cik_startup()
8443 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET, in cik_startup()
8448 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; in cik_startup()
8449 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET, in cik_startup()
8454 r = cik_cp_resume(rdev); in cik_startup()
8458 r = cik_sdma_resume(rdev); in cik_startup()
8462 cik_uvd_resume(rdev); in cik_startup()
8463 cik_vce_resume(rdev); in cik_startup()
8465 r = radeon_ib_pool_init(rdev); in cik_startup()
8467 dev_err(rdev->dev, "IB initialization failed (%d).\n", r); in cik_startup()
8471 r = radeon_vm_manager_init(rdev); in cik_startup()
8473 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r); in cik_startup()
8477 r = radeon_audio_init(rdev); in cik_startup()
8493 int cik_resume(struct radeon_device *rdev) in cik_resume() argument
8498 atom_asic_init(rdev->mode_info.atom_context); in cik_resume()
8501 cik_init_golden_registers(rdev); in cik_resume()
8503 if (rdev->pm.pm_method == PM_METHOD_DPM) in cik_resume()
8504 radeon_pm_resume(rdev); in cik_resume()
8506 rdev->accel_working = true; in cik_resume()
8507 r = cik_startup(rdev); in cik_resume()
8510 rdev->accel_working = false; in cik_resume()
8527 int cik_suspend(struct radeon_device *rdev) in cik_suspend() argument
8529 radeon_pm_suspend(rdev); in cik_suspend()
8530 radeon_audio_fini(rdev); in cik_suspend()
8531 radeon_vm_manager_fini(rdev); in cik_suspend()
8532 cik_cp_enable(rdev, false); in cik_suspend()
8533 cik_sdma_enable(rdev, false); in cik_suspend()
8534 if (rdev->has_uvd) { in cik_suspend()
8535 uvd_v1_0_fini(rdev); in cik_suspend()
8536 radeon_uvd_suspend(rdev); in cik_suspend()
8538 if (rdev->has_vce) in cik_suspend()
8539 radeon_vce_suspend(rdev); in cik_suspend()
8540 cik_fini_pg(rdev); in cik_suspend()
8541 cik_fini_cg(rdev); in cik_suspend()
8542 cik_irq_suspend(rdev); in cik_suspend()
8543 radeon_wb_disable(rdev); in cik_suspend()
8544 cik_pcie_gart_disable(rdev); in cik_suspend()
8564 int cik_init(struct radeon_device *rdev) in cik_init() argument
8570 if (!radeon_get_bios(rdev)) { in cik_init()
8571 if (ASIC_IS_AVIVO(rdev)) in cik_init()
8575 if (!rdev->is_atom_bios) { in cik_init()
8576 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n"); in cik_init()
8579 r = radeon_atombios_init(rdev); in cik_init()
8584 if (!radeon_card_posted(rdev)) { in cik_init()
8585 if (!rdev->bios) { in cik_init()
8586 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n"); in cik_init()
8590 atom_asic_init(rdev->mode_info.atom_context); in cik_init()
8593 cik_init_golden_registers(rdev); in cik_init()
8595 cik_scratch_init(rdev); in cik_init()
8597 radeon_surface_init(rdev); in cik_init()
8599 radeon_get_clock_info(rdev->ddev); in cik_init()
8602 r = radeon_fence_driver_init(rdev); in cik_init()
8607 r = cik_mc_init(rdev); in cik_init()
8611 r = radeon_bo_init(rdev); in cik_init()
8615 if (rdev->flags & RADEON_IS_IGP) { in cik_init()
8616 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || in cik_init()
8617 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) { in cik_init()
8618 r = cik_init_microcode(rdev); in cik_init()
8625 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw || in cik_init()
8626 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw || in cik_init()
8627 !rdev->mc_fw) { in cik_init()
8628 r = cik_init_microcode(rdev); in cik_init()
8637 radeon_pm_init(rdev); in cik_init()
8639 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; in cik_init()
8641 r600_ring_init(rdev, ring, 1024 * 1024); in cik_init()
8643 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]; in cik_init()
8645 r600_ring_init(rdev, ring, 1024 * 1024); in cik_init()
8646 r = radeon_doorbell_get(rdev, &ring->doorbell_index); in cik_init()
8650 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]; in cik_init()
8652 r600_ring_init(rdev, ring, 1024 * 1024); in cik_init()
8653 r = radeon_doorbell_get(rdev, &ring->doorbell_index); in cik_init()
8657 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX]; in cik_init()
8659 r600_ring_init(rdev, ring, 256 * 1024); in cik_init()
8661 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX]; in cik_init()
8663 r600_ring_init(rdev, ring, 256 * 1024); in cik_init()
8665 cik_uvd_init(rdev); in cik_init()
8666 cik_vce_init(rdev); in cik_init()
8668 rdev->ih.ring_obj = NULL; in cik_init()
8669 r600_ih_ring_init(rdev, 64 * 1024); in cik_init()
8671 r = r600_pcie_gart_init(rdev); in cik_init()
8675 rdev->accel_working = true; in cik_init()
8676 r = cik_startup(rdev); in cik_init()
8678 dev_err(rdev->dev, "disabling GPU acceleration\n"); in cik_init()
8679 cik_cp_fini(rdev); in cik_init()
8680 cik_sdma_fini(rdev); in cik_init()
8681 cik_irq_fini(rdev); in cik_init()
8682 sumo_rlc_fini(rdev); in cik_init()
8683 cik_mec_fini(rdev); in cik_init()
8684 radeon_wb_fini(rdev); in cik_init()
8685 radeon_ib_pool_fini(rdev); in cik_init()
8686 radeon_vm_manager_fini(rdev); in cik_init()
8687 radeon_irq_kms_fini(rdev); in cik_init()
8688 cik_pcie_gart_fini(rdev); in cik_init()
8689 rdev->accel_working = false; in cik_init()
8696 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) { in cik_init()
8713 void cik_fini(struct radeon_device *rdev) in cik_fini() argument
8715 radeon_pm_fini(rdev); in cik_fini()
8716 cik_cp_fini(rdev); in cik_fini()
8717 cik_sdma_fini(rdev); in cik_fini()
8718 cik_fini_pg(rdev); in cik_fini()
8719 cik_fini_cg(rdev); in cik_fini()
8720 cik_irq_fini(rdev); in cik_fini()
8721 sumo_rlc_fini(rdev); in cik_fini()
8722 cik_mec_fini(rdev); in cik_fini()
8723 radeon_wb_fini(rdev); in cik_fini()
8724 radeon_vm_manager_fini(rdev); in cik_fini()
8725 radeon_ib_pool_fini(rdev); in cik_fini()
8726 radeon_irq_kms_fini(rdev); in cik_fini()
8727 uvd_v1_0_fini(rdev); in cik_fini()
8728 radeon_uvd_fini(rdev); in cik_fini()
8729 radeon_vce_fini(rdev); in cik_fini()
8730 cik_pcie_gart_fini(rdev); in cik_fini()
8731 r600_vram_scratch_fini(rdev); in cik_fini()
8732 radeon_gem_fini(rdev); in cik_fini()
8733 radeon_fence_driver_fini(rdev); in cik_fini()
8734 radeon_bo_fini(rdev); in cik_fini()
8735 radeon_atombios_fini(rdev); in cik_fini()
8736 kfree(rdev->bios); in cik_fini()
8737 rdev->bios = NULL; in cik_fini()
8743 struct radeon_device *rdev = dev->dev_private; in dce8_program_fmt() local
8817 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev, in dce8_line_buffer_adjust() argument
8840 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; in dce8_line_buffer_adjust()
8844 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4; in dce8_line_buffer_adjust()
8856 for (i = 0; i < rdev->usec_timeout; i++) { in dce8_line_buffer_adjust()
8888 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev) in cik_get_number_of_dram_channels() argument
9246 static void dce8_program_watermarks(struct radeon_device *rdev, in dce8_program_watermarks() argument
9265 if ((rdev->pm.pm_method == PM_METHOD_DPM) && in dce8_program_watermarks()
9266 rdev->pm.dpm_enabled) { in dce8_program_watermarks()
9268 radeon_dpm_get_mclk(rdev, false) * 10; in dce8_program_watermarks()
9270 radeon_dpm_get_sclk(rdev, false) * 10; in dce8_program_watermarks()
9272 wm_high.yclk = rdev->pm.current_mclk * 10; in dce8_program_watermarks()
9273 wm_high.sclk = rdev->pm.current_sclk * 10; in dce8_program_watermarks()
9289 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev); in dce8_program_watermarks()
9300 (rdev->disp_priority == 2)) { in dce8_program_watermarks()
9305 if ((rdev->pm.pm_method == PM_METHOD_DPM) && in dce8_program_watermarks()
9306 rdev->pm.dpm_enabled) { in dce8_program_watermarks()
9308 radeon_dpm_get_mclk(rdev, true) * 10; in dce8_program_watermarks()
9310 radeon_dpm_get_sclk(rdev, true) * 10; in dce8_program_watermarks()
9312 wm_low.yclk = rdev->pm.current_mclk * 10; in dce8_program_watermarks()
9313 wm_low.sclk = rdev->pm.current_sclk * 10; in dce8_program_watermarks()
9329 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev); in dce8_program_watermarks()
9340 (rdev->disp_priority == 2)) { in dce8_program_watermarks()
9382 void dce8_bandwidth_update(struct radeon_device *rdev) in dce8_bandwidth_update() argument
9388 if (!rdev->mode_info.mode_config_initialized) in dce8_bandwidth_update()
9391 radeon_update_display_priority(rdev); in dce8_bandwidth_update()
9393 for (i = 0; i < rdev->num_crtc; i++) { in dce8_bandwidth_update()
9394 if (rdev->mode_info.crtcs[i]->base.enabled) in dce8_bandwidth_update()
9397 for (i = 0; i < rdev->num_crtc; i++) { in dce8_bandwidth_update()
9398 mode = &rdev->mode_info.crtcs[i]->base.mode; in dce8_bandwidth_update()
9399 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode); in dce8_bandwidth_update()
9400 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads); in dce8_bandwidth_update()
9412 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev) in cik_get_gpu_clock_counter() argument
9416 mutex_lock(&rdev->gpu_clock_mutex); in cik_get_gpu_clock_counter()
9420 mutex_unlock(&rdev->gpu_clock_mutex); in cik_get_gpu_clock_counter()
9424 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock, in cik_set_uvd_clock() argument
9431 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, in cik_set_uvd_clock()
9452 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk) in cik_set_uvd_clocks() argument
9456 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS); in cik_set_uvd_clocks()
9460 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS); in cik_set_uvd_clocks()
9464 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk) in cik_set_vce_clocks() argument
9470 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK, in cik_set_vce_clocks()
9499 static void cik_pcie_gen3_enable(struct radeon_device *rdev) in cik_pcie_gen3_enable() argument
9501 struct pci_dev *root = rdev->pdev->bus->self; in cik_pcie_gen3_enable()
9507 if (pci_is_root_bus(rdev->pdev->bus)) in cik_pcie_gen3_enable()
9513 if (rdev->flags & RADEON_IS_IGP) in cik_pcie_gen3_enable()
9516 if (!(rdev->flags & RADEON_IS_PCIE)) in cik_pcie_gen3_enable()
9544 if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev)) in cik_pcie_gen3_enable()
9556 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL, in cik_pcie_gen3_enable()
9563 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL, in cik_pcie_gen3_enable()
9582 pcie_capability_read_word(rdev->pdev, in cik_pcie_gen3_enable()
9590 pcie_capability_read_word(rdev->pdev, in cik_pcie_gen3_enable()
9596 pcie_capability_read_word(rdev->pdev, in cik_pcie_gen3_enable()
9618 pcie_capability_read_word(rdev->pdev, in cik_pcie_gen3_enable()
9623 pcie_capability_write_word(rdev->pdev, in cik_pcie_gen3_enable()
9639 pcie_capability_read_word(rdev->pdev, in cik_pcie_gen3_enable()
9647 pcie_capability_write_word(rdev->pdev, in cik_pcie_gen3_enable()
9663 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16); in cik_pcie_gen3_enable()
9671 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16); in cik_pcie_gen3_enable()
9677 for (i = 0; i < rdev->usec_timeout; i++) { in cik_pcie_gen3_enable()
9685 static void cik_program_aspm(struct radeon_device *rdev) in cik_program_aspm() argument
9695 if (rdev->flags & RADEON_IS_IGP) in cik_program_aspm()
9698 if (!(rdev->flags & RADEON_IS_PCIE)) in cik_program_aspm()
9763 !pci_is_root_bus(rdev->pdev->bus)) { in cik_program_aspm()
9764 struct pci_dev *root = rdev->pdev->bus->self; in cik_program_aspm()