1/* 2 * Copyright (c) 2016-2021, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9 10 .globl flush_dcache_range 11 .globl clean_dcache_range 12 .globl inv_dcache_range 13 .globl dcsw_op_louis 14 .globl dcsw_op_all 15 .globl dcsw_op_level1 16 .globl dcsw_op_level2 17 .globl dcsw_op_level3 18 19/* 20 * This macro can be used for implementing various data cache operations `op` 21 */ 22.macro do_dcache_maintenance_by_mva op, coproc, opc1, CRn, CRm, opc2 23 /* Exit early if size is zero */ 24 cmp r1, #0 25 beq exit_loop_\op 26 dcache_line_size r2, r3 27 add r1, r0, r1 28 sub r3, r2, #1 29 bic r0, r0, r3 30loop_\op: 31 stcopr r0, \coproc, \opc1, \CRn, \CRm, \opc2 32 add r0, r0, r2 33 cmp r0, r1 34 blo loop_\op 35 dsb sy 36exit_loop_\op: 37 bx lr 38.endm 39 40.macro check_plat_can_cmo 41#if CONDITIONAL_CMO 42 mov r3, lr 43 mov r2, r0 44 bl plat_can_cmo 45 mov lr, r3 46 cmp r0, #0 47 bne 1f 48 bx lr 491: 50 mov r0, r2 51#endif 52.endm 53 54 /* ------------------------------------------ 55 * Clean+Invalidate from base address till 56 * size. 'r0' = addr, 'r1' = size 57 * ------------------------------------------ 58 */ 59func flush_dcache_range 60 check_plat_can_cmo 61 do_dcache_maintenance_by_mva cimvac, DCCIMVAC 62endfunc flush_dcache_range 63 64 /* ------------------------------------------ 65 * Clean from base address till size. 66 * 'r0' = addr, 'r1' = size 67 * ------------------------------------------ 68 */ 69func clean_dcache_range 70 check_plat_can_cmo 71 do_dcache_maintenance_by_mva cmvac, DCCMVAC 72endfunc clean_dcache_range 73 74 /* ------------------------------------------ 75 * Invalidate from base address till 76 * size. 'r0' = addr, 'r1' = size 77 * ------------------------------------------ 78 */ 79func inv_dcache_range 80 check_plat_can_cmo 81 do_dcache_maintenance_by_mva imvac, DCIMVAC 82endfunc inv_dcache_range 83 84 /* ---------------------------------------------------------------- 85 * Data cache operations by set/way to the level specified 86 * 87 * The main function, do_dcsw_op requires: 88 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 89 * as defined in arch.h 90 * r1: The cache level to begin operation from 91 * r2: clidr_el1 92 * r3: The last cache level to operate on 93 * and will carry out the operation on each data cache from level 0 94 * to the level in r3 in sequence 95 * 96 * The dcsw_op macro sets up the r2 and r3 parameters based on 97 * clidr_el1 cache information before invoking the main function 98 * ---------------------------------------------------------------- 99 */ 100 101 .macro dcsw_op shift, fw, ls 102 ldcopr r2, CLIDR 103 ubfx r3, r2, \shift, \fw 104 lsl r3, r3, \ls 105 mov r1, #0 106 b do_dcsw_op 107 .endm 108 109func do_dcsw_op 110 push {r4-r12, lr} 111 ldcopr r8, ID_MMFR4 // stash FEAT_CCIDX identifier in r8 112 ubfx r8, r8, #ID_MMFR4_CCIDX_SHIFT, #ID_MMFR4_CCIDX_LENGTH 113 adr r11, dcsw_loop_table // compute cache op based on the operation type 114 add r6, r11, r0, lsl #3 // cache op is 2x32-bit instructions 115loop1: 116 add r10, r1, r1, LSR #1 // Work out 3x current cache level 117 mov r12, r2, LSR r10 // extract cache type bits from clidr 118 and r12, r12, #7 // mask the bits for current cache only 119 cmp r12, #2 // see what cache we have at this level 120 blo level_done // no cache or only instruction cache at this level 121 122 stcopr r1, CSSELR // select current cache level in csselr 123 isb // isb to sych the new cssr&csidr 124 ldcopr r12, CCSIDR // read the new ccsidr 125 and r10, r12, #7 // extract the length of the cache lines 126 add r10, r10, #4 // add 4 (r10 = line length offset) 127 128 cmp r8, #0 // check for FEAT_CCIDX for Associativity 129 beq 1f 130 ubfx r4, r12, #3, #21 // r4 = associativity CCSIDR[23:3] 131 b 2f 1321: 133 ubfx r4, r12, #3, #10 // r4 = associativity CCSIDR[12:3] 1342: 135 clz r5, r4 // r5 = the bit position of the way size increment 136 mov r9, r4 // r9 working copy of the aligned max way number 137 138loop2: 139 cmp r8, #0 // check for FEAT_CCIDX for NumSets 140 beq 3f 141 ldcopr r12, CCSIDR2 // FEAT_CCIDX numsets is in CCSIDR2 142 ubfx r7, r12, #0, #24 // r7 = numsets CCSIDR2[23:0] 143 b loop3 1443: 145 ubfx r7, r12, #13, #15 // r7 = numsets CCSIDR[27:13] 146loop3: 147 orr r0, r1, r9, LSL r5 // factor in the way number and cache level into r0 148 orr r0, r0, r7, LSL r10 // factor in the set number 149 150 blx r6 151 subs r7, r7, #1 // decrement the set number 152 bhs loop3 153 subs r9, r9, #1 // decrement the way number 154 bhs loop2 155level_done: 156 add r1, r1, #2 // increment the cache number 157 cmp r3, r1 158 // Ensure completion of previous cache maintenance instruction. Note 159 // this also mitigates erratum 814220 on Cortex-A7 160 dsb sy 161 bhi loop1 162 163 mov r6, #0 164 stcopr r6, CSSELR //select cache level 0 in csselr 165 dsb sy 166 isb 167 pop {r4-r12, pc} 168 169dcsw_loop_table: 170 stcopr r0, DCISW 171 bx lr 172 stcopr r0, DCCISW 173 bx lr 174 stcopr r0, DCCSW 175 bx lr 176 177endfunc do_dcsw_op 178 179 /* --------------------------------------------------------------- 180 * Data cache operations by set/way till PoU. 181 * 182 * The function requires : 183 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 184 * as defined in arch.h 185 * --------------------------------------------------------------- 186 */ 187func dcsw_op_louis 188 check_plat_can_cmo 189 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 190endfunc dcsw_op_louis 191 192 /* --------------------------------------------------------------- 193 * Data cache operations by set/way till PoC. 194 * 195 * The function requires : 196 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 197 * as defined in arch.h 198 * --------------------------------------------------------------- 199 */ 200func dcsw_op_all 201 check_plat_can_cmo 202 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 203endfunc dcsw_op_all 204 205 206 /* --------------------------------------------------------------- 207 * Helper macro for data cache operations by set/way for the 208 * level specified 209 * --------------------------------------------------------------- 210 */ 211 .macro dcsw_op_level level 212 ldcopr r2, CLIDR 213 mov r3, \level 214 sub r1, r3, #2 215 b do_dcsw_op 216 .endm 217 218 /* --------------------------------------------------------------- 219 * Data cache operations by set/way for level 1 cache 220 * 221 * The main function, do_dcsw_op requires: 222 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 223 * as defined in arch.h 224 * --------------------------------------------------------------- 225 */ 226func dcsw_op_level1 227 check_plat_can_cmo 228 dcsw_op_level #(1 << LEVEL_SHIFT) 229endfunc dcsw_op_level1 230 231 /* --------------------------------------------------------------- 232 * Data cache operations by set/way for level 2 cache 233 * 234 * The main function, do_dcsw_op requires: 235 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 236 * as defined in arch.h 237 * --------------------------------------------------------------- 238 */ 239func dcsw_op_level2 240 check_plat_can_cmo 241 dcsw_op_level #(2 << LEVEL_SHIFT) 242endfunc dcsw_op_level2 243 244 /* --------------------------------------------------------------- 245 * Data cache operations by set/way for level 3 cache 246 * 247 * The main function, do_dcsw_op requires: 248 * r0: The operation type (DC_OP_ISW, DC_OP_CISW, DC_OP_CSW), 249 * as defined in arch.h 250 * --------------------------------------------------------------- 251 */ 252func dcsw_op_level3 253 check_plat_can_cmo 254 dcsw_op_level #(3 << LEVEL_SHIFT) 255endfunc dcsw_op_level3 256