1/* 2 * Copyright (c) 2013-2022, Arm Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9 10 .globl flush_dcache_range 11 .globl flush_dcache_to_popa_range 12 .globl clean_dcache_range 13 .globl inv_dcache_range 14 .globl dcsw_op_louis 15 .globl dcsw_op_all 16 .globl dcsw_op_level1 17 .globl dcsw_op_level2 18 .globl dcsw_op_level3 19 20/* 21 * This macro can be used for implementing various data cache operations `op` 22 */ 23.macro do_dcache_maintenance_by_mva op 24 /* Exit early if size is zero */ 25 cbz x1, exit_loop_\op 26 dcache_line_size x2, x3 27 add x1, x0, x1 28 sub x3, x2, #1 29 bic x0, x0, x3 30loop_\op: 31 dc \op, x0 32 add x0, x0, x2 33 cmp x0, x1 34 b.lo loop_\op 35 dsb sy 36exit_loop_\op: 37 ret 38.endm 39 /* ------------------------------------------ 40 * Clean+Invalidate from base address till 41 * size. 'x0' = addr, 'x1' = size 42 * ------------------------------------------ 43 */ 44func flush_dcache_range 45 do_dcache_maintenance_by_mva civac 46endfunc flush_dcache_range 47 48 /* ------------------------------------------ 49 * Clean from base address till size. 50 * 'x0' = addr, 'x1' = size 51 * ------------------------------------------ 52 */ 53func clean_dcache_range 54 do_dcache_maintenance_by_mva cvac 55endfunc clean_dcache_range 56 57 /* ------------------------------------------ 58 * Invalidate from base address till 59 * size. 'x0' = addr, 'x1' = size 60 * ------------------------------------------ 61 */ 62func inv_dcache_range 63 do_dcache_maintenance_by_mva ivac 64endfunc inv_dcache_range 65 66 67 /* 68 * On implementations with FEAT_MTE2, 69 * Root firmware must issue DC_CIGDPAPA instead of DC_CIPAPA , 70 * in order to additionally clean and invalidate Allocation Tags 71 * associated with the affected locations. 72 * 73 * ------------------------------------------ 74 * Clean+Invalidate by PA to POPA 75 * from base address till size. 76 * 'x0' = addr, 'x1' = size 77 * ------------------------------------------ 78 */ 79func flush_dcache_to_popa_range 80 /* Exit early if size is zero */ 81 cbz x1, exit_loop_dc_cipapa 82 dcache_line_size x2, x3 83 sub x3, x2, #1 84 bic x0, x0, x3 85 add x1, x1, x0 86loop_dc_cipapa: 87 sys #6, c7, c14, #1, x0 /* DC CIPAPA,<Xt> */ 88 add x0, x0, x2 89 cmp x0, x1 90 b.lo loop_dc_cipapa 91 dsb osh 92exit_loop_dc_cipapa: 93 ret 94endfunc flush_dcache_to_popa_range 95 96 /* --------------------------------------------------------------- 97 * Data cache operations by set/way to the level specified 98 * 99 * The main function, do_dcsw_op requires: 100 * x0: The operation type (0-2), as defined in arch.h 101 * x3: The last cache level to operate on 102 * x9: clidr_el1 103 * x10: The cache level to begin operation from 104 * and will carry out the operation on each data cache from level 0 105 * to the level in x3 in sequence 106 * 107 * The dcsw_op macro sets up the x3 and x9 parameters based on 108 * clidr_el1 cache information before invoking the main function 109 * --------------------------------------------------------------- 110 */ 111 112 .macro dcsw_op shift, fw, ls 113 mrs x9, clidr_el1 114 ubfx x3, x9, \shift, \fw 115 lsl x3, x3, \ls 116 mov x10, xzr 117 b do_dcsw_op 118 .endm 119 120func do_dcsw_op 121 cbz x3, exit 122 mrs x12, ID_AA64MMFR2_EL1 // stash FEAT_CCIDX identifier in x12 123 ubfx x12, x12, #ID_AA64MMFR2_EL1_CCIDX_SHIFT, #ID_AA64MMFR2_EL1_CCIDX_LENGTH 124 adr x14, dcsw_loop_table // compute inner loop address 125 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions 126#if ENABLE_BTI 127 add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction 128#endif 129 mov x0, x9 130 mov w8, #1 131loop1: 132 add x2, x10, x10, lsr #1 // work out 3x current cache level 133 lsr x1, x0, x2 // extract cache type bits from clidr 134 and x1, x1, #7 // mask the bits for current cache only 135 cmp x1, #2 // see what cache we have at this level 136 b.lo level_done // nothing to do if no cache or icache 137 138 msr csselr_el1, x10 // select current cache level in csselr 139 isb // isb to sych the new cssr&csidr 140 mrs x1, ccsidr_el1 // read the new ccsidr 141 and x2, x1, #7 // extract the length of the cache lines 142 add x2, x2, #4 // add 4 (line length offset) 143 144 cbz x12, 1f // check for FEAT_CCIDX for Associativity 145 ubfx x4, x1, #3, #21 // x4 = associativity CCSIDR_EL1[23:3] 146 b 2f 1471: 148 ubfx x4, x1, #3, #10 // x4 = associativity CCSIDR_EL1[12:3] 1492: 150 clz w5, w4 // bit position of way size increment 151 lsl w9, w4, w5 // w9 = aligned max way number 152 lsl w16, w8, w5 // w16 = way number loop decrement 153 orr w9, w10, w9 // w9 = combine way and cache number 154 155 cbz x12, 3f // check for FEAT_CCIDX for NumSets 156 ubfx x6, x1, #32, #24 // x6 (w6) = numsets CCSIDR_EL1[55:32] 157 // ISA will not allow x->w ubfx 158 b 4f 1593: 160 ubfx w6, w1, #13, #15 // w6 = numsets CCSIDR_EL1[27:13] 1614: 162 lsl w17, w8, w2 // w17 = set number loop decrement 163 dsb sy // barrier before we start this level 164 br x14 // jump to DC operation specific loop 165 166 .macro dcsw_loop _op 167#if ENABLE_BTI 168 bti j 169#endif 170loop2_\_op: 171 lsl w7, w6, w2 // w7 = aligned max set number 172 173loop3_\_op: 174 orr w11, w9, w7 // combine cache, way and set number 175 dc \_op, x11 176 subs w7, w7, w17 // decrement set number 177 b.hs loop3_\_op 178 179 subs x9, x9, x16 // decrement way number 180 b.hs loop2_\_op 181 182 b level_done 183 .endm 184 185level_done: 186 add x10, x10, #2 // increment cache number 187 cmp x3, x10 188 b.hi loop1 189 msr csselr_el1, xzr // select cache level 0 in csselr 190 dsb sy // barrier to complete final cache operation 191 isb 192exit: 193 ret 194endfunc do_dcsw_op 195 196dcsw_loop_table: 197 dcsw_loop isw 198 dcsw_loop cisw 199 dcsw_loop csw 200 201 202func dcsw_op_louis 203 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 204endfunc dcsw_op_louis 205 206 207func dcsw_op_all 208 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 209endfunc dcsw_op_all 210 211 /* --------------------------------------------------------------- 212 * Helper macro for data cache operations by set/way for the 213 * level specified 214 * --------------------------------------------------------------- 215 */ 216 .macro dcsw_op_level level 217 mrs x9, clidr_el1 218 mov x3, \level 219 sub x10, x3, #2 220 b do_dcsw_op 221 .endm 222 223 /* --------------------------------------------------------------- 224 * Data cache operations by set/way for level 1 cache 225 * 226 * The main function, do_dcsw_op requires: 227 * x0: The operation type (0-2), as defined in arch.h 228 * --------------------------------------------------------------- 229 */ 230func dcsw_op_level1 231 dcsw_op_level #(1 << LEVEL_SHIFT) 232endfunc dcsw_op_level1 233 234 /* --------------------------------------------------------------- 235 * Data cache operations by set/way for level 2 cache 236 * 237 * The main function, do_dcsw_op requires: 238 * x0: The operation type (0-2), as defined in arch.h 239 * --------------------------------------------------------------- 240 */ 241func dcsw_op_level2 242 dcsw_op_level #(2 << LEVEL_SHIFT) 243endfunc dcsw_op_level2 244 245 /* --------------------------------------------------------------- 246 * Data cache operations by set/way for level 3 cache 247 * 248 * The main function, do_dcsw_op requires: 249 * x0: The operation type (0-2), as defined in arch.h 250 * --------------------------------------------------------------- 251 */ 252func dcsw_op_level3 253 dcsw_op_level #(3 << LEVEL_SHIFT) 254endfunc dcsw_op_level3 255