1/* 2 * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are met: 6 * 7 * Redistributions of source code must retain the above copyright notice, this 8 * list of conditions and the following disclaimer. 9 * 10 * Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * Neither the name of ARM nor the names of its contributors may be used 15 * to endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31#include <arch.h> 32#include <asm_macros.S> 33 34 .globl flush_dcache_range 35 .globl clean_dcache_range 36 .globl inv_dcache_range 37 .globl dcsw_op_louis 38 .globl dcsw_op_all 39 .globl dcsw_op_level1 40 .globl dcsw_op_level2 41 .globl dcsw_op_level3 42 43/* 44 * This macro can be used for implementing various data cache operations `op` 45 */ 46.macro do_dcache_maintenance_by_mva op 47 dcache_line_size x2, x3 48 add x1, x0, x1 49 sub x3, x2, #1 50 bic x0, x0, x3 51loop_\op: 52 dc \op, x0 53 add x0, x0, x2 54 cmp x0, x1 55 b.lo loop_\op 56 dsb sy 57 ret 58.endm 59 /* ------------------------------------------ 60 * Clean+Invalidate from base address till 61 * size. 'x0' = addr, 'x1' = size 62 * ------------------------------------------ 63 */ 64func flush_dcache_range 65 do_dcache_maintenance_by_mva civac 66endfunc flush_dcache_range 67 68 /* ------------------------------------------ 69 * Clean from base address till size. 70 * 'x0' = addr, 'x1' = size 71 * ------------------------------------------ 72 */ 73func clean_dcache_range 74 do_dcache_maintenance_by_mva cvac 75endfunc clean_dcache_range 76 77 /* ------------------------------------------ 78 * Invalidate from base address till 79 * size. 'x0' = addr, 'x1' = size 80 * ------------------------------------------ 81 */ 82func inv_dcache_range 83 do_dcache_maintenance_by_mva ivac 84endfunc inv_dcache_range 85 86 87 /* --------------------------------------------------------------- 88 * Data cache operations by set/way to the level specified 89 * 90 * The main function, do_dcsw_op requires: 91 * x0: The operation type (0-2), as defined in arch.h 92 * x3: The last cache level to operate on 93 * x9: clidr_el1 94 * x10: The cache level to begin operation from 95 * and will carry out the operation on each data cache from level 0 96 * to the level in x3 in sequence 97 * 98 * The dcsw_op macro sets up the x3 and x9 parameters based on 99 * clidr_el1 cache information before invoking the main function 100 * --------------------------------------------------------------- 101 */ 102 103 .macro dcsw_op shift, fw, ls 104 mrs x9, clidr_el1 105 ubfx x3, x9, \shift, \fw 106 lsl x3, x3, \ls 107 mov x10, xzr 108 b do_dcsw_op 109 .endm 110 111func do_dcsw_op 112 cbz x3, exit 113 adr x14, dcsw_loop_table // compute inner loop address 114 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions 115 mov x0, x9 116 mov w8, #1 117loop1: 118 add x2, x10, x10, lsr #1 // work out 3x current cache level 119 lsr x1, x0, x2 // extract cache type bits from clidr 120 and x1, x1, #7 // mask the bits for current cache only 121 cmp x1, #2 // see what cache we have at this level 122 b.lo level_done // nothing to do if no cache or icache 123 124 msr csselr_el1, x10 // select current cache level in csselr 125 isb // isb to sych the new cssr&csidr 126 mrs x1, ccsidr_el1 // read the new ccsidr 127 and x2, x1, #7 // extract the length of the cache lines 128 add x2, x2, #4 // add 4 (line length offset) 129 ubfx x4, x1, #3, #10 // maximum way number 130 clz w5, w4 // bit position of way size increment 131 lsl w9, w4, w5 // w9 = aligned max way number 132 lsl w16, w8, w5 // w16 = way number loop decrement 133 orr w9, w10, w9 // w9 = combine way and cache number 134 ubfx w6, w1, #13, #15 // w6 = max set number 135 lsl w17, w8, w2 // w17 = set number loop decrement 136 dsb sy // barrier before we start this level 137 br x14 // jump to DC operation specific loop 138 139 .macro dcsw_loop _op 140loop2_\_op: 141 lsl w7, w6, w2 // w7 = aligned max set number 142 143loop3_\_op: 144 orr w11, w9, w7 // combine cache, way and set number 145 dc \_op, x11 146 subs w7, w7, w17 // decrement set number 147 b.hs loop3_\_op 148 149 subs x9, x9, x16 // decrement way number 150 b.hs loop2_\_op 151 152 b level_done 153 .endm 154 155level_done: 156 add x10, x10, #2 // increment cache number 157 cmp x3, x10 158 b.hi loop1 159 msr csselr_el1, xzr // select cache level 0 in csselr 160 dsb sy // barrier to complete final cache operation 161 isb 162exit: 163 ret 164endfunc do_dcsw_op 165 166dcsw_loop_table: 167 dcsw_loop isw 168 dcsw_loop cisw 169 dcsw_loop csw 170 171 172func dcsw_op_louis 173 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 174endfunc dcsw_op_louis 175 176 177func dcsw_op_all 178 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 179endfunc dcsw_op_all 180 181 /* --------------------------------------------------------------- 182 * Helper macro for data cache operations by set/way for the 183 * level specified 184 * --------------------------------------------------------------- 185 */ 186 .macro dcsw_op_level level 187 mrs x9, clidr_el1 188 mov x3, \level 189 sub x10, x3, #2 190 b do_dcsw_op 191 .endm 192 193 /* --------------------------------------------------------------- 194 * Data cache operations by set/way for level 1 cache 195 * 196 * The main function, do_dcsw_op requires: 197 * x0: The operation type (0-2), as defined in arch.h 198 * --------------------------------------------------------------- 199 */ 200func dcsw_op_level1 201 dcsw_op_level #(1 << LEVEL_SHIFT) 202endfunc dcsw_op_level1 203 204 /* --------------------------------------------------------------- 205 * Data cache operations by set/way for level 2 cache 206 * 207 * The main function, do_dcsw_op requires: 208 * x0: The operation type (0-2), as defined in arch.h 209 * --------------------------------------------------------------- 210 */ 211func dcsw_op_level2 212 dcsw_op_level #(2 << LEVEL_SHIFT) 213endfunc dcsw_op_level2 214 215 /* --------------------------------------------------------------- 216 * Data cache operations by set/way for level 3 cache 217 * 218 * The main function, do_dcsw_op requires: 219 * x0: The operation type (0-2), as defined in arch.h 220 * --------------------------------------------------------------- 221 */ 222func dcsw_op_level3 223 dcsw_op_level #(3 << LEVEL_SHIFT) 224endfunc dcsw_op_level3 225