1/* 2 * Copyright (c) 2013-2014, ARM Limited and Contributors. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions are met: 6 * 7 * Redistributions of source code must retain the above copyright notice, this 8 * list of conditions and the following disclaimer. 9 * 10 * Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * Neither the name of ARM nor the names of its contributors may be used 15 * to endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 22 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31#include <arch.h> 32#include <asm_macros.S> 33 34 .globl flush_dcache_range 35 .globl inv_dcache_range 36 .globl dcsw_op_louis 37 .globl dcsw_op_all 38 .globl dcsw_op_level1 39 .globl dcsw_op_level2 40 .globl dcsw_op_level3 41 42 /* ------------------------------------------ 43 * Clean+Invalidate from base address till 44 * size. 'x0' = addr, 'x1' = size 45 * ------------------------------------------ 46 */ 47func flush_dcache_range 48 dcache_line_size x2, x3 49 add x1, x0, x1 50 sub x3, x2, #1 51 bic x0, x0, x3 52flush_loop: 53 dc civac, x0 54 add x0, x0, x2 55 cmp x0, x1 56 b.lo flush_loop 57 dsb sy 58 ret 59endfunc flush_dcache_range 60 61 62 /* ------------------------------------------ 63 * Invalidate from base address till 64 * size. 'x0' = addr, 'x1' = size 65 * ------------------------------------------ 66 */ 67func inv_dcache_range 68 dcache_line_size x2, x3 69 add x1, x0, x1 70 sub x3, x2, #1 71 bic x0, x0, x3 72inv_loop: 73 dc ivac, x0 74 add x0, x0, x2 75 cmp x0, x1 76 b.lo inv_loop 77 dsb sy 78 ret 79endfunc inv_dcache_range 80 81 82 /* --------------------------------------------------------------- 83 * Data cache operations by set/way to the level specified 84 * 85 * The main function, do_dcsw_op requires: 86 * x0: The operation type (0-2), as defined in arch.h 87 * x3: The last cache level to operate on 88 * x9: clidr_el1 89 * x10: The cache level to begin operation from 90 * and will carry out the operation on each data cache from level 0 91 * to the level in x3 in sequence 92 * 93 * The dcsw_op macro sets up the x3 and x9 parameters based on 94 * clidr_el1 cache information before invoking the main function 95 * --------------------------------------------------------------- 96 */ 97 98 .macro dcsw_op shift, fw, ls 99 mrs x9, clidr_el1 100 ubfx x3, x9, \shift, \fw 101 lsl x3, x3, \ls 102 mov x10, xzr 103 b do_dcsw_op 104 .endm 105 106func do_dcsw_op 107 cbz x3, exit 108 adr x14, dcsw_loop_table // compute inner loop address 109 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions 110 mov x0, x9 111 mov w8, #1 112loop1: 113 add x2, x10, x10, lsr #1 // work out 3x current cache level 114 lsr x1, x0, x2 // extract cache type bits from clidr 115 and x1, x1, #7 // mask the bits for current cache only 116 cmp x1, #2 // see what cache we have at this level 117 b.lt level_done // nothing to do if no cache or icache 118 119 msr csselr_el1, x10 // select current cache level in csselr 120 isb // isb to sych the new cssr&csidr 121 mrs x1, ccsidr_el1 // read the new ccsidr 122 and x2, x1, #7 // extract the length of the cache lines 123 add x2, x2, #4 // add 4 (line length offset) 124 ubfx x4, x1, #3, #10 // maximum way number 125 clz w5, w4 // bit position of way size increment 126 lsl w9, w4, w5 // w9 = aligned max way number 127 lsl w16, w8, w5 // w16 = way number loop decrement 128 orr w9, w10, w9 // w9 = combine way and cache number 129 ubfx w6, w1, #13, #15 // w6 = max set number 130 lsl w17, w8, w2 // w17 = set number loop decrement 131 dsb sy // barrier before we start this level 132 br x14 // jump to DC operation specific loop 133 134 .macro dcsw_loop _op 135loop2_\_op: 136 lsl w7, w6, w2 // w7 = aligned max set number 137 138loop3_\_op: 139 orr w11, w9, w7 // combine cache, way and set number 140 dc \_op, x11 141 subs w7, w7, w17 // decrement set number 142 b.ge loop3_\_op 143 144 subs x9, x9, x16 // decrement way number 145 b.ge loop2_\_op 146 147 b level_done 148 .endm 149 150level_done: 151 add x10, x10, #2 // increment cache number 152 cmp x3, x10 153 b.gt loop1 154 msr csselr_el1, xzr // select cache level 0 in csselr 155 dsb sy // barrier to complete final cache operation 156 isb 157exit: 158 ret 159endfunc do_dcsw_op 160 161dcsw_loop_table: 162 dcsw_loop isw 163 dcsw_loop cisw 164 dcsw_loop csw 165 166 167func dcsw_op_louis 168 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 169endfunc dcsw_op_louis 170 171 172func dcsw_op_all 173 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 174endfunc dcsw_op_all 175 176 /* --------------------------------------------------------------- 177 * Helper macro for data cache operations by set/way for the 178 * level specified 179 * --------------------------------------------------------------- 180 */ 181 .macro dcsw_op_level level 182 mrs x9, clidr_el1 183 mov x3, \level 184 sub x10, x3, #2 185 b do_dcsw_op 186 .endm 187 188 /* --------------------------------------------------------------- 189 * Data cache operations by set/way for level 1 cache 190 * 191 * The main function, do_dcsw_op requires: 192 * x0: The operation type (0-2), as defined in arch.h 193 * --------------------------------------------------------------- 194 */ 195func dcsw_op_level1 196 dcsw_op_level #(1 << LEVEL_SHIFT) 197endfunc dcsw_op_level1 198 199 /* --------------------------------------------------------------- 200 * Data cache operations by set/way for level 2 cache 201 * 202 * The main function, do_dcsw_op requires: 203 * x0: The operation type (0-2), as defined in arch.h 204 * --------------------------------------------------------------- 205 */ 206func dcsw_op_level2 207 dcsw_op_level #(2 << LEVEL_SHIFT) 208endfunc dcsw_op_level2 209 210 /* --------------------------------------------------------------- 211 * Data cache operations by set/way for level 3 cache 212 * 213 * The main function, do_dcsw_op requires: 214 * x0: The operation type (0-2), as defined in arch.h 215 * --------------------------------------------------------------- 216 */ 217func dcsw_op_level3 218 dcsw_op_level #(3 << LEVEL_SHIFT) 219endfunc dcsw_op_level3 220