1*4882a593Smuzhiyun/* -*- linux-c -*- ------------------------------------------------------- * 2*4882a593Smuzhiyun * 3*4882a593Smuzhiyun * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved 4*4882a593Smuzhiyun * 5*4882a593Smuzhiyun * This program is free software; you can redistribute it and/or modify 6*4882a593Smuzhiyun * it under the terms of the GNU General Public License as published by 7*4882a593Smuzhiyun * the Free Software Foundation, Inc., 53 Temple Place Ste 330, 8*4882a593Smuzhiyun * Boston MA 02111-1307, USA; either version 2 of the License, or 9*4882a593Smuzhiyun * (at your option) any later version; incorporated herein by reference. 10*4882a593Smuzhiyun * 11*4882a593Smuzhiyun * ----------------------------------------------------------------------- */ 12*4882a593Smuzhiyun 13*4882a593Smuzhiyun/* 14*4882a593Smuzhiyun * raid6altivec$#.c 15*4882a593Smuzhiyun * 16*4882a593Smuzhiyun * $#-way unrolled portable integer math RAID-6 instruction set 17*4882a593Smuzhiyun * 18*4882a593Smuzhiyun * This file is postprocessed using unroll.awk 19*4882a593Smuzhiyun * 20*4882a593Smuzhiyun * <benh> hpa: in process, 21*4882a593Smuzhiyun * you can just "steal" the vec unit with enable_kernel_altivec() (but 22*4882a593Smuzhiyun * bracked this with preempt_disable/enable or in a lock) 23*4882a593Smuzhiyun */ 24*4882a593Smuzhiyun 25*4882a593Smuzhiyun#include <linux/raid/pq.h> 26*4882a593Smuzhiyun 27*4882a593Smuzhiyun#ifdef CONFIG_ALTIVEC 28*4882a593Smuzhiyun 29*4882a593Smuzhiyun#include <altivec.h> 30*4882a593Smuzhiyun#ifdef __KERNEL__ 31*4882a593Smuzhiyun# include <asm/cputable.h> 32*4882a593Smuzhiyun# include <asm/switch_to.h> 33*4882a593Smuzhiyun#endif /* __KERNEL__ */ 34*4882a593Smuzhiyun 35*4882a593Smuzhiyun/* 36*4882a593Smuzhiyun * This is the C data type to use. We use a vector of 37*4882a593Smuzhiyun * signed char so vec_cmpgt() will generate the right 38*4882a593Smuzhiyun * instruction. 39*4882a593Smuzhiyun */ 40*4882a593Smuzhiyun 41*4882a593Smuzhiyuntypedef vector signed char unative_t; 42*4882a593Smuzhiyun 43*4882a593Smuzhiyun#define NBYTES(x) ((vector signed char) {x,x,x,x, x,x,x,x, x,x,x,x, x,x,x,x}) 44*4882a593Smuzhiyun#define NSIZE sizeof(unative_t) 45*4882a593Smuzhiyun 46*4882a593Smuzhiyun/* 47*4882a593Smuzhiyun * The SHLBYTE() operation shifts each byte left by 1, *not* 48*4882a593Smuzhiyun * rolling over into the next byte 49*4882a593Smuzhiyun */ 50*4882a593Smuzhiyunstatic inline __attribute_const__ unative_t SHLBYTE(unative_t v) 51*4882a593Smuzhiyun{ 52*4882a593Smuzhiyun return vec_add(v,v); 53*4882a593Smuzhiyun} 54*4882a593Smuzhiyun 55*4882a593Smuzhiyun/* 56*4882a593Smuzhiyun * The MASK() operation returns 0xFF in any byte for which the high 57*4882a593Smuzhiyun * bit is 1, 0x00 for any byte for which the high bit is 0. 58*4882a593Smuzhiyun */ 59*4882a593Smuzhiyunstatic inline __attribute_const__ unative_t MASK(unative_t v) 60*4882a593Smuzhiyun{ 61*4882a593Smuzhiyun unative_t zv = NBYTES(0); 62*4882a593Smuzhiyun 63*4882a593Smuzhiyun /* vec_cmpgt returns a vector bool char; thus the need for the cast */ 64*4882a593Smuzhiyun return (unative_t)vec_cmpgt(zv, v); 65*4882a593Smuzhiyun} 66*4882a593Smuzhiyun 67*4882a593Smuzhiyun 68*4882a593Smuzhiyun/* This is noinline to make damned sure that gcc doesn't move any of the 69*4882a593Smuzhiyun Altivec code around the enable/disable code */ 70*4882a593Smuzhiyunstatic void noinline 71*4882a593Smuzhiyunraid6_altivec$#_gen_syndrome_real(int disks, size_t bytes, void **ptrs) 72*4882a593Smuzhiyun{ 73*4882a593Smuzhiyun u8 **dptr = (u8 **)ptrs; 74*4882a593Smuzhiyun u8 *p, *q; 75*4882a593Smuzhiyun int d, z, z0; 76*4882a593Smuzhiyun 77*4882a593Smuzhiyun unative_t wd$$, wq$$, wp$$, w1$$, w2$$; 78*4882a593Smuzhiyun unative_t x1d = NBYTES(0x1d); 79*4882a593Smuzhiyun 80*4882a593Smuzhiyun z0 = disks - 3; /* Highest data disk */ 81*4882a593Smuzhiyun p = dptr[z0+1]; /* XOR parity */ 82*4882a593Smuzhiyun q = dptr[z0+2]; /* RS syndrome */ 83*4882a593Smuzhiyun 84*4882a593Smuzhiyun for ( d = 0 ; d < bytes ; d += NSIZE*$# ) { 85*4882a593Smuzhiyun wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; 86*4882a593Smuzhiyun for ( z = z0-1 ; z >= 0 ; z-- ) { 87*4882a593Smuzhiyun wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; 88*4882a593Smuzhiyun wp$$ = vec_xor(wp$$, wd$$); 89*4882a593Smuzhiyun w2$$ = MASK(wq$$); 90*4882a593Smuzhiyun w1$$ = SHLBYTE(wq$$); 91*4882a593Smuzhiyun w2$$ = vec_and(w2$$, x1d); 92*4882a593Smuzhiyun w1$$ = vec_xor(w1$$, w2$$); 93*4882a593Smuzhiyun wq$$ = vec_xor(w1$$, wd$$); 94*4882a593Smuzhiyun } 95*4882a593Smuzhiyun *(unative_t *)&p[d+NSIZE*$$] = wp$$; 96*4882a593Smuzhiyun *(unative_t *)&q[d+NSIZE*$$] = wq$$; 97*4882a593Smuzhiyun } 98*4882a593Smuzhiyun} 99*4882a593Smuzhiyun 100*4882a593Smuzhiyunstatic void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs) 101*4882a593Smuzhiyun{ 102*4882a593Smuzhiyun preempt_disable(); 103*4882a593Smuzhiyun enable_kernel_altivec(); 104*4882a593Smuzhiyun 105*4882a593Smuzhiyun raid6_altivec$#_gen_syndrome_real(disks, bytes, ptrs); 106*4882a593Smuzhiyun 107*4882a593Smuzhiyun disable_kernel_altivec(); 108*4882a593Smuzhiyun preempt_enable(); 109*4882a593Smuzhiyun} 110*4882a593Smuzhiyun 111*4882a593Smuzhiyunint raid6_have_altivec(void); 112*4882a593Smuzhiyun#if $# == 1 113*4882a593Smuzhiyunint raid6_have_altivec(void) 114*4882a593Smuzhiyun{ 115*4882a593Smuzhiyun /* This assumes either all CPUs have Altivec or none does */ 116*4882a593Smuzhiyun# ifdef __KERNEL__ 117*4882a593Smuzhiyun return cpu_has_feature(CPU_FTR_ALTIVEC); 118*4882a593Smuzhiyun# else 119*4882a593Smuzhiyun return 1; 120*4882a593Smuzhiyun# endif 121*4882a593Smuzhiyun} 122*4882a593Smuzhiyun#endif 123*4882a593Smuzhiyun 124*4882a593Smuzhiyunconst struct raid6_calls raid6_altivec$# = { 125*4882a593Smuzhiyun raid6_altivec$#_gen_syndrome, 126*4882a593Smuzhiyun NULL, /* XOR not yet implemented */ 127*4882a593Smuzhiyun raid6_have_altivec, 128*4882a593Smuzhiyun "altivecx$#", 129*4882a593Smuzhiyun 0 130*4882a593Smuzhiyun}; 131*4882a593Smuzhiyun 132*4882a593Smuzhiyun#endif /* CONFIG_ALTIVEC */ 133