1 /*
2 * Driver for /dev/crypto device (aka CryptoDev)
3 *
4 * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
5 * Copyright (c) 2009-2013 Nikos Mavrogiannopoulos <nmav@gnutls.org>
6 *
7 * This file is part of linux cryptodev.
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version 2
12 * of the License, or (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc.,
22 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
23 */
24
25 /*
26 * Device /dev/crypto provides an interface for
27 * accessing kernel CryptoAPI algorithms (ciphers,
28 * hashes) from userspace programs.
29 *
30 * /dev/crypto interface was originally introduced in
31 * OpenBSD and this module attempts to keep the API.
32 *
33 */
34 #include <crypto/hash.h>
35 #include <linux/crypto.h>
36 #include <linux/mm.h>
37 #include <linux/highmem.h>
38 #include <linux/ioctl.h>
39 #include <linux/random.h>
40 #include <linux/syscalls.h>
41 #include <linux/pagemap.h>
42 #include <linux/poll.h>
43 #include <linux/uaccess.h>
44 #include <crypto/scatterwalk.h>
45 #include <linux/scatterlist.h>
46 #include "cryptodev.h"
47 #include "zc.h"
48 #include "cryptlib.h"
49 #include "version.h"
50
51 /* This file contains the traditional operations of encryption
52 * and hashing of /dev/crypto.
53 */
54
55 static int
hash_n_crypt(struct csession * ses_ptr,struct crypt_op * cop,struct scatterlist * src_sg,struct scatterlist * dst_sg,uint32_t len)56 hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
57 struct scatterlist *src_sg, struct scatterlist *dst_sg,
58 uint32_t len)
59 {
60 int ret;
61
62 /* Always hash before encryption and after decryption. Maybe
63 * we should introduce a flag to switch... TBD later on.
64 */
65 if (cop->op == COP_ENCRYPT) {
66 if (ses_ptr->hdata.init != 0) {
67 ret = cryptodev_hash_update(&ses_ptr->hdata,
68 src_sg, len);
69 if (unlikely(ret))
70 goto out_err;
71 }
72 if (ses_ptr->cdata.init != 0) {
73 ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
74 src_sg, dst_sg, len);
75
76 if (unlikely(ret))
77 goto out_err;
78 }
79 } else {
80 if (ses_ptr->cdata.init != 0) {
81 ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
82 src_sg, dst_sg, len);
83
84 if (unlikely(ret))
85 goto out_err;
86 }
87
88 if (ses_ptr->hdata.init != 0) {
89 ret = cryptodev_hash_update(&ses_ptr->hdata,
90 dst_sg, len);
91 if (unlikely(ret))
92 goto out_err;
93 }
94 }
95 return 0;
96 out_err:
97 derr(0, "CryptoAPI failure: %d", ret);
98 return ret;
99 }
100
101 /* This is the main crypto function - feed it with plaintext
102 and get a ciphertext (or vice versa :-) */
103 static int
__crypto_run_std(struct csession * ses_ptr,struct crypt_op * cop)104 __crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
105 {
106 char *data;
107 char __user *src, *dst;
108 struct scatterlist sg;
109 size_t nbytes, bufsize;
110 int ret = 0;
111
112 nbytes = cop->len;
113 data = (char *)__get_free_page(GFP_KERNEL);
114
115 if (unlikely(!data)) {
116 derr(1, "Error getting free page.");
117 return -ENOMEM;
118 }
119
120 bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
121
122 src = cop->src;
123 dst = cop->dst;
124
125 while (nbytes > 0) {
126 size_t current_len = nbytes > bufsize ? bufsize : nbytes;
127
128 if (unlikely(copy_from_user(data, src, current_len))) {
129 derr(1, "Error copying %zu bytes from user address %p.", current_len, src);
130 ret = -EFAULT;
131 break;
132 }
133
134 sg_init_one(&sg, data, current_len);
135
136 ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
137
138 if (unlikely(ret)) {
139 derr(1, "hash_n_crypt failed.");
140 break;
141 }
142
143 if (ses_ptr->cdata.init != 0) {
144 if (unlikely(copy_to_user(dst, data, current_len))) {
145 derr(1, "could not copy to user.");
146 ret = -EFAULT;
147 break;
148 }
149 }
150
151 dst += current_len;
152 nbytes -= current_len;
153 src += current_len;
154 }
155
156 free_page((unsigned long)data);
157 return ret;
158 }
159
160
161
162 /* This is the main crypto function - zero-copy edition */
163 static int
__crypto_run_zc(struct csession * ses_ptr,struct kernel_crypt_op * kcop)164 __crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
165 {
166 struct scatterlist *src_sg, *dst_sg;
167 struct crypt_op *cop = &kcop->cop;
168 int ret = 0;
169
170 ret = cryptodev_get_userbuf(ses_ptr, cop->src, cop->len, cop->dst, cop->len,
171 kcop->task, kcop->mm, &src_sg, &dst_sg);
172 if (unlikely(ret)) {
173 derr(1, "Error getting user pages. Falling back to non zero copy.");
174 return __crypto_run_std(ses_ptr, cop);
175 }
176
177 ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
178
179 cryptodev_release_user_pages(ses_ptr);
180 return ret;
181 }
182
crypto_run(struct fcrypt * fcr,struct kernel_crypt_op * kcop)183 int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)
184 {
185 struct csession *ses_ptr;
186 struct crypt_op *cop = &kcop->cop;
187 int ret;
188
189 if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
190 ddebug(1, "invalid operation op=%u", cop->op);
191 return -EINVAL;
192 }
193
194 /* this also enters ses_ptr->sem */
195 ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
196 if (unlikely(!ses_ptr)) {
197 derr(1, "invalid session ID=0x%08X", cop->ses);
198 return -EINVAL;
199 }
200
201 if (ses_ptr->hdata.init != 0 && (cop->flags == 0 || cop->flags & COP_FLAG_RESET)) {
202 ret = cryptodev_hash_reset(&ses_ptr->hdata);
203 if (unlikely(ret)) {
204 derr(1, "error in cryptodev_hash_reset()");
205 goto out_unlock;
206 }
207 }
208
209 if (ses_ptr->cdata.init != 0) {
210 int blocksize = ses_ptr->cdata.blocksize;
211
212 if (unlikely(cop->len % blocksize)) {
213 derr(1, "data size (%u) isn't a multiple of block size (%u)",
214 cop->len, blocksize);
215 ret = -EINVAL;
216 goto out_unlock;
217 }
218
219 cryptodev_cipher_set_iv(&ses_ptr->cdata, kcop->iv,
220 min(ses_ptr->cdata.ivsize, kcop->ivlen));
221 }
222
223 if (likely(cop->len)) {
224 if (!(cop->flags & COP_FLAG_NO_ZC)) {
225 if (unlikely(ses_ptr->alignmask && !IS_ALIGNED((unsigned long)cop->src, ses_ptr->alignmask + 1))) {
226 dwarning(2, "source address %p is not %d byte aligned - disabling zero copy",
227 cop->src, ses_ptr->alignmask + 1);
228 cop->flags |= COP_FLAG_NO_ZC;
229 }
230
231 if (unlikely(ses_ptr->alignmask && !IS_ALIGNED((unsigned long)cop->dst, ses_ptr->alignmask + 1))) {
232 dwarning(2, "destination address %p is not %d byte aligned - disabling zero copy",
233 cop->dst, ses_ptr->alignmask + 1);
234 cop->flags |= COP_FLAG_NO_ZC;
235 }
236 }
237
238 if (cop->flags & COP_FLAG_NO_ZC)
239 ret = __crypto_run_std(ses_ptr, &kcop->cop);
240 else
241 ret = __crypto_run_zc(ses_ptr, kcop);
242 if (unlikely(ret))
243 goto out_unlock;
244 }
245
246 if (ses_ptr->cdata.init != 0) {
247 cryptodev_cipher_get_iv(&ses_ptr->cdata, kcop->iv,
248 min(ses_ptr->cdata.ivsize, kcop->ivlen));
249 }
250
251 if (ses_ptr->hdata.init != 0 &&
252 ((cop->flags & COP_FLAG_FINAL) ||
253 (!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
254
255 ret = cryptodev_hash_final(&ses_ptr->hdata, kcop->hash_output);
256 if (unlikely(ret)) {
257 derr(0, "CryptoAPI failure: %d", ret);
258 goto out_unlock;
259 }
260 kcop->digestsize = ses_ptr->hdata.digestsize;
261 }
262
263 out_unlock:
264 crypto_put_session(ses_ptr);
265 return ret;
266 }
267