1 /* LibTomCrypt, modular cryptographic library -- Tom St Denis */
2 /* SPDX-License-Identifier: Unlicense */
3 #include "tomcrypt_private.h"
4
5 /**
6 @file pkcs_1_oaep_decode.c
7 OAEP Padding for PKCS #1, Tom St Denis
8 */
9
10 #ifdef LTC_PKCS_1
11
12 /**
13 PKCS #1 v2.00 OAEP decode
14 @param msg The encoded data to decode
15 @param msglen The length of the encoded data (octets)
16 @param lparam The session or system data (can be NULL)
17 @param lparamlen The length of the lparam
18 @param modulus_bitlen The bit length of the RSA modulus
19 @param mgf_hash The hash algorithm used for the MGF
20 @param lparam_hash The hash algorithm used when hashing the lparam (can be -1)
21 @param out [out] Destination of decoding
22 @param outlen [in/out] The max size and resulting size of the decoding
23 @param res [out] Result of decoding, 1==valid, 0==invalid
24 @return CRYPT_OK if successful
25 */
pkcs_1_oaep_decode(const unsigned char * msg,unsigned long msglen,const unsigned char * lparam,unsigned long lparamlen,unsigned long modulus_bitlen,int mgf_hash,int lparam_hash,unsigned char * out,unsigned long * outlen,int * res)26 int pkcs_1_oaep_decode(const unsigned char *msg, unsigned long msglen,
27 const unsigned char *lparam, unsigned long lparamlen,
28 unsigned long modulus_bitlen,
29 int mgf_hash, int lparam_hash,
30 unsigned char *out, unsigned long *outlen,
31 int *res)
32 {
33 unsigned char *DB, *seed, *mask;
34 unsigned long hLen, x, y, modulus_len;
35 int err, ret, lparam_hash_used;
36
37 LTC_ARGCHK(msg != NULL);
38 LTC_ARGCHK(out != NULL);
39 LTC_ARGCHK(outlen != NULL);
40 LTC_ARGCHK(res != NULL);
41
42 /* default to invalid packet */
43 *res = 0;
44
45 /* test valid hash */
46 if ((err = hash_is_valid(mgf_hash)) != CRYPT_OK) {
47 return err;
48 }
49 if (lparam_hash != -1) {
50 if ((err = hash_is_valid(lparam_hash)) != CRYPT_OK) {
51 return err;
52 }
53 lparam_hash_used = lparam_hash;
54 } else {
55 lparam_hash_used = mgf_hash;
56 }
57 hLen = hash_descriptor[lparam_hash_used]->hashsize;
58 modulus_len = (modulus_bitlen >> 3) + (modulus_bitlen & 7 ? 1 : 0);
59
60 /* test hash/message size */
61 if ((2*hLen >= (modulus_len - 2)) || (msglen != modulus_len)) {
62 return CRYPT_PK_INVALID_SIZE;
63 }
64
65 /* allocate ram for DB/mask/salt of size modulus_len */
66 DB = XMALLOC(modulus_len);
67 mask = XMALLOC(modulus_len);
68 seed = XMALLOC(hLen);
69 if (DB == NULL || mask == NULL || seed == NULL) {
70 if (DB != NULL) {
71 XFREE(DB);
72 }
73 if (mask != NULL) {
74 XFREE(mask);
75 }
76 if (seed != NULL) {
77 XFREE(seed);
78 }
79 return CRYPT_MEM;
80 }
81
82 /* ok so it's now in the form
83
84 0x00 || maskedseed || maskedDB
85
86 1 || hLen || modulus_len - hLen - 1
87
88 */
89
90 ret = CRYPT_OK;
91
92 /* must have leading 0x00 byte */
93 if (msg[0] != 0x00) {
94 ret = CRYPT_INVALID_PACKET;
95 }
96
97 /* now read the masked seed */
98 x = 1;
99 XMEMCPY(seed, msg + x, hLen);
100 x += hLen;
101
102 /* now read the masked DB */
103 XMEMCPY(DB, msg + x, modulus_len - hLen - 1);
104 x += modulus_len - hLen - 1;
105
106 /* compute MGF1 of maskedDB (hLen) */
107 if ((err = pkcs_1_mgf1(mgf_hash, DB, modulus_len - hLen - 1, mask, hLen)) != CRYPT_OK) {
108 goto LBL_ERR;
109 }
110
111 /* XOR against seed */
112 for (y = 0; y < hLen; y++) {
113 seed[y] ^= mask[y];
114 }
115
116 /* compute MGF1 of seed (k - hlen - 1) */
117 if ((err = pkcs_1_mgf1(mgf_hash, seed, hLen, mask, modulus_len - hLen - 1)) != CRYPT_OK) {
118 goto LBL_ERR;
119 }
120
121 /* xor against DB */
122 for (y = 0; y < (modulus_len - hLen - 1); y++) {
123 DB[y] ^= mask[y];
124 }
125
126 /* now DB == lhash || PS || 0x01 || M, PS == k - mlen - 2hlen - 2 zeroes */
127
128 /* compute lhash and store it in seed [reuse temps!] */
129 x = modulus_len;
130 if (lparam != NULL) {
131 if ((err = hash_memory(lparam_hash_used, lparam, lparamlen, seed, &x)) != CRYPT_OK) {
132 goto LBL_ERR;
133 }
134 } else {
135 /* can't pass hash_memory a NULL so use DB with zero length */
136 if ((err = hash_memory(lparam_hash_used, DB, 0, seed, &x)) != CRYPT_OK) {
137 goto LBL_ERR;
138 }
139 }
140
141 /* compare the lhash'es */
142 if (XMEM_NEQ(seed, DB, hLen) != 0) {
143 ret = CRYPT_INVALID_PACKET;
144 }
145
146 /* now zeroes before a 0x01 */
147 for (x = hLen; x < (modulus_len - hLen - 1) && DB[x] == 0x00; x++) {
148 /* step... */
149 }
150
151 /* error if wasn't 0x01 */
152 if (x == (modulus_len - hLen - 1) || DB[x] != 0x01) {
153 ret = CRYPT_INVALID_PACKET;
154 }
155
156 /* rest is the message (and skip 0x01) */
157 if ((modulus_len - hLen - 1 - ++x) > *outlen) {
158 ret = CRYPT_INVALID_PACKET;
159 }
160
161 if (ret == CRYPT_OK) {
162 /* copy message */
163 *outlen = modulus_len - hLen - 1 - x;
164 XMEMCPY(out, DB + x, modulus_len - hLen - 1 - x);
165
166 /* valid packet */
167 *res = 1;
168 }
169 err = ret;
170
171 LBL_ERR:
172 #ifdef LTC_CLEAN_STACK
173 zeromem(DB, modulus_len);
174 zeromem(seed, hLen);
175 zeromem(mask, modulus_len);
176 #endif
177
178 XFREE(seed);
179 XFREE(mask);
180 XFREE(DB);
181
182 return err;
183 }
184
185 #endif /* LTC_PKCS_1 */
186