xref: /OK3568_Linux_fs/buildroot/package/vtun/0003-openssl11.patch (revision 4882a59341e53eb6f0b4789bf948001014eff981)

Fix build with OpenSSL 1.1.x

Extracted vtun-openssl.patch from
http://dl.fedoraproject.org/pub/fedora/linux/releases/27/Everything/source/tree/Packages/v/vtun-3.0.4-4.fc27.src.rpm

Signed-off-by: Bernd Kuhls <bernd.kuhls@t-online.de>

diff -NarU5 a/lfd_encrypt.c b/lfd_encrypt.c
--- a/lfd_encrypt.c	2016-10-01 17:27:51.000000000 -0400
+++ b/lfd_encrypt.c	2017-03-20 08:43:48.013308435 -0400
@@ -93,15 +93,15 @@
 static int dec_init_first_time;
 static unsigned long sequence_num;
 static char * pkey;
 static char * iv_buf;

-static EVP_CIPHER_CTX ctx_enc;	/* encrypt */
-static EVP_CIPHER_CTX ctx_dec;	/* decrypt */
+static EVP_CIPHER_CTX *ctx_enc;	/* encrypt */
+static EVP_CIPHER_CTX *ctx_dec;	/* decrypt */

-static EVP_CIPHER_CTX ctx_enc_ecb;	/* sideband ecb encrypt */
-static EVP_CIPHER_CTX ctx_dec_ecb;	/* sideband ecb decrypt */
+static EVP_CIPHER_CTX *ctx_enc_ecb;	/* sideband ecb encrypt */
+static EVP_CIPHER_CTX *ctx_dec_ecb;	/* sideband ecb decrypt */

 static int send_msg(int len, char *in, char **out);
 static int recv_msg(int len, char *in, char **out);
 static int send_ib_mesg(int *len, char **in);
 static int recv_ib_mesg(int *len, char **in);
@@ -180,37 +180,37 @@
       case VTUN_ENC_AES256CBC:
          blocksize = 16;
          keysize = 32;
          sb_init = 1;
          cipher_type = EVP_aes_256_ecb();
-         pctx_enc = &ctx_enc_ecb;
-         pctx_dec = &ctx_dec_ecb;
+         pctx_enc = ctx_enc_ecb;
+         pctx_dec = ctx_dec_ecb;
       break;

       case VTUN_ENC_AES256ECB:
          blocksize = 16;
          keysize = 32;
-         pctx_enc = &ctx_enc;
-         pctx_dec = &ctx_dec;
+         pctx_enc = ctx_enc;
+         pctx_dec = ctx_dec;
          cipher_type = EVP_aes_256_ecb();
          strcpy(cipher_name,"AES-256-ECB");
       break;
       case VTUN_ENC_AES128OFB:
       case VTUN_ENC_AES128CFB:
       case VTUN_ENC_AES128CBC:
          blocksize = 16;
          keysize = 16;
          sb_init=1;
          cipher_type = EVP_aes_128_ecb();
-         pctx_enc = &ctx_enc_ecb;
-         pctx_dec = &ctx_dec_ecb;
+         pctx_enc = ctx_enc_ecb;
+         pctx_dec = ctx_dec_ecb;
       break;
       case VTUN_ENC_AES128ECB:
          blocksize = 16;
          keysize = 16;
-         pctx_enc = &ctx_enc;
-         pctx_dec = &ctx_dec;
+         pctx_enc = ctx_enc;
+         pctx_dec = ctx_dec;
          cipher_type = EVP_aes_128_ecb();
          strcpy(cipher_name,"AES-128-ECB");
       break;

       case VTUN_ENC_BF256OFB:
@@ -219,20 +219,20 @@
          blocksize = 8;
          keysize = 32;
          var_key = 1;
          sb_init = 1;
          cipher_type = EVP_bf_ecb();
-         pctx_enc = &ctx_enc_ecb;
-         pctx_dec = &ctx_dec_ecb;
+         pctx_enc = ctx_enc_ecb;
+         pctx_dec = ctx_dec_ecb;
       break;

       case VTUN_ENC_BF256ECB:
          blocksize = 8;
          keysize = 32;
          var_key = 1;
-         pctx_enc = &ctx_enc;
-         pctx_dec = &ctx_dec;
+         pctx_enc = ctx_enc;
+         pctx_dec = ctx_dec;
          cipher_type = EVP_bf_ecb();
          strcpy(cipher_name,"Blowfish-256-ECB");
       break;

       case VTUN_ENC_BF128OFB:
@@ -241,26 +241,28 @@
          blocksize = 8;
          keysize = 16;
          var_key = 1;
          sb_init = 1;
          cipher_type = EVP_bf_ecb();
-         pctx_enc = &ctx_enc_ecb;
-         pctx_dec = &ctx_dec_ecb;
+         pctx_enc = ctx_enc_ecb;
+         pctx_dec = ctx_dec_ecb;
       break;
       case VTUN_ENC_BF128ECB: /* blowfish 128 ecb is the default */
       default:
          blocksize = 8;
          keysize = 16;
          var_key = 1;
-         pctx_enc = &ctx_enc;
-         pctx_dec = &ctx_dec;
+         pctx_enc = ctx_enc;
+         pctx_dec = ctx_dec;
          cipher_type = EVP_bf_ecb();
          strcpy(cipher_name,"Blowfish-128-ECB");
       break;
    } /* switch(host->cipher) */

    if (prep_key(&pkey, keysize, host) != 0) return -1;
+   pctx_enc = EVP_CIPHER_CTX_new();
+   pctx_dec = EVP_CIPHER_CTX_new();
    EVP_CIPHER_CTX_init(pctx_enc);
    EVP_CIPHER_CTX_init(pctx_dec);
    EVP_EncryptInit_ex(pctx_enc, cipher_type, NULL, NULL, NULL);
    EVP_DecryptInit_ex(pctx_dec, cipher_type, NULL, NULL, NULL);
    if (var_key)
@@ -292,14 +294,14 @@
    free_key(pkey); pkey = NULL;

    lfd_free(enc_buf); enc_buf = NULL;
    lfd_free(dec_buf); dec_buf = NULL;

-   EVP_CIPHER_CTX_cleanup(&ctx_enc);
-   EVP_CIPHER_CTX_cleanup(&ctx_dec);
-   EVP_CIPHER_CTX_cleanup(&ctx_enc_ecb);
-   EVP_CIPHER_CTX_cleanup(&ctx_dec_ecb);
+   EVP_CIPHER_CTX_free(ctx_enc);
+   EVP_CIPHER_CTX_free(ctx_dec);
+   EVP_CIPHER_CTX_free(ctx_enc_ecb);
+   EVP_CIPHER_CTX_free(ctx_dec_ecb);

    return 0;
 }

 static int encrypt_buf(int len, char *in, char **out)
@@ -321,11 +323,11 @@

    memset(in_ptr+len, pad, pad);
    outlen=len+pad;
    if (pad == blocksize)
       RAND_bytes(in_ptr+len, blocksize-1);
-   EVP_EncryptUpdate(&ctx_enc, out_ptr, &outlen, in_ptr, len+pad);
+   EVP_EncryptUpdate(ctx_enc, out_ptr, &outlen, in_ptr, len+pad);
    *out = enc_buf;

    sequence_num++;

    return outlen+msg_len;
@@ -341,11 +343,11 @@
    in = *out;
    in_ptr = in;

    outlen=len;
    if (!len) return 0;
-   EVP_DecryptUpdate(&ctx_dec, out_ptr, &outlen, in_ptr, len);
+   EVP_DecryptUpdate(ctx_dec, out_ptr, &outlen, in_ptr, len);
    recv_ib_mesg(&outlen, &out_ptr);
    if (!outlen) return 0;
    tmp_ptr = out_ptr + outlen; tmp_ptr--;
    pad = *tmp_ptr;
    if (pad < 1 || pad > blocksize) {
@@ -429,17 +431,18 @@
       /* if we're here, something weird's going on */
          return -1;
       break;
    } /* switch(cipher) */

-   EVP_CIPHER_CTX_init(&ctx_enc);
-   EVP_EncryptInit_ex(&ctx_enc, cipher_type, NULL, NULL, NULL);
+   ctx_enc = EVP_CIPHER_CTX_new();
+   EVP_CIPHER_CTX_init(ctx_enc);
+   EVP_EncryptInit_ex(ctx_enc, cipher_type, NULL, NULL, NULL);
    if (var_key)
-      EVP_CIPHER_CTX_set_key_length(&ctx_enc, keysize);
-   EVP_EncryptInit_ex(&ctx_enc, NULL, NULL, pkey, NULL);
-   EVP_EncryptInit_ex(&ctx_enc, NULL, NULL, NULL, iv);
-   EVP_CIPHER_CTX_set_padding(&ctx_enc, 0);
+      EVP_CIPHER_CTX_set_key_length(ctx_enc, keysize);
+   EVP_EncryptInit_ex(ctx_enc, NULL, NULL, pkey, NULL);
+   EVP_EncryptInit_ex(ctx_enc, NULL, NULL, NULL, iv);
+   EVP_CIPHER_CTX_set_padding(ctx_enc, 0);
    if (enc_init_first_time)
    {
       sprintf(tmpstr,"%s encryption initialized", cipher_name);
       vtun_syslog(LOG_INFO, tmpstr);
       enc_init_first_time = 0;
@@ -519,17 +522,18 @@
       /* if we're here, something weird's going on */
          return -1;
       break;
    } /* switch(cipher) */

-   EVP_CIPHER_CTX_init(&ctx_dec);
-   EVP_DecryptInit_ex(&ctx_dec, cipher_type, NULL, NULL, NULL);
+   ctx_dec = EVP_CIPHER_CTX_new();
+   EVP_CIPHER_CTX_init(ctx_dec);
+   EVP_DecryptInit_ex(ctx_dec, cipher_type, NULL, NULL, NULL);
    if (var_key)
-      EVP_CIPHER_CTX_set_key_length(&ctx_dec, keysize);
-   EVP_DecryptInit_ex(&ctx_dec, NULL, NULL, pkey, NULL);
-   EVP_DecryptInit_ex(&ctx_dec, NULL, NULL, NULL, iv);
-   EVP_CIPHER_CTX_set_padding(&ctx_dec, 0);
+      EVP_CIPHER_CTX_set_key_length(ctx_dec, keysize);
+   EVP_DecryptInit_ex(ctx_dec, NULL, NULL, pkey, NULL);
+   EVP_DecryptInit_ex(ctx_dec, NULL, NULL, NULL, iv);
+   EVP_CIPHER_CTX_set_padding(ctx_dec, 0);
    if (dec_init_first_time)
    {
       sprintf(tmpstr,"%s decryption initialized", cipher_name);
       vtun_syslog(LOG_INFO, tmpstr);
       dec_init_first_time = 0;
@@ -557,11 +561,11 @@
          memset(iv,0,blocksize); free(iv); iv = NULL;
          RAND_bytes(in_ptr, in - in_ptr);

          in_ptr = in - blocksize*2;
          outlen = blocksize*2;
-         EVP_EncryptUpdate(&ctx_enc_ecb, in_ptr,
+         EVP_EncryptUpdate(ctx_enc_ecb, in_ptr,
             &outlen, in_ptr, blocksize*2);
          *out = in_ptr;
          len = outlen;
          cipher_enc_state = CIPHER_SEQUENCE;
       break;
@@ -584,11 +588,11 @@
    {
       case CIPHER_INIT:
          in_ptr = in;
          iv = malloc(blocksize);
          outlen = blocksize*2;
-         EVP_DecryptUpdate(&ctx_dec_ecb, in_ptr, &outlen, in_ptr, blocksize*2);
+         EVP_DecryptUpdate(ctx_dec_ecb, in_ptr, &outlen, in_ptr, blocksize*2);

          if ( !strncmp(in_ptr, "ivec", 4) )
          {
             memcpy(iv, in_ptr+4, blocksize);
             cipher_dec_init(iv);
@@ -627,11 +631,11 @@
                   "Max. gibberish threshold reached");
 #endif
                if (cipher_enc_state != CIPHER_INIT)
                {
                   cipher_enc_state = CIPHER_INIT;
-                  EVP_CIPHER_CTX_cleanup(&ctx_enc);
+                  EVP_CIPHER_CTX_free(ctx_enc);
 #ifdef LFD_ENCRYPT_DEBUG
                   vtun_syslog(LOG_INFO,
                      "Forcing local encryptor re-init");
 #endif
                }
@@ -708,11 +712,11 @@
          *len -= blocksize;

          if (cipher_enc_state != CIPHER_INIT)
          {
             cipher_enc_state = CIPHER_INIT;
-            EVP_CIPHER_CTX_cleanup(&ctx_enc);
+            EVP_CIPHER_CTX_free(ctx_enc);
          }
 #ifdef LFD_ENCRYPT_DEBUG
          vtun_syslog(LOG_INFO, "Remote requests encryptor re-init");
 #endif
       }
@@ -722,11 +726,11 @@

          if (cipher_dec_state != CIPHER_INIT &&
              cipher_enc_state != CIPHER_REQ_INIT &&
              cipher_enc_state != CIPHER_INIT)
          {
-            EVP_CIPHER_CTX_cleanup (&ctx_dec);
+            EVP_CIPHER_CTX_free (ctx_dec);
             cipher_dec_state = CIPHER_INIT;
             cipher_enc_state = CIPHER_REQ_INIT;
          }
 #ifdef LFD_ENCRYPT_DEBUG
          vtun_syslog(LOG_INFO, "Local decryptor out of sync");