1 files changed, 27 insertions, 10 deletions

diff --git a/src/iperf_auth.c b/src/iperf_auth.c
index c89bde7..72e85fc 100644
--- a/src/iperf_auth.c
+++ b/src/iperf_auth.c
@@ -230,7 +230,7 @@ int test_load_private_key_from_file(const char *file){
     return 0;
 }
 
-int encrypt_rsa_message(const char *plaintext, EVP_PKEY *public_key, unsigned char **encryptedtext) {
+int encrypt_rsa_message(const char *plaintext, EVP_PKEY *public_key, unsigned char **encryptedtext, int use_pkcs1_padding) {
 #if OPENSSL_VERSION_MAJOR >= 3
     EVP_PKEY_CTX *ctx;
 #else
@@ -257,12 +257,19 @@ int encrypt_rsa_message(const char *plaintext, EVP_PKEY *public_key, unsigned ch
 
     BIO *bioBuff   = BIO_new_mem_buf((void*)plaintext, (int)strlen(plaintext));
     rsa_buffer_len = BIO_read(bioBuff, rsa_buffer, keysize * 2);
+
+    int padding = RSA_PKCS1_OAEP_PADDING;
+    if (use_pkcs1_padding){
+        padding = RSA_PKCS1_PADDING;
+    }
 #if OPENSSL_VERSION_MAJOR >= 3
     EVP_PKEY_encrypt_init(ctx);
+    EVP_PKEY_CTX_set_rsa_padding(ctx, padding);
+
     EVP_PKEY_encrypt(ctx, *encryptedtext, &encryptedtext_len, rsa_buffer, rsa_buffer_len);
     EVP_PKEY_CTX_free(ctx);
 #else
-    encryptedtext_len = RSA_public_encrypt(rsa_buffer_len, rsa_buffer, *encryptedtext, rsa, RSA_PKCS1_PADDING);
+    encryptedtext_len = RSA_public_encrypt(rsa_buffer_len, rsa_buffer, *encryptedtext, rsa, padding);
     RSA_free(rsa);
 #endif
 
@@ -280,7 +287,7 @@ int encrypt_rsa_message(const char *plaintext, EVP_PKEY *public_key, unsigned ch
     return 0;
 }
 
-int decrypt_rsa_message(const unsigned char *encryptedtext, const int encryptedtext_len, EVP_PKEY *private_key, unsigned char **plaintext) {
+int decrypt_rsa_message(const unsigned char *encryptedtext, const int encryptedtext_len, EVP_PKEY *private_key, unsigned char **plaintext, int use_pkcs1_padding) {
 #if OPENSSL_VERSION_MAJOR >= 3
     EVP_PKEY_CTX *ctx;
 #else
@@ -307,21 +314,31 @@ int decrypt_rsa_message(const unsigned char *encryptedtext, const int encryptedt
 
     BIO *bioBuff   = BIO_new_mem_buf((void*)encryptedtext, encryptedtext_len);
     rsa_buffer_len = BIO_read(bioBuff, rsa_buffer, keysize * 2);
+
+    int padding = RSA_PKCS1_OAEP_PADDING;
+    if (use_pkcs1_padding){
+        padding = RSA_PKCS1_PADDING;
+    }
 #if OPENSSL_VERSION_MAJOR >= 3
     plaintext_len = keysize;
     EVP_PKEY_decrypt_init(ctx);
+    int ret = EVP_PKEY_CTX_set_rsa_padding(ctx, padding);
+    if (ret < 0){
+        goto errreturn;
+    }
     EVP_PKEY_decrypt(ctx, *plaintext, &plaintext_len, rsa_buffer, rsa_buffer_len);
     EVP_PKEY_CTX_free(ctx);
 #else
-    plaintext_len = RSA_private_decrypt(rsa_buffer_len, rsa_buffer, *plaintext, rsa, RSA_PKCS1_PADDING);
+    plaintext_len = RSA_private_decrypt(rsa_buffer_len, rsa_buffer, *plaintext, rsa, padding);
     RSA_free(rsa);
 #endif
 
     OPENSSL_free(rsa_buffer);
     BIO_free(bioBuff);
 
-    if (plaintext_len <= 0) {
-        goto errreturn;
+    /* Treat a decryption error as an empty string. */
+    if (plaintext_len < 0) {
+        plaintext_len = 0;
     }
 
     return plaintext_len;
@@ -331,7 +348,7 @@ int decrypt_rsa_message(const unsigned char *encryptedtext, const int encryptedt
     return 0;
 }
 
-int encode_auth_setting(const char *username, const char *password, EVP_PKEY *public_key, char **authtoken){
+int encode_auth_setting(const char *username, const char *password, EVP_PKEY *public_key, char **authtoken, int use_pkcs1_padding){
     time_t t = time(NULL);
     time_t utc_seconds = mktime(localtime(&t));
 
@@ -348,7 +365,7 @@ int encode_auth_setting(const char *username, const char *password, EVP_PKEY *pu
 
     unsigned char *encrypted = NULL;
     int encrypted_len;
-    encrypted_len = encrypt_rsa_message(text, public_key, &encrypted);
+    encrypted_len = encrypt_rsa_message(text, public_key, &encrypted, use_pkcs1_padding);
     free(text);
     if (encrypted_len < 0) {
       return -1;
@@ -359,7 +376,7 @@ int encode_auth_setting(const char *username, const char *password, EVP_PKEY *pu
     return (0); //success
 }
 
-int decode_auth_setting(int enable_debug, const char *authtoken, EVP_PKEY *private_key, char **username, char **password, time_t *ts){
+int decode_auth_setting(int enable_debug, const char *authtoken, EVP_PKEY *private_key, char **username, char **password, time_t *ts, int use_pkcs1_padding){
     unsigned char *encrypted_b64 = NULL;
     size_t encrypted_len_b64;
     int64_t utc_seconds;
@@ -367,7 +384,7 @@ int decode_auth_setting(int enable_debug, const char *authtoken, EVP_PKEY *priva
 
     unsigned char *plaintext = NULL;
     int plaintext_len;
-    plaintext_len = decrypt_rsa_message(encrypted_b64, encrypted_len_b64, private_key, &plaintext);
+    plaintext_len = decrypt_rsa_message(encrypted_b64, encrypted_len_b64, private_key, &plaintext, use_pkcs1_padding);
     free(encrypted_b64);
     if (plaintext_len < 0) {
         return -1;