From e2d313005ec4673506107941cbe6e1b45f8ae066 Mon Sep 17 00:00:00 2001 From: Christian Hitz Date: Wed, 27 Mar 2024 13:02:50 +0100 Subject: [PATCH] handle failure in encrypt_rfbdes() in callers --- include/rfb/rfbproto.h | 2 +- src/common/vncauth.c | 23 +++++++++++++++++------ src/libvncclient/rfbclient.c | 29 ++++++++++++++++++++++------- src/libvncserver/main.c | 11 +++++++++-- 4 files changed, 49 insertions(+), 16 deletions(-) diff --git a/include/rfb/rfbproto.h b/include/rfb/rfbproto.h index ebcd303cc..62e090804 100644 --- a/include/rfb/rfbproto.h +++ b/include/rfb/rfbproto.h @@ -1557,7 +1557,7 @@ typedef union { extern int rfbEncryptAndStorePasswd(char *passwd, char *fname); extern char *rfbDecryptPasswdFromFile(char *fname); extern void rfbRandomBytes(unsigned char *bytes); -extern void rfbEncryptBytes(unsigned char *bytes, char *passwd); +extern int rfbEncryptBytes(unsigned char *bytes, char *passwd); #endif diff --git a/src/common/vncauth.c b/src/common/vncauth.c index c3f318c3a..0b8aa38d8 100644 --- a/src/common/vncauth.c +++ b/src/common/vncauth.c @@ -102,7 +102,10 @@ rfbEncryptAndStorePasswd(char *passwd, char *fname) /* Do encryption in-place - this way we overwrite our copy of the plaintext password */ - encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd)); + if (encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd)) == 0) { + fclose(fp); + return 1; + } for (i = 0; i < 8; i++) { putc(encryptedPasswd[i], fp); @@ -180,7 +183,7 @@ rfbRandomBytes(unsigned char *bytes) * Encrypt CHALLENGESIZE bytes in memory using a password. */ -void +int rfbEncryptBytes(unsigned char *bytes, char *passwd) { unsigned char key[8]; @@ -197,19 +200,27 @@ rfbEncryptBytes(unsigned char *bytes, char *passwd) } } - encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE); + if (encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE) == 0) { + return 1; + } + return 0; } -void +int rfbEncryptBytes2(unsigned char *where, const int length, unsigned char *key) { int i, j, out_len; for (i = 0; i< 8; i++) where[i] ^= key[i]; - encrypt_rfbdes(where, &out_len, key, where, 8); + if (encrypt_rfbdes(where, &out_len, key, where, 8) == 0) { + return 1; + } for (i = 8; i < length; i += 8) { for (j = 0; j < 8; j++) { where[i + j] ^= where[i + j - 8]; } - encrypt_rfbdes(where + i, &out_len, key, where + i, 8); + if (encrypt_rfbdes(where + i, &out_len, key, where + i, 8) == 0) { + return 1; + } } + return 0; } diff --git a/src/libvncclient/rfbclient.c b/src/libvncclient/rfbclient.c index 6f0afac20..4734d1a8b 100644 --- a/src/libvncclient/rfbclient.c +++ b/src/libvncclient/rfbclient.c @@ -404,8 +404,8 @@ rfbBool ConnectToRFBRepeater(rfbClient* client,const char *repeaterHost, int rep return TRUE; } -extern void rfbClientEncryptBytes(unsigned char* bytes, char* passwd); -extern void rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key); +extern int rfbClientEncryptBytes(unsigned char* bytes, char* passwd); +extern int rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key); static void ReadReason(rfbClient* client) @@ -585,7 +585,10 @@ HandleVncAuth(rfbClient *client) passwd[8] = '\0'; } - rfbClientEncryptBytes(challenge, passwd); + if (rfbClientEncryptBytes(challenge, passwd) != 0) { + rfbClientLog("Encryption failed\n"); + return FALSE; + } /* Lose the password from memory */ for (i = strlen(passwd); i >= 0; i--) { @@ -733,8 +736,14 @@ HandleUltraMSLogonIIAuth(rfbClient *client) strncpy((char *)password, cred->userCredential.password, sizeof(password)-1); FreeUserCredential(cred); - rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key); - rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key); + if (rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key) != 0) { + rfbClientLog("Encrypting username failed\n"); + return FALSE; + } + if (rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key) != 0) { + rfbClientLog("Encrypting password failed\n"); + return FALSE; + } if (!WriteToRFBServer(client, (char *)pub, sizeof(pub))) return FALSE; if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE; @@ -789,8 +798,14 @@ HandleMSLogonAuth(rfbClient *client) pub = rfbClientSwap64IfLE(pub); key = rfbClientSwap64IfLE(key); - rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)&key); - rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)&key); + if (rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key) != 0) { + rfbClientLog("Encrypting username failed\n"); + return FALSE; + } + if (rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key) != 0) { + rfbClientLog("Encrypting password failed\n"); + return FALSE; + } if (!WriteToRFBServer(client, (char *)&pub, 8)) return FALSE; if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE; diff --git a/src/libvncserver/main.c b/src/libvncserver/main.c index 1efa83879..5d160c32f 100644 --- a/src/libvncserver/main.c +++ b/src/libvncserver/main.c @@ -792,7 +792,11 @@ static rfbBool rfbDefaultPasswordCheck(rfbClientPtr cl,const char* response,int return(FALSE); } - rfbEncryptBytes(cl->authChallenge, passwd); + if (rfbEncryptBytes(cl->authChallenge, passwd) != 0) { + rfbErr("Encryption failed\n"); + free(passwd); + return(FALSE); + } /* Lose the password from memory */ for (i = strlen(passwd); i >= 0; i--) { @@ -820,7 +824,10 @@ rfbBool rfbCheckPasswordByList(rfbClientPtr cl,const char* response,int len) for(passwds=(char**)cl->screen->authPasswdData;*passwds;passwds++,i++) { uint8_t auth_tmp[CHALLENGESIZE]; memcpy((char *)auth_tmp, (char *)cl->authChallenge, CHALLENGESIZE); - rfbEncryptBytes(auth_tmp, *passwds); + if (rfbEncryptBytes(auth_tmp, *passwds) != 0) { + rfbErr("Encryption failed\n"); + return(FALSE); + } if (memcmp(auth_tmp, response, len) == 0) { if(i>=cl->screen->authPasswdFirstViewOnly)