rfbsrc: make authentication handshake safe for multiple instances

Move DES key into context struct.
This commit is contained in:
Tim-Philipp Müller 2013-05-11 00:29:40 +01:00
parent 0fc4af8498
commit 14d20271c1
3 changed files with 39 additions and 32 deletions

View file

@ -34,9 +34,9 @@
static void scrunch (unsigned char *, unsigned long *);
static void unscrun (unsigned long *, unsigned char *);
static void desfunc (unsigned long *, unsigned long *);
static void cookey (unsigned long *);
static void cookey (DESContext * ctx, unsigned long *);
static unsigned long KnL[32] = { 0L };
//static unsigned long KnL[32] = { 0L };
//static unsigned long KnR[32] = { 0L };
//static unsigned long Kn3[32] = { 0L };
@ -47,11 +47,11 @@ static unsigned long KnL[32] = { 0L };
* 0x89,0xab,0xcd,0xef,0x01,0x23,0x45,0x67 };
*/
static uint16_t bytebit[8] = {
static const uint16_t bytebit[8] = {
01, 02, 04, 010, 020, 040, 0100, 0200
};
static unsigned long bigbyte[24] = {
static const unsigned long bigbyte[24] = {
0x800000L, 0x400000L, 0x200000L, 0x100000L,
0x80000L, 0x40000L, 0x20000L, 0x10000L,
0x8000L, 0x4000L, 0x2000L, 0x1000L,
@ -62,18 +62,18 @@ static unsigned long bigbyte[24] = {
/* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
static unsigned char pc1[56] = {
static const unsigned char pc1[56] = {
56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17,
9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35,
62, 54, 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21,
13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3
};
static unsigned char totrot[16] = {
static const unsigned char totrot[16] = {
1, 2, 4, 6, 8, 10, 12, 14, 15, 17, 19, 21, 23, 25, 27, 28
};
static unsigned char pc2[48] = {
static const unsigned char pc2[48] = {
13, 16, 10, 23, 0, 4, 2, 27, 14, 5, 20, 9,
22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1,
40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
@ -81,7 +81,7 @@ static unsigned char pc2[48] = {
};
void
deskey (unsigned char *key, int32_t edf)
deskey (DESContext * ctx, unsigned char *key, int32_t edf)
{ /* Thanks to James Gillogly & Phil Karn! */
register int32_t i, j, l, m, n;
unsigned char pc1m[56], pcr[56];
@ -120,12 +120,12 @@ deskey (unsigned char *key, int32_t edf)
kn[n] |= bigbyte[j];
}
}
cookey (kn);
cookey (ctx, kn);
return;
}
static void
cookey (register unsigned long *raw1)
cookey (DESContext * ctx, register unsigned long *raw1)
{
register unsigned long *cook, *raw0;
unsigned long dough[32];
@ -143,39 +143,39 @@ cookey (register unsigned long *raw1)
*cook |= (*raw1 & 0x0003f000L) >> 4;
*cook++ |= (*raw1 & 0x0000003fL);
}
usekey (dough);
usekey (ctx, dough);
return;
}
void
cpkey (register unsigned long *into)
cpkey (DESContext * ctx, register unsigned long *into)
{
register unsigned long *from, *endp;
from = KnL, endp = &KnL[32];
from = ctx->KnL, endp = &ctx->KnL[32];
while (from < endp)
*into++ = *from++;
return;
}
void
usekey (register unsigned long *from)
usekey (DESContext * ctx, register unsigned long *from)
{
register unsigned long *to, *endp;
to = KnL, endp = &KnL[32];
to = ctx->KnL, endp = &ctx->KnL[32];
while (to < endp)
*to++ = *from++;
return;
}
void
des (unsigned char *inblock, unsigned char *outblock)
des (DESContext * ctx, unsigned char *inblock, unsigned char *outblock)
{
unsigned long work[2];
scrunch (inblock, work);
desfunc (work, KnL);
desfunc (work, ctx->KnL);
unscrun (work, outblock);
return;
}
@ -208,7 +208,7 @@ unscrun (register unsigned long *outof, register unsigned char *into)
return;
}
static unsigned long SP1[64] = {
static const unsigned long SP1[64] = {
0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
@ -227,7 +227,7 @@ static unsigned long SP1[64] = {
0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L
};
static unsigned long SP2[64] = {
static const unsigned long SP2[64] = {
0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
@ -246,7 +246,7 @@ static unsigned long SP2[64] = {
0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L
};
static unsigned long SP3[64] = {
static const unsigned long SP3[64] = {
0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
@ -265,7 +265,7 @@ static unsigned long SP3[64] = {
0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L
};
static unsigned long SP4[64] = {
static const unsigned long SP4[64] = {
0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
@ -284,7 +284,7 @@ static unsigned long SP4[64] = {
0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L
};
static unsigned long SP5[64] = {
static const unsigned long SP5[64] = {
0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
@ -303,7 +303,7 @@ static unsigned long SP5[64] = {
0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L
};
static unsigned long SP6[64] = {
static const unsigned long SP6[64] = {
0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
@ -322,7 +322,7 @@ static unsigned long SP6[64] = {
0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L
};
static unsigned long SP7[64] = {
static const unsigned long SP7[64] = {
0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
@ -341,7 +341,7 @@ static unsigned long SP7[64] = {
0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L
};
static unsigned long SP8[64] = {
static const unsigned long SP8[64] = {
0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,

View file

@ -22,7 +22,12 @@
#define EN0 0 /* MODE == encrypt */
#define DE1 1 /* MODE == decrypt */
extern void deskey (unsigned char *, int);
/* init to 0 */
typedef struct {
unsigned long KnL[32];
} DESContext;
extern void deskey (DESContext * ctx, unsigned char *, int);
/* hexkey[8] MODE
* Sets the internal key register according to the hexadecimal
@ -30,20 +35,20 @@ extern void deskey (unsigned char *, int);
* for encryption or decryption according to MODE.
*/
extern void usekey (unsigned long *);
extern void usekey (DESContext * ctx, unsigned long *);
/* cookedkey[32]
* Loads the internal key register with the data in cookedkey.
*/
extern void cpkey (unsigned long *);
extern void cpkey (DESContext * ctx, unsigned long *);
/* cookedkey[32]
* Copies the contents of the internal key register into the storage
* located at &cookedkey[0].
*/
extern void des (unsigned char *, unsigned char *);
extern void des (DESContext * ctx, unsigned char *, unsigned char *);
/* from[8] to[8]
* Encrypts/Decrypts (according to the key currently loaded in the

View file

@ -424,6 +424,7 @@ rfb_decoder_state_wait_for_security (RfbDecoder * decoder)
break;
case SECURITY_VNC:{
unsigned char key[8], *challenge;
DESContext des_ctx;
gsize password_len;
/*
@ -449,9 +450,10 @@ rfb_decoder_state_wait_for_security (RfbDecoder * decoder)
return FALSE;
/* encrypt 16 challenge bytes in place using key */
deskey (key, EN0);
des (challenge, challenge);
des (challenge + 8, challenge + 8);
memset (&des_ctx, 0, sizeof (DESContext));
deskey (&des_ctx, key, EN0);
des (&des_ctx, challenge, challenge);
des (&des_ctx, challenge + 8, challenge + 8);
/* .. and send back to server */
rfb_decoder_send (decoder, challenge, 16);