1 //========================================================================
5 // Copyright 1996-2002 Glyph & Cog, LLC
7 //========================================================================
10 #pragma implementation
17 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state);
18 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c);
19 static void md5(Guchar *msg, int msgLen, Guchar *digest);
21 static Guchar passwordPad[32] = {
22 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
23 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
24 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80,
25 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
28 //------------------------------------------------------------------------
30 //------------------------------------------------------------------------
32 Decrypt::Decrypt(Guchar *fileKey, int keyLength, int objNum, int objGen) {
35 // construct object key
36 for (i = 0; i < keyLength; ++i) {
37 objKey[i] = fileKey[i];
39 objKey[keyLength] = objNum & 0xff;
40 objKey[keyLength + 1] = (objNum >> 8) & 0xff;
41 objKey[keyLength + 2] = (objNum >> 16) & 0xff;
42 objKey[keyLength + 3] = objGen & 0xff;
43 objKey[keyLength + 4] = (objGen >> 8) & 0xff;
44 md5(objKey, keyLength + 5, objKey);
46 // set up for decryption
48 if ((objKeyLength = keyLength + 5) > 16) {
51 rc4InitKey(objKey, objKeyLength, state);
54 void Decrypt::reset() {
56 rc4InitKey(objKey, objKeyLength, state);
59 Guchar Decrypt::decryptByte(Guchar c) {
60 return rc4DecryptByte(state, &x, &y, c);
63 GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength,
64 GString *ownerKey, GString *userKey,
65 int permissions, GString *fileID,
66 GString *ownerPassword, GString *userPassword,
67 Guchar *fileKey, GBool *ownerPasswordOk) {
69 GString *userPassword2;
74 // try using the supplied owner password to generate the user password
76 len = ownerPassword->getLength();
78 memcpy(test, ownerPassword->getCString(), len);
79 memcpy(test + len, passwordPad, 32 - len);
81 memcpy(test, ownerPassword->getCString(), 32);
84 memcpy(test, passwordPad, 32);
87 if (encRevision == 3) {
88 for (i = 0; i < 50; ++i) {
92 rc4InitKey(test, keyLength, fState);
94 for (i = 0; i < 32; ++i) {
95 test[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i));
97 userPassword2 = new GString((char *)test, 32);
98 if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
99 permissions, fileID, userPassword2, fileKey)) {
100 *ownerPasswordOk = gTrue;
101 delete userPassword2;
104 *ownerPasswordOk = gFalse;
105 delete userPassword2;
107 // try using the supplied user password
108 return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
109 permissions, fileID, userPassword, fileKey);
112 GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength,
113 GString *ownerKey, GString *userKey,
114 int permissions, GString *fileID,
115 GString *userPassword, Guchar *fileKey) {
125 buf = (Guchar *)gmalloc(68 + fileID->getLength());
127 len = userPassword->getLength();
129 memcpy(buf, userPassword->getCString(), len);
130 memcpy(buf + len, passwordPad, 32 - len);
132 memcpy(buf, userPassword->getCString(), 32);
135 memcpy(buf, passwordPad, 32);
137 memcpy(buf + 32, ownerKey->getCString(), 32);
138 buf[64] = permissions & 0xff;
139 buf[65] = (permissions >> 8) & 0xff;
140 buf[66] = (permissions >> 16) & 0xff;
141 buf[67] = (permissions >> 24) & 0xff;
142 memcpy(buf + 68, fileID->getCString(), fileID->getLength());
143 md5(buf, 68 + fileID->getLength(), fileKey);
144 if (encRevision == 3) {
145 for (i = 0; i < 50; ++i) {
146 md5(fileKey, 16, fileKey);
150 // test user password
151 if (encRevision == 2) {
152 rc4InitKey(fileKey, keyLength, fState);
154 for (i = 0; i < 32; ++i) {
155 test[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i));
157 ok = memcmp(test, passwordPad, 32) == 0;
158 } else if (encRevision == 3) {
159 memcpy(test, userKey->getCString(), 32);
160 for (i = 19; i >= 0; --i) {
161 for (j = 0; j < keyLength; ++j) {
162 tmpKey[j] = fileKey[j] ^ i;
164 rc4InitKey(tmpKey, keyLength, fState);
166 for (j = 0; j < 32; ++j) {
167 test[j] = rc4DecryptByte(fState, &fx, &fy, test[j]);
170 memcpy(buf, passwordPad, 32);
171 memcpy(buf + 32, fileID->getCString(), fileID->getLength());
172 md5(buf, 32 + fileID->getLength(), buf);
173 ok = memcmp(test, buf, 16) == 0;
182 //------------------------------------------------------------------------
183 // RC4-compatible decryption
184 //------------------------------------------------------------------------
186 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) {
187 Guchar index1, index2;
191 for (i = 0; i < 256; ++i)
194 for (i = 0; i < 256; ++i) {
195 index2 = (key[index1] + state[i] + index2) % 256;
197 state[i] = state[index2];
199 index1 = (index1 + 1) % keyLen;
203 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) {
204 Guchar x1, y1, tx, ty;
206 x1 = *x = (*x + 1) % 256;
207 y1 = *y = (state[*x] + *y) % 256;
212 return c ^ state[(tx + ty) % 256];
215 //------------------------------------------------------------------------
216 // MD5 message digest
217 //------------------------------------------------------------------------
219 // this works around a bug in older Sun compilers
220 static inline Gulong rotateLeft(Gulong x, int r) {
222 return ((x << r) | (x >> (32 - r))) & 0xffffffff;
225 static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d,
226 Gulong Xk, Gulong s, Gulong Ti) {
227 return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s);
230 static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d,
231 Gulong Xk, Gulong s, Gulong Ti) {
232 return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s);
235 static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d,
236 Gulong Xk, Gulong s, Gulong Ti) {
237 return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s);
240 static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d,
241 Gulong Xk, Gulong s, Gulong Ti) {
242 return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s);
245 static void md5(Guchar *msg, int msgLen, Guchar *digest) {
247 Gulong a, b, c, d, aa, bb, cc, dd;
251 // compute number of 64-byte blocks
252 // (length + pad byte (0x80) + 8 bytes for length)
253 n64 = (msgLen + 1 + 8 + 63) / 64;
255 // initialize a, b, c, d
261 // loop through blocks
263 for (i = 0; i < n64; ++i) {
265 // grab a 64-byte block
266 for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4)
267 x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k];
270 x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k];
271 else if (k == msgLen - 2)
272 x[j] = 0x800000 + (msg[k+1] << 8) + msg[k];
273 else if (k == msgLen - 1)
274 x[j] = 0x8000 + msg[k];
290 a = md5Round1(a, b, c, d, x[0], 7, 0xd76aa478);
291 d = md5Round1(d, a, b, c, x[1], 12, 0xe8c7b756);
292 c = md5Round1(c, d, a, b, x[2], 17, 0x242070db);
293 b = md5Round1(b, c, d, a, x[3], 22, 0xc1bdceee);
294 a = md5Round1(a, b, c, d, x[4], 7, 0xf57c0faf);
295 d = md5Round1(d, a, b, c, x[5], 12, 0x4787c62a);
296 c = md5Round1(c, d, a, b, x[6], 17, 0xa8304613);
297 b = md5Round1(b, c, d, a, x[7], 22, 0xfd469501);
298 a = md5Round1(a, b, c, d, x[8], 7, 0x698098d8);
299 d = md5Round1(d, a, b, c, x[9], 12, 0x8b44f7af);
300 c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1);
301 b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be);
302 a = md5Round1(a, b, c, d, x[12], 7, 0x6b901122);
303 d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193);
304 c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e);
305 b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821);
308 a = md5Round2(a, b, c, d, x[1], 5, 0xf61e2562);
309 d = md5Round2(d, a, b, c, x[6], 9, 0xc040b340);
310 c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51);
311 b = md5Round2(b, c, d, a, x[0], 20, 0xe9b6c7aa);
312 a = md5Round2(a, b, c, d, x[5], 5, 0xd62f105d);
313 d = md5Round2(d, a, b, c, x[10], 9, 0x02441453);
314 c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681);
315 b = md5Round2(b, c, d, a, x[4], 20, 0xe7d3fbc8);
316 a = md5Round2(a, b, c, d, x[9], 5, 0x21e1cde6);
317 d = md5Round2(d, a, b, c, x[14], 9, 0xc33707d6);
318 c = md5Round2(c, d, a, b, x[3], 14, 0xf4d50d87);
319 b = md5Round2(b, c, d, a, x[8], 20, 0x455a14ed);
320 a = md5Round2(a, b, c, d, x[13], 5, 0xa9e3e905);
321 d = md5Round2(d, a, b, c, x[2], 9, 0xfcefa3f8);
322 c = md5Round2(c, d, a, b, x[7], 14, 0x676f02d9);
323 b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a);
326 a = md5Round3(a, b, c, d, x[5], 4, 0xfffa3942);
327 d = md5Round3(d, a, b, c, x[8], 11, 0x8771f681);
328 c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122);
329 b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c);
330 a = md5Round3(a, b, c, d, x[1], 4, 0xa4beea44);
331 d = md5Round3(d, a, b, c, x[4], 11, 0x4bdecfa9);
332 c = md5Round3(c, d, a, b, x[7], 16, 0xf6bb4b60);
333 b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70);
334 a = md5Round3(a, b, c, d, x[13], 4, 0x289b7ec6);
335 d = md5Round3(d, a, b, c, x[0], 11, 0xeaa127fa);
336 c = md5Round3(c, d, a, b, x[3], 16, 0xd4ef3085);
337 b = md5Round3(b, c, d, a, x[6], 23, 0x04881d05);
338 a = md5Round3(a, b, c, d, x[9], 4, 0xd9d4d039);
339 d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5);
340 c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8);
341 b = md5Round3(b, c, d, a, x[2], 23, 0xc4ac5665);
344 a = md5Round4(a, b, c, d, x[0], 6, 0xf4292244);
345 d = md5Round4(d, a, b, c, x[7], 10, 0x432aff97);
346 c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7);
347 b = md5Round4(b, c, d, a, x[5], 21, 0xfc93a039);
348 a = md5Round4(a, b, c, d, x[12], 6, 0x655b59c3);
349 d = md5Round4(d, a, b, c, x[3], 10, 0x8f0ccc92);
350 c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d);
351 b = md5Round4(b, c, d, a, x[1], 21, 0x85845dd1);
352 a = md5Round4(a, b, c, d, x[8], 6, 0x6fa87e4f);
353 d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0);
354 c = md5Round4(c, d, a, b, x[6], 15, 0xa3014314);
355 b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1);
356 a = md5Round4(a, b, c, d, x[4], 6, 0xf7537e82);
357 d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235);
358 c = md5Round4(c, d, a, b, x[2], 15, 0x2ad7d2bb);
359 b = md5Round4(b, c, d, a, x[9], 21, 0xeb86d391);
361 // increment a, b, c, d
368 // break digest into bytes
369 digest[0] = a & 0xff;
370 digest[1] = (a >>= 8) & 0xff;
371 digest[2] = (a >>= 8) & 0xff;
372 digest[3] = (a >>= 8) & 0xff;
373 digest[4] = b & 0xff;
374 digest[5] = (b >>= 8) & 0xff;
375 digest[6] = (b >>= 8) & 0xff;
376 digest[7] = (b >>= 8) & 0xff;
377 digest[8] = c & 0xff;
378 digest[9] = (c >>= 8) & 0xff;
379 digest[10] = (c >>= 8) & 0xff;
380 digest[11] = (c >>= 8) & 0xff;
381 digest[12] = d & 0xff;
382 digest[13] = (d >>= 8) & 0xff;
383 digest[14] = (d >>= 8) & 0xff;
384 digest[15] = (d >>= 8) & 0xff;