1 //========================================================================
5 // Copyright 1996 Derek B. Noonburg
7 //========================================================================
10 #pragma implementation
16 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state);
17 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c);
18 static void md5(Guchar *msg, int msgLen, Guchar *digest);
20 static Guchar passwordPad[32] = {
21 0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
22 0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08,
23 0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80,
24 0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
27 //------------------------------------------------------------------------
29 //------------------------------------------------------------------------
31 Decrypt::Decrypt(Guchar *fileKey, int objNum, int objGen) {
32 // construct object key
33 objKey[0] = fileKey[0];
34 objKey[1] = fileKey[1];
35 objKey[2] = fileKey[2];
36 objKey[3] = fileKey[3];
37 objKey[4] = fileKey[4];
38 objKey[5] = objNum & 0xff;
39 objKey[6] = (objNum >> 8) & 0xff;
40 objKey[7] = (objNum >> 16) & 0xff;
41 objKey[8] = objGen & 0xff;
42 objKey[9] = (objGen >> 8) & 0xff;
43 md5(objKey, 10, objKey);
45 // set up for decryption
47 rc4InitKey(objKey, 10, state);
50 void Decrypt::reset() {
52 rc4InitKey(objKey, 10, state);
55 Guchar Decrypt::decryptByte(Guchar c) {
56 return rc4DecryptByte(state, &x, &y, c);
59 GBool Decrypt::makeFileKey(GString *ownerKey, GString *userKey,
60 int permissions, GString *fileID,
61 GString *userPassword, Guchar *fileKey) {
70 buf = (Guchar *)gmalloc(68 + fileID->getLength());
72 len = userPassword->getLength();
74 memcpy(buf, userPassword->getCString(), len);
75 memcpy(buf + len, passwordPad, 32 - len);
77 memcpy(buf, userPassword->getCString(), 32);
80 memcpy(buf, passwordPad, 32);
82 memcpy(buf + 32, ownerKey->getCString(), 32);
83 buf[64] = permissions & 0xff;
84 buf[65] = (permissions >> 8) & 0xff;
85 buf[66] = (permissions >> 16) & 0xff;
86 buf[67] = (permissions >> 24) & 0xff;
87 memcpy(buf + 68, fileID->getCString(), fileID->getLength());
88 md5(buf, 68 + fileID->getLength(), fileKey);
92 rc4InitKey(fileKey, 5, fState);
93 for (i = 0; i < 32; ++i) {
94 userTest[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i));
96 ok = memcmp(userTest, passwordPad, 32) == 0;
102 //------------------------------------------------------------------------
103 // RC4-compatible decryption
104 //------------------------------------------------------------------------
106 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) {
107 Guchar index1, index2;
111 for (i = 0; i < 256; ++i)
114 for (i = 0; i < 256; ++i) {
115 index2 = (key[index1] + state[i] + index2) % 256;
117 state[i] = state[index2];
119 index1 = (index1 + 1) % keyLen;
123 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) {
124 Guchar x1, y1, tx, ty;
126 x1 = *x = (*x + 1) % 256;
127 y1 = *y = (state[*x] + *y) % 256;
132 return c ^ state[(tx + ty) % 256];
135 //------------------------------------------------------------------------
136 // MD5 message digest
137 //------------------------------------------------------------------------
139 static inline Gulong rotateLeft(Gulong x, int r) {
141 return ((x << r) | (x >> (32 - r))) & 0xffffffff;
144 static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d,
145 Gulong Xk, Gulong s, Gulong Ti) {
146 return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s);
149 static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d,
150 Gulong Xk, Gulong s, Gulong Ti) {
151 return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s);
154 static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d,
155 Gulong Xk, Gulong s, Gulong Ti) {
156 return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s);
159 static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d,
160 Gulong Xk, Gulong s, Gulong Ti) {
161 return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s);
164 static void md5(Guchar *msg, int msgLen, Guchar *digest) {
166 Gulong a, b, c, d, aa, bb, cc, dd;
170 // compute number of 64-byte blocks
171 // (length + pad byte (0x80) + 8 bytes for length)
172 n64 = (msgLen + 1 + 8 + 63) / 64;
174 // initialize a, b, c, d
180 // loop through blocks
182 for (i = 0; i < n64; ++i) {
184 // grab a 64-byte block
185 for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4)
186 x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k];
189 x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k];
190 else if (k == msgLen - 2)
191 x[j] = 0x800000 + (msg[k+1] << 8) + msg[k];
192 else if (k == msgLen - 1)
193 x[j] = 0x8000 + msg[k];
209 a = md5Round1(a, b, c, d, x[0], 7, 0xd76aa478);
210 d = md5Round1(d, a, b, c, x[1], 12, 0xe8c7b756);
211 c = md5Round1(c, d, a, b, x[2], 17, 0x242070db);
212 b = md5Round1(b, c, d, a, x[3], 22, 0xc1bdceee);
213 a = md5Round1(a, b, c, d, x[4], 7, 0xf57c0faf);
214 d = md5Round1(d, a, b, c, x[5], 12, 0x4787c62a);
215 c = md5Round1(c, d, a, b, x[6], 17, 0xa8304613);
216 b = md5Round1(b, c, d, a, x[7], 22, 0xfd469501);
217 a = md5Round1(a, b, c, d, x[8], 7, 0x698098d8);
218 d = md5Round1(d, a, b, c, x[9], 12, 0x8b44f7af);
219 c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1);
220 b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be);
221 a = md5Round1(a, b, c, d, x[12], 7, 0x6b901122);
222 d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193);
223 c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e);
224 b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821);
227 a = md5Round2(a, b, c, d, x[1], 5, 0xf61e2562);
228 d = md5Round2(d, a, b, c, x[6], 9, 0xc040b340);
229 c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51);
230 b = md5Round2(b, c, d, a, x[0], 20, 0xe9b6c7aa);
231 a = md5Round2(a, b, c, d, x[5], 5, 0xd62f105d);
232 d = md5Round2(d, a, b, c, x[10], 9, 0x02441453);
233 c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681);
234 b = md5Round2(b, c, d, a, x[4], 20, 0xe7d3fbc8);
235 a = md5Round2(a, b, c, d, x[9], 5, 0x21e1cde6);
236 d = md5Round2(d, a, b, c, x[14], 9, 0xc33707d6);
237 c = md5Round2(c, d, a, b, x[3], 14, 0xf4d50d87);
238 b = md5Round2(b, c, d, a, x[8], 20, 0x455a14ed);
239 a = md5Round2(a, b, c, d, x[13], 5, 0xa9e3e905);
240 d = md5Round2(d, a, b, c, x[2], 9, 0xfcefa3f8);
241 c = md5Round2(c, d, a, b, x[7], 14, 0x676f02d9);
242 b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a);
245 a = md5Round3(a, b, c, d, x[5], 4, 0xfffa3942);
246 d = md5Round3(d, a, b, c, x[8], 11, 0x8771f681);
247 c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122);
248 b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c);
249 a = md5Round3(a, b, c, d, x[1], 4, 0xa4beea44);
250 d = md5Round3(d, a, b, c, x[4], 11, 0x4bdecfa9);
251 c = md5Round3(c, d, a, b, x[7], 16, 0xf6bb4b60);
252 b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70);
253 a = md5Round3(a, b, c, d, x[13], 4, 0x289b7ec6);
254 d = md5Round3(d, a, b, c, x[0], 11, 0xeaa127fa);
255 c = md5Round3(c, d, a, b, x[3], 16, 0xd4ef3085);
256 b = md5Round3(b, c, d, a, x[6], 23, 0x04881d05);
257 a = md5Round3(a, b, c, d, x[9], 4, 0xd9d4d039);
258 d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5);
259 c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8);
260 b = md5Round3(b, c, d, a, x[2], 23, 0xc4ac5665);
263 a = md5Round4(a, b, c, d, x[0], 6, 0xf4292244);
264 d = md5Round4(d, a, b, c, x[7], 10, 0x432aff97);
265 c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7);
266 b = md5Round4(b, c, d, a, x[5], 21, 0xfc93a039);
267 a = md5Round4(a, b, c, d, x[12], 6, 0x655b59c3);
268 d = md5Round4(d, a, b, c, x[3], 10, 0x8f0ccc92);
269 c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d);
270 b = md5Round4(b, c, d, a, x[1], 21, 0x85845dd1);
271 a = md5Round4(a, b, c, d, x[8], 6, 0x6fa87e4f);
272 d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0);
273 c = md5Round4(c, d, a, b, x[6], 15, 0xa3014314);
274 b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1);
275 a = md5Round4(a, b, c, d, x[4], 6, 0xf7537e82);
276 d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235);
277 c = md5Round4(c, d, a, b, x[2], 15, 0x2ad7d2bb);
278 b = md5Round4(b, c, d, a, x[9], 21, 0xeb86d391);
280 // increment a, b, c, d
287 // break digest into bytes
288 digest[0] = a & 0xff;
289 digest[1] = (a >>= 8) & 0xff;
290 digest[2] = (a >>= 8) & 0xff;
291 digest[3] = (a >>= 8) & 0xff;
292 digest[4] = b & 0xff;
293 digest[5] = (b >>= 8) & 0xff;
294 digest[6] = (b >>= 8) & 0xff;
295 digest[7] = (b >>= 8) & 0xff;
296 digest[8] = c & 0xff;
297 digest[9] = (c >>= 8) & 0xff;
298 digest[10] = (c >>= 8) & 0xff;
299 digest[11] = (c >>= 8) & 0xff;
300 digest[12] = d & 0xff;
301 digest[13] = (d >>= 8) & 0xff;
302 digest[14] = (d >>= 8) & 0xff;
303 digest[15] = (d >>= 8) & 0xff;