SHOGUN  v2.0.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
Hash.cpp
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2009 Soeren Sonnenburg
8  * Copyright (C) 2009 Fraunhofer Institute FIRST and Max-Planck-Society
9  *
10  * The MD5 hashing function was integrated from public sources.
11  * Its copyright follows.
12  *
13  * MD5
14  *
15  * This code implements the MD5 message-digest algorithm.
16  * The algorithm is due to Ron Rivest. This code was
17  * written by Colin Plumb in 1993, no copyright is claimed.
18  * This code is in the public domain; do with it what you wish.
19  *
20  * Equivalent code is available from RSA Data Security, Inc.
21  * This code has been tested against that, and is equivalent,
22  * except that you don't need to include two pages of legalese
23  * with every copy.
24  *
25  * To compute the message digest of a chunk of bytes, declare an
26  * MD5Context structure, pass it to MD5Init, call MD5Update as
27  * needed on buffers full of bytes, and then call MD5Final, which
28  * will fill a supplied 16-byte array with the digest.
29  */
30 
31 #include <shogun/lib/common.h>
32 #include <shogun/lib/Hash.h>
33 #include <ctype.h>
34 
35 using namespace shogun;
36 
37 uint32_t CHash::crc32(uint8_t *data, int32_t len)
38 {
39  uint32_t result;
40  int32_t i,j;
41  uint8_t octet;
42 
43  result = 0-1;
44  for (i=0; i<len; i++)
45  {
46  octet = *(data++);
47  for (j=0; j<8; j++)
48  {
49  if ((octet >> 7) ^ (result >> 31))
50  {
51  result = (result << 1) ^ 0x04c11db7;
52  }
53  else
54  {
55  result = (result << 1);
56  }
57  octet <<= 1;
58  }
59  }
60 
61  return ~result;
62 }
63 
64 void CHash::MD5(unsigned char *x, unsigned l, unsigned char *buf)
65 {
66  struct MD5Context ctx;
67 
68  MD5Init(&ctx);
69  MD5Update(&ctx, x, l);
70  MD5Final(buf, &ctx);
71 }
72 
73 #ifndef HIGHFIRST
74 #define byteReverse(buf, len) /* Nothing */
75 #else
76 void byteReverse(unsigned char *buf, unsigned uint32_t longs);
77 
78 #ifndef ASM_MD5
79 /*
80  * Note: this code is harmless on little-endian machines.
81  */
82 void byteReverse(unsigned char *buf, unsigned uint32_t longs)
83 {
84  uint32_t t;
85  do {
86  t = (uint32_t) ((unsigned) buf[3] << 8 | buf[2]) << 16 |
87  ((unsigned) buf[1] << 8 | buf[0]);
88  *(uint32_t *) buf = t;
89  buf += 4;
90  } while (--longs);
91 }
92 #endif
93 #endif
94 
95 void CHash::MD5Init(struct MD5Context *ctx)
96 {
97  ctx->buf[0] = 0x67452301;
98  ctx->buf[1] = 0xefcdab89;
99  ctx->buf[2] = 0x98badcfe;
100  ctx->buf[3] = 0x10325476;
101 
102  ctx->bits[0] = 0;
103  ctx->bits[1] = 0;
104 }
105 
106 void CHash::MD5Update(struct MD5Context *ctx, unsigned char const *buf,
107  unsigned len)
108 {
109  uint32_t t;
110 
111  /* Update bitcount */
112 
113  t = ctx->bits[0];
114  if ((ctx->bits[0] = t + ((uint32_t) len << 3)) < t)
115  ctx->bits[1]++; /* Carry from low to high */
116  ctx->bits[1] += len >> 29;
117 
118  t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */
119 
120  /* Handle any leading odd-sized chunks */
121 
122  if (t) {
123  unsigned char *p = (unsigned char *) ctx->in + t;
124 
125  t = 64 - t;
126  if (len < t) {
127  memcpy(p, buf, len);
128  return;
129  }
130  memcpy(p, buf, t);
131  byteReverse(ctx->in, 16);
132  MD5Transform(ctx->buf, (uint32_t *) ctx->in);
133  buf += t;
134  len -= t;
135  }
136  /* Process data in 64-byte chunks */
137 
138  while (len >= 64) {
139  memcpy(ctx->in, buf, 64);
140  byteReverse(ctx->in, 16);
141  MD5Transform(ctx->buf, (uint32_t *) ctx->in);
142  buf += 64;
143  len -= 64;
144  }
145 
146  /* Handle any remaining bytes of data. */
147 
148  memcpy(ctx->in, buf, len);
149 }
150 
151 void CHash::MD5Final(unsigned char digest[16], struct MD5Context *ctx)
152 {
153  unsigned count;
154  unsigned char *p;
155 
156  /* Compute number of bytes mod 64 */
157  count = (ctx->bits[0] >> 3) & 0x3F;
158 
159  /* Set the first char of padding to 0x80. This is safe since there is
160  always at least one byte free */
161  p = ctx->in + count;
162  *p++ = 0x80;
163 
164  /* Bytes of padding needed to make 64 bytes */
165  count = 64 - 1 - count;
166 
167  /* Pad out to 56 mod 64 */
168  if (count < 8) {
169  /* Two lots of padding: Pad the first block to 64 bytes */
170  memset(p, 0, count);
171  byteReverse(ctx->in, 16);
172  MD5Transform(ctx->buf, (uint32_t *) ctx->in);
173 
174  /* Now fill the next block with 56 bytes */
175  memset(ctx->in, 0, 56);
176  } else {
177  /* Pad block to 56 bytes */
178  memset(p, 0, count - 8);
179  }
180  byteReverse(ctx->in, 14);
181 
182  /* Append length in bits and transform */
183  ctx->uin[14] = ctx->bits[0];
184  ctx->uin[15] = ctx->bits[1];
185 
186  MD5Transform(ctx->buf, (uint32_t *) ctx->in);
187  byteReverse((unsigned char *) ctx->buf, 4);
188  memcpy(digest, ctx->buf, 16);
189  memset(ctx, 0, sizeof(*ctx)); /* In case it's sensitive */
190 }
191 
192 #ifndef ASM_MD5
193 
194 /* The four core functions - F1 is optimized somewhat */
195 
196 /* #define F1(x, y, z) (x & y | ~x & z) */
197 #define F1(x, y, z) (z ^ (x & (y ^ z)))
198 #define F2(x, y, z) F1(z, x, y)
199 #define F3(x, y, z) (x ^ y ^ z)
200 #define F4(x, y, z) (y ^ (x | ~z))
201 
202 /* This is the central step in the MD5 algorithm. */
203 #ifdef __PUREC__
204 #define MD5STEP(f, w, x, y, z, data, s) \
205  ( w += f /*(x, y, z)*/ + data, w = w<<s | w>>(32-s), w += x )
206 #else
207 #define MD5STEP(f, w, x, y, z, data, s) \
208  ( w += f(x, y, z) + data, w = w<<s | w>>(32-s), w += x )
209 #endif
210 
211 void CHash::MD5Transform(uint32_t buf[4], uint32_t const in[16])
212 {
213  register uint32_t a, b, c, d;
214 
215  a = buf[0];
216  b = buf[1];
217  c = buf[2];
218  d = buf[3];
219 
220 #ifdef __PUREC__ /* PureC Weirdness... (GG) */
221  MD5STEP(F1(b, c, d), a, b, c, d, in[0] + 0xd76aa478L, 7);
222  MD5STEP(F1(a, b, c), d, a, b, c, in[1] + 0xe8c7b756L, 12);
223  MD5STEP(F1(d, a, b), c, d, a, b, in[2] + 0x242070dbL, 17);
224  MD5STEP(F1(c, d, a), b, c, d, a, in[3] + 0xc1bdceeeL, 22);
225  MD5STEP(F1(b, c, d), a, b, c, d, in[4] + 0xf57c0fafL, 7);
226  MD5STEP(F1(a, b, c), d, a, b, c, in[5] + 0x4787c62aL, 12);
227  MD5STEP(F1(d, a, b), c, d, a, b, in[6] + 0xa8304613L, 17);
228  MD5STEP(F1(c, d, a), b, c, d, a, in[7] + 0xfd469501L, 22);
229  MD5STEP(F1(b, c, d), a, b, c, d, in[8] + 0x698098d8L, 7);
230  MD5STEP(F1(a, b, c), d, a, b, c, in[9] + 0x8b44f7afL, 12);
231  MD5STEP(F1(d, a, b), c, d, a, b, in[10] + 0xffff5bb1L, 17);
232  MD5STEP(F1(c, d, a), b, c, d, a, in[11] + 0x895cd7beL, 22);
233  MD5STEP(F1(b, c, d), a, b, c, d, in[12] + 0x6b901122L, 7);
234  MD5STEP(F1(a, b, c), d, a, b, c, in[13] + 0xfd987193L, 12);
235  MD5STEP(F1(d, a, b), c, d, a, b, in[14] + 0xa679438eL, 17);
236  MD5STEP(F1(c, d, a), b, c, d, a, in[15] + 0x49b40821L, 22);
237 
238  MD5STEP(F2(b, c, d), a, b, c, d, in[1] + 0xf61e2562L, 5);
239  MD5STEP(F2(a, b, c), d, a, b, c, in[6] + 0xc040b340L, 9);
240  MD5STEP(F2(d, a, b), c, d, a, b, in[11] + 0x265e5a51L, 14);
241  MD5STEP(F2(c, d, a), b, c, d, a, in[0] + 0xe9b6c7aaL, 20);
242  MD5STEP(F2(b, c, d), a, b, c, d, in[5] + 0xd62f105dL, 5);
243  MD5STEP(F2(a, b, c), d, a, b, c, in[10] + 0x02441453L, 9);
244  MD5STEP(F2(d, a, b), c, d, a, b, in[15] + 0xd8a1e681L, 14);
245  MD5STEP(F2(c, d, a), b, c, d, a, in[4] + 0xe7d3fbc8L, 20);
246  MD5STEP(F2(b, c, d), a, b, c, d, in[9] + 0x21e1cde6L, 5);
247  MD5STEP(F2(a, b, c), d, a, b, c, in[14] + 0xc33707d6L, 9);
248  MD5STEP(F2(d, a, b), c, d, a, b, in[3] + 0xf4d50d87L, 14);
249  MD5STEP(F2(c, d, a), b, c, d, a, in[8] + 0x455a14edL, 20);
250  MD5STEP(F2(b, c, d), a, b, c, d, in[13] + 0xa9e3e905L, 5);
251  MD5STEP(F2(a, b, c), d, a, b, c, in[2] + 0xfcefa3f8L, 9);
252  MD5STEP(F2(d, a, b), c, d, a, b, in[7] + 0x676f02d9L, 14);
253  MD5STEP(F2(c, d, a), b, c, d, a, in[12] + 0x8d2a4c8aL, 20);
254 
255  MD5STEP(F3(b, c, d), a, b, c, d, in[5] + 0xfffa3942L, 4);
256  MD5STEP(F3(a, b, c), d, a, b, c, in[8] + 0x8771f681L, 11);
257  MD5STEP(F3(d, a, b), c, d, a, b, in[11] + 0x6d9d6122L, 16);
258  MD5STEP(F3(c, d, a), b, c, d, a, in[14] + 0xfde5380cL, 23);
259  MD5STEP(F3(b, c, d), a, b, c, d, in[1] + 0xa4beea44L, 4);
260  MD5STEP(F3(a, b, c), d, a, b, c, in[4] + 0x4bdecfa9L, 11);
261  MD5STEP(F3(d, a, b), c, d, a, b, in[7] + 0xf6bb4b60L, 16);
262  MD5STEP(F3(c, d, a), b, c, d, a, in[10] + 0xbebfbc70L, 23);
263  MD5STEP(F3(b, c, d), a, b, c, d, in[13] + 0x289b7ec6L, 4);
264  MD5STEP(F3(a, b, c), d, a, b, c, in[0] + 0xeaa127faL, 11);
265  MD5STEP(F3(d, a, b), c, d, a, b, in[3] + 0xd4ef3085L, 16);
266  MD5STEP(F3(c, d, a), b, c, d, a, in[6] + 0x04881d05L, 23);
267  MD5STEP(F3(b, c, d), a, b, c, d, in[9] + 0xd9d4d039L, 4);
268  MD5STEP(F3(a, b, c), d, a, b, c, in[12] + 0xe6db99e5L, 11);
269  MD5STEP(F3(d, a, b), c, d, a, b, in[15] + 0x1fa27cf8L, 16);
270  MD5STEP(F3(c, d, a), b, c, d, a, in[2] + 0xc4ac5665L, 23);
271 
272  MD5STEP(F4(b, c, d), a, b, c, d, in[0] + 0xf4292244L, 6);
273  MD5STEP(F4(a, b, c), d, a, b, c, in[7] + 0x432aff97L, 10);
274  MD5STEP(F4(d, a, b), c, d, a, b, in[14] + 0xab9423a7L, 15);
275  MD5STEP(F4(c, d, a), b, c, d, a, in[5] + 0xfc93a039L, 21);
276  MD5STEP(F4(b, c, d), a, b, c, d, in[12] + 0x655b59c3L, 6);
277  MD5STEP(F4(a, b, c), d, a, b, c, in[3] + 0x8f0ccc92L, 10);
278  MD5STEP(F4(d, a, b), c, d, a, b, in[10] + 0xffeff47dL, 15);
279  MD5STEP(F4(c, d, a), b, c, d, a, in[1] + 0x85845dd1L, 21);
280  MD5STEP(F4(b, c, d), a, b, c, d, in[8] + 0x6fa87e4fL, 6);
281  MD5STEP(F4(a, b, c), d, a, b, c, in[15] + 0xfe2ce6e0L, 10);
282  MD5STEP(F4(d, a, b), c, d, a, b, in[6] + 0xa3014314L, 15);
283  MD5STEP(F4(c, d, a), b, c, d, a, in[13] + 0x4e0811a1L, 21);
284  MD5STEP(F4(b, c, d), a, b, c, d, in[4] + 0xf7537e82L, 6);
285  MD5STEP(F4(a, b, c), d, a, b, c, in[11] + 0xbd3af235L, 10);
286  MD5STEP(F4(d, a, b), c, d, a, b, in[2] + 0x2ad7d2bbL, 15);
287  MD5STEP(F4(c, d, a), b, c, d, a, in[9] + 0xeb86d391L, 21);
288 #else
289  MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);
290  MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);
291  MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);
292  MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22);
293  MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7);
294  MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12);
295  MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17);
296  MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22);
297  MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7);
298  MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12);
299  MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
300  MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
301  MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);
302  MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
303  MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
304  MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);
305 
306  MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);
307  MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);
308  MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
309  MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20);
310  MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5);
311  MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);
312  MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
313  MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20);
314  MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5);
315  MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);
316  MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14);
317  MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20);
318  MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);
319  MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);
320  MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);
321  MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);
322 
323  MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);
324  MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);
325  MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
326  MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
327  MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4);
328  MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11);
329  MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16);
330  MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
331  MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);
332  MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11);
333  MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16);
334  MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23);
335  MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4);
336  MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
337  MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
338  MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);
339 
340  MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);
341  MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);
342  MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
343  MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21);
344  MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);
345  MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10);
346  MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
347  MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21);
348  MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6);
349  MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
350  MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15);
351  MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
352  MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6);
353  MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
354  MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);
355  MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);
356 #endif
357 
358  buf[0] += a;
359  buf[1] += b;
360  buf[2] += c;
361  buf[3] += d;
362 }
363 #endif
364 
365 uint32_t CHash::MurmurHash3(uint8_t* data, int32_t len, uint32_t seed)
366 {
367  return PMurHash32(seed, data, len);
368 }
369 
370 void CHash::IncrementalMurmurHash3(uint32_t *ph1, uint32_t *pcarry, uint8_t* data, int32_t len)
371 {
372  PMurHash32_Process(ph1, pcarry, data, len);
373 }
374 
375 uint32_t CHash::FinalizeIncrementalMurmurHash3(uint32_t h, uint32_t carry, uint32_t total_length)
376 {
377  return PMurHash32_Result(h, carry, total_length);
378 }
379 
380 uint32_t CHash::MurmurHashString(substring s, uint32_t h)
381 {
382  uint32_t ret = 0;
383 
384  // Trim leading whitespace
385  for(; *(s.start) <= 0x20 && s.start < s.end; s.start++);
386 
387  // Trim trailing white space
388  for(; *(s.end-1) <= 0x20 && s.end > s.start; s.end--);
389 
390  char *p = s.start;
391  while (p != s.end)
392  if (isdigit(*p))
393  ret = 10*ret + *(p++) - '0';
394  else
395  return MurmurHash3((uint8_t *)s.start, s.end - s.start, h);
396 
397  return ret + h;
398 }

SHOGUN Machine Learning Toolbox - Documentation