added: dupe check in xbt files

git-svn-id: https://xbmc.svn.sourceforge.net/svnroot/xbmc/trunk@33752 568bbfeb-2a22-0410-94d2-cc84cf5bfa90

6 files changed, 373 insertions, 20 deletions

diff --git a/tools/TexturePacker/Makefile.in b/tools/TexturePacker/Makefile.in
index 52a0de5eb2..be9600733b 100644
--- a/tools/TexturePacker/Makefile.in
+++ b/tools/TexturePacker/Makefile.in
@@ -11,6 +11,7 @@ OBJS = \
 	SDL_anigif.o \
 	XBTFWriter.o \
 	XBMCTex.o \
+	md5.o \
 	../../guilib/XBTF.o
 
 TARGET = TexturePacker
diff --git a/tools/TexturePacker/XBMCTex.cpp b/tools/TexturePacker/XBMCTex.cpp
index 63b85bb011..4075f64001 100644
--- a/tools/TexturePacker/XBMCTex.cpp
+++ b/tools/TexturePacker/XBMCTex.cpp
@@ -22,6 +22,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <dirent.h>
+#include <map>
 #include <squish.h>
 #include <string>
 #define __STDC_FORMAT_MACROS
@@ -52,6 +53,13 @@
 
 #undef main
 
+extern "C" 
+{
+#include "md5.h"
+}
+
+using namespace std;
+
 const char *GetFormatString(unsigned int format)
 {
   switch (format)
@@ -297,7 +305,7 @@ CXBTFFrame createXBTFFrame(SDL_Surface* image, CXBTFWriter& writer, double maxMS
   delete[] compressed;
   SDL_FreeSurface(argbImage);
   return frame;
-#else	// For ARM, dont use DXT compression!!!
+#else // For ARM, dont use DXT compression!!!
   CXBTFFrame frame = appendContent(writer, image->w, image->h, argb, image->w * image->h * 4, XB_FMT_A8R8G8B8, flags);
   SDL_FreeSurface(argbImage);
   return frame;
@@ -312,10 +320,38 @@ void Usage()
   puts("  -output <dir>    Output directory/filename. Default: Textures.xpr");
 }
 
-int createBundle(const std::string& InputDir, const std::string& OutputFile, double maxMSE, unsigned int flags)
+static bool checkDupe(struct MD5Context* ctx,
+                      map<string,unsigned int>& hashes,
+                      vector<unsigned int>& dupes, unsigned int pos)
 {
+  char digest[17];
+  MD5Final((unsigned char*)digest,ctx);
+  digest[16] = 0;
+  map<string,unsigned int>::iterator it = hashes.find(digest);
+  if (it != hashes.end())
+  {
+    dupes[pos] = it->second; 
+    return true;
+  }
+
+  hashes.insert(make_pair(digest,pos));
+  dupes[pos] = pos;
+
+  return false;
+}
+
+int createBundle(const std::string& InputDir, const std::string& OutputFile, double maxMSE, unsigned int flags, bool dupecheck)
+{
+  map<string,unsigned int> hashes;
+  vector<unsigned int> dupes;
   CXBTF xbtf;
   CreateSkeletonHeader(xbtf, InputDir);
+  dupes.resize(xbtf.GetFiles().size());
+  if (!dupecheck)
+  {
+    for (unsigned int i=0;i<dupes.size();++i)
+      dupes[i] = i;
+  }
 
   CXBTFWriter writer(xbtf, OutputFile);
   if (!writer.Create())
@@ -327,6 +363,8 @@ int createBundle(const std::string& InputDir, const std::string& OutputFile, dou
   std::vector<CXBTFFile>& files = xbtf.GetFiles();
   for (size_t i = 0; i < files.size(); i++)
   {
+    struct MD5Context ctx;
+    MD5Init(&ctx);
     CXBTFFile& file = files[i];
 
     std::string fullPath = InputDir;
@@ -346,15 +384,30 @@ int createBundle(const std::string& InputDir, const std::string& OutputFile, dou
         continue;
       }
 
+      bool skip=false;
       printf("%s", output.c_str());
+      if (dupecheck)
+      {
+        MD5Update(&ctx,(const uint8_t*)image->pixels,image->h*image->pitch);
+        if (checkDupe(&ctx,hashes,dupes,i))
+        {
+          printf("duplicate, skipping compression\n");
+          file.GetFrames().insert(file.GetFrames().end(),
+                                  files[dupes[i]].GetFrames().begin(),
+                                  files[dupes[i]].GetFrames().end());
+          skip = true;
+        }
+      }
 
-      CXBTFFrame frame = createXBTFFrame(image, writer, maxMSE, flags);
-
-      printf("%s (%d,%d @ %"PRIu64" bytes)\n", GetFormatString(frame.GetFormat()), frame.GetWidth(), frame.GetHeight(), frame.GetUnpackedSize());
+      if (!skip)
+      {
+        CXBTFFrame frame = createXBTFFrame(image, writer, maxMSE, flags);
 
-      file.SetLoop(0);
-      file.GetFrames().push_back(frame);
+        printf("%s (%d,%d @ %"PRIu64" bytes)\n", GetFormatString(frame.GetFormat()), frame.GetWidth(), frame.GetHeight(), frame.GetUnpackedSize());
 
+        file.SetLoop(0);
+        file.GetFrames().push_back(frame);
+      }
       SDL_FreeSurface(image);
     }
     else
@@ -364,16 +417,35 @@ int createBundle(const std::string& InputDir, const std::string& OutputFile, dou
       AG_LoadGIF(fullPath.c_str(), gpAG, gnAG);
 
       printf("%s\n", output.c_str());
-
-      for (int j = 0; j < gnAG; j++)
+      bool skip=false;
+      if (dupecheck)
       {
-        printf("    frame %4i                                ", j);
-        CXBTFFrame frame = createXBTFFrame(gpAG[j].surface, writer, maxMSE, flags);
-        frame.SetDuration(gpAG[j].delay);
-        file.GetFrames().push_back(frame);
-        printf("%s (%d,%d @ %"PRIu64" bytes)\n", GetFormatString(frame.GetFormat()), frame.GetWidth(), frame.GetHeight(), frame.GetUnpackedSize());
+        for (int j = 0; j < gnAG; j++)
+          MD5Update(&ctx,(const uint8_t*)gpAG[j].surface->pixels,
+                                         gpAG[j].surface->h*
+                                         gpAG[j].surface->pitch);
+
+        if (checkDupe(&ctx,hashes,dupes,i))
+        {
+          printf("duplicate, skipping compression\n");
+          file.GetFrames().insert(file.GetFrames().end(),
+                                  files[dupes[i]].GetFrames().begin(),
+                                  files[dupes[i]].GetFrames().end());
+          skip = true;
+        }
       }
 
+      if (!skip)
+      {
+        for (int j = 0; j < gnAG; j++)
+        {
+          printf("    frame %4i                                ", j);
+          CXBTFFrame frame = createXBTFFrame(gpAG[j].surface, writer, maxMSE, flags);
+          frame.SetDuration(gpAG[j].delay);
+          file.GetFrames().push_back(frame);
+          printf("%s (%d,%d @ %"PRIu64" bytes)\n", GetFormatString(frame.GetFormat()), frame.GetWidth(), frame.GetHeight(), frame.GetUnpackedSize());
+        }
+      }
       AG_FreeSurfaces(gpAG, gnAG);
       delete [] gpAG;
 
@@ -381,7 +453,7 @@ int createBundle(const std::string& InputDir, const std::string& OutputFile, dou
     }
   }
 
-  if (!writer.UpdateHeader())
+  if (!writer.UpdateHeader(dupes))
   {
     printf("Error writing header to file\n");
     return 1;
@@ -402,6 +474,7 @@ int main(int argc, char* argv[])
     return 1;
 
   bool valid = false;
+  bool dupecheck = false;
   CmdLineArgs args(argc, (const char**)argv);
 
   if (args.size() == 1)
@@ -425,6 +498,8 @@ int main(int argc, char* argv[])
       InputDir = args[++i];
       valid = true;
     }
+    else if (!strcmp(args[i], "-dupecheck"))
+      dupecheck = true;
     else if (!stricmp(args[i], "-output") || !stricmp(args[i], "-o"))
     {
       OutputFilename = args[++i];
@@ -452,5 +527,5 @@ int main(int argc, char* argv[])
 
   double maxMSE = 1.5;    // HQ only please
   unsigned int flags = FLAGS_USE_LZO; // TODO: currently no YCoCg (commandline option?)
-  createBundle(InputDir, OutputFilename, maxMSE, flags);
+  createBundle(InputDir, OutputFilename, maxMSE, flags, dupecheck);
 }
diff --git a/tools/TexturePacker/XBTFWriter.cpp b/tools/TexturePacker/XBTFWriter.cpp
index d56e3a077d..d5beb143d7 100644
--- a/tools/TexturePacker/XBTFWriter.cpp
+++ b/tools/TexturePacker/XBTFWriter.cpp
@@ -95,7 +95,7 @@ bool CXBTFWriter::AppendContent(unsigned char const* data, size_t length)
   return true;
 }
 
-bool CXBTFWriter::UpdateHeader()
+bool CXBTFWriter::UpdateHeader(const std::vector<unsigned int>& dupes)
 {
   if (m_file == NULL)
   {
@@ -129,8 +129,13 @@ bool CXBTFWriter::UpdateHeader()
     for (size_t j = 0; j < frames.size(); j++)
     {
       CXBTFFrame& frame = frames[j];
-      frame.SetOffset(offset);
-      offset += frame.GetPackedSize();
+      if (dupes[i] != i)
+        frame.SetOffset(files[dupes[i]].GetFrames()[j].GetOffset());
+      else
+      {
+        frame.SetOffset(offset);
+        offset += frame.GetPackedSize();
+      }
 
       WRITE_U32(frame.GetWidth(), m_file);
       WRITE_U32(frame.GetHeight(), m_file);
diff --git a/tools/TexturePacker/XBTFWriter.h b/tools/TexturePacker/XBTFWriter.h
index 021ae6286e..d86094e449 100644
--- a/tools/TexturePacker/XBTFWriter.h
+++ b/tools/TexturePacker/XBTFWriter.h
@@ -22,6 +22,7 @@
  *
  */
 
+#include <vector>
 #include <string>
 #include <stdio.h>
 #include "XBTF.h"
@@ -33,7 +34,7 @@ public:
   bool Create();
   bool Close();
   bool AppendContent(unsigned char const* data, size_t length);
-  bool UpdateHeader();
+  bool UpdateHeader(const std::vector<unsigned int>& dupes);
 
 private:
   CXBTF& m_xbtf;
diff --git a/tools/TexturePacker/md5.c b/tools/TexturePacker/md5.c
new file mode 100644
index 0000000000..ce0c310963
--- /dev/null
+++ b/tools/TexturePacker/md5.c
@@ -0,0 +1,231 @@
+/*
+ * This code implements the MD5 message-digest algorithm.
+ * The algorithm is due to Ron Rivest.  This code was
+ * written by Colin Plumb in 1993, no copyright is claimed.
+ * This code is in the public domain; do with it what you wish.
+ *
+ * Equivalent code is available from RSA Data Security, Inc.
+ * This code has been tested against that, and is equivalent,
+ * except that you don't need to include two pages of legalese
+ * with every copy.
+ *
+ * To compute the message digest of a chunk of bytes, declare an
+ * MD5Context structure, pass it to MD5Init, call MD5Update as
+ * needed on buffers full of bytes, and then call MD5Final, which
+ * will fill a supplied 16-byte array with the digest.
+ *
+ * Changed so as no longer to depend on Colin Plumb's `usual.h' header
+ * definitions; now uses stuff from dpkg's config.h.
+ *  - Ian Jackson <ian@chiark.greenend.org.uk>.
+ * Still in the public domain.
+ */
+
+#include "md5.h"
+
+
+#ifdef WORDS_BIGENDIAN
+void
+byteSwap(uint32_t *buf, unsigned words)
+{
+	uint8_t *p = (uint8_t*)buf;
+
+	do {
+		*buf++ = (uint32_t)((unsigned)p[3] << 8 | p[2]) << 16 |
+			((unsigned)p[1] << 8 | p[0]);
+		p += 4;
+	} while (--words);
+}
+#else
+#define byteSwap(buf,words)
+#endif
+
+/* The four core functions - F1 is optimized somewhat */
+
+/* #define F1(x, y, z) (x & y | ~x & z) */
+#define F1(x, y, z) (z ^ (x & (y ^ z)))
+#define F2(x, y, z) F1(z, x, y)
+#define F3(x, y, z) (x ^ y ^ z)
+#define F4(x, y, z) (y ^ (x | ~z))
+
+/* This is the central step in the MD5 algorithm. */
+#define MD5STEP(f,w,x,y,z,in,s) \
+	 (w += f(x,y,z) + in, w = (w<<s | w>>(32-s)) + x)
+
+/*
+ * The core of the MD5 algorithm, this alters an existing MD5 hash to
+ * reflect the addition of 16 longwords of new data.  MD5Update blocks
+ * the data and converts bytes into longwords for this routine.
+ */
+static void
+MD5Transform(uint32_t buf[4], uint32_t const in[16])
+{
+	register uint32_t a, b, c, d;
+
+	a = buf[0];
+	b = buf[1];
+	c = buf[2];
+	d = buf[3];
+
+	MD5STEP(F1, a, b, c, d, in[0] + 0xd76aa478, 7);
+	MD5STEP(F1, d, a, b, c, in[1] + 0xe8c7b756, 12);
+	MD5STEP(F1, c, d, a, b, in[2] + 0x242070db, 17);
+	MD5STEP(F1, b, c, d, a, in[3] + 0xc1bdceee, 22);
+	MD5STEP(F1, a, b, c, d, in[4] + 0xf57c0faf, 7);
+	MD5STEP(F1, d, a, b, c, in[5] + 0x4787c62a, 12);
+	MD5STEP(F1, c, d, a, b, in[6] + 0xa8304613, 17);
+	MD5STEP(F1, b, c, d, a, in[7] + 0xfd469501, 22);
+	MD5STEP(F1, a, b, c, d, in[8] + 0x698098d8, 7);
+	MD5STEP(F1, d, a, b, c, in[9] + 0x8b44f7af, 12);
+	MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
+	MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
+	MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122, 7);
+	MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
+	MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
+	MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);
+
+	MD5STEP(F2, a, b, c, d, in[1] + 0xf61e2562, 5);
+	MD5STEP(F2, d, a, b, c, in[6] + 0xc040b340, 9);
+	MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
+	MD5STEP(F2, b, c, d, a, in[0] + 0xe9b6c7aa, 20);
+	MD5STEP(F2, a, b, c, d, in[5] + 0xd62f105d, 5);
+	MD5STEP(F2, d, a, b, c, in[10] + 0x02441453, 9);
+	MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
+	MD5STEP(F2, b, c, d, a, in[4] + 0xe7d3fbc8, 20);
+	MD5STEP(F2, a, b, c, d, in[9] + 0x21e1cde6, 5);
+	MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6, 9);
+	MD5STEP(F2, c, d, a, b, in[3] + 0xf4d50d87, 14);
+	MD5STEP(F2, b, c, d, a, in[8] + 0x455a14ed, 20);
+	MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905, 5);
+	MD5STEP(F2, d, a, b, c, in[2] + 0xfcefa3f8, 9);
+	MD5STEP(F2, c, d, a, b, in[7] + 0x676f02d9, 14);
+	MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);
+
+	MD5STEP(F3, a, b, c, d, in[5] + 0xfffa3942, 4);
+	MD5STEP(F3, d, a, b, c, in[8] + 0x8771f681, 11);
+	MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
+	MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
+	MD5STEP(F3, a, b, c, d, in[1] + 0xa4beea44, 4);
+	MD5STEP(F3, d, a, b, c, in[4] + 0x4bdecfa9, 11);
+	MD5STEP(F3, c, d, a, b, in[7] + 0xf6bb4b60, 16);
+	MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
+	MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6, 4);
+	MD5STEP(F3, d, a, b, c, in[0] + 0xeaa127fa, 11);
+	MD5STEP(F3, c, d, a, b, in[3] + 0xd4ef3085, 16);
+	MD5STEP(F3, b, c, d, a, in[6] + 0x04881d05, 23);
+	MD5STEP(F3, a, b, c, d, in[9] + 0xd9d4d039, 4);
+	MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
+	MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
+	MD5STEP(F3, b, c, d, a, in[2] + 0xc4ac5665, 23);
+
+	MD5STEP(F4, a, b, c, d, in[0] + 0xf4292244, 6);
+	MD5STEP(F4, d, a, b, c, in[7] + 0x432aff97, 10);
+	MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
+	MD5STEP(F4, b, c, d, a, in[5] + 0xfc93a039, 21);
+	MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3, 6);
+	MD5STEP(F4, d, a, b, c, in[3] + 0x8f0ccc92, 10);
+	MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
+	MD5STEP(F4, b, c, d, a, in[1] + 0x85845dd1, 21);
+	MD5STEP(F4, a, b, c, d, in[8] + 0x6fa87e4f, 6);
+	MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
+	MD5STEP(F4, c, d, a, b, in[6] + 0xa3014314, 15);
+	MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
+	MD5STEP(F4, a, b, c, d, in[4] + 0xf7537e82, 6);
+	MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
+	MD5STEP(F4, c, d, a, b, in[2] + 0x2ad7d2bb, 15);
+	MD5STEP(F4, b, c, d, a, in[9] + 0xeb86d391, 21);
+
+	buf[0] += a;
+	buf[1] += b;
+	buf[2] += c;
+	buf[3] += d;
+}
+
+/*
+ * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
+ * initialization constants.
+ */
+void MD5Init(struct MD5Context *ctx)
+{
+	ctx->buf[0] = 0x67452301;
+	ctx->buf[1] = 0xefcdab89;
+	ctx->buf[2] = 0x98badcfe;
+	ctx->buf[3] = 0x10325476;
+
+	ctx->bytes[0] = 0;
+	ctx->bytes[1] = 0;
+}
+
+/*
+ * Update context to reflect the concatenation of another buffer full
+ * of bytes.
+ */
+void MD5Update(struct MD5Context *ctx, const uint8_t *buf, unsigned len)
+{
+	uint32_t t;
+
+	/* Update byte count */
+
+	t = ctx->bytes[0];
+	if ((ctx->bytes[0] = t + len) < t)
+		ctx->bytes[1]++;	/* Carry from low to high */
+
+	t = 64 - (t & 0x3f);	/* Space available in ctx->in (at least 1) */
+	if (t > len) {
+		memcpy((uint8_t*)ctx->in + 64 - t, buf, len);
+		return;
+	}
+	/* First chunk is an odd size */
+	memcpy((uint8_t*)ctx->in + 64 - t, buf, t);
+	byteSwap(ctx->in, 16);
+	MD5Transform(ctx->buf, ctx->in);
+	buf += t;
+	len -= t;
+
+	/* Process data in 64-byte chunks */
+	while (len >= 64) {
+		memcpy(ctx->in, buf, 64);
+		byteSwap(ctx->in, 16);
+		MD5Transform(ctx->buf, ctx->in);
+		buf += 64;
+		len -= 64;
+	}
+
+	/* Handle any remaining bytes of data. */
+	memcpy(ctx->in, buf, len);
+}
+
+/*
+ * Final wrapup - pad to 64-byte boundary with the bit pattern
+ * 1 0* (64-bit count of bits processed, MSB-first)
+ */
+void MD5Final(uint8_t digest[16], struct MD5Context *ctx)
+{
+	int count = ctx->bytes[0] & 0x3f;	/* Number of bytes in ctx->in */
+	uint8_t *p = (uint8_t*)ctx->in + count;
+
+	/* Set the first char of padding to 0x80.  There is always room. */
+	*p++ = 0x80;
+
+	/* Bytes of padding needed to make 56 bytes (-8..55) */
+	count = 56 - 1 - count;
+
+	if (count < 0) {	/* Padding forces an extra block */
+		memset(p, 0, count + 8);
+		byteSwap(ctx->in, 16);
+		MD5Transform(ctx->buf, ctx->in);
+		p = (uint8_t*)ctx->in;
+		count = 56;
+	}
+	memset(p, 0, count);
+	byteSwap(ctx->in, 14);
+
+	/* Append length in bits and transform */
+	ctx->in[14] = ctx->bytes[0] << 3;
+	ctx->in[15] = ctx->bytes[1] << 3 | ctx->bytes[0] >> 29;
+	MD5Transform(ctx->buf, ctx->in);
+
+	byteSwap(ctx->buf, 4);
+	memcpy(digest, ctx->buf, 16);
+	memset(ctx, 0, sizeof(*ctx));	/* In case it's sensitive */
+}
+
diff --git a/tools/TexturePacker/md5.h b/tools/TexturePacker/md5.h
new file mode 100644
index 0000000000..456e5be17f
--- /dev/null
+++ b/tools/TexturePacker/md5.h
@@ -0,0 +1,40 @@
+/*
+ * This code implements the MD5 message-digest algorithm.
+ * The algorithm is due to Ron Rivest.  This code was
+ * written by Colin Plumb in 1993, no copyright is claimed.
+ * This code is in the public domain; do with it what you wish.
+ *
+ * Equivalent code is available from RSA Data Security, Inc.
+ * This code has been tested against that, and is equivalent,
+ * except that you don't need to include two pages of legalese
+ * with every copy.
+ *
+ * To compute the message digest of a chunk of bytes, declare an
+ * MD5Context structure, pass it to MD5Init, call MD5Update as
+ * needed on buffers full of bytes, and then call MD5Final, which
+ * will fill a supplied 16-byte array with the digest.
+ *
+ * Changed so as no longer to depend on Colin Plumb's `usual.h' header
+ * definitions; now uses stuff from dpkg's config.h.
+ *  - Ian Jackson <ian@chiark.greenend.org.uk>.
+ * Still in the public domain.
+ */
+
+#ifndef MD5_H
+#define MD5_H
+
+#include <string.h>		/* for memcpy() */
+#include <stdint.h>
+
+struct MD5Context 
+{
+  uint32_t buf[4];
+  uint32_t bytes[2];
+  uint32_t in[16];
+};
+
+void MD5Init(struct MD5Context *ctx);
+void MD5Update(struct MD5Context *ctx, const uint8_t *buf, unsigned len);
+void MD5Final(unsigned char digest[16], struct MD5Context *ctx);
+
+#endif