From 5a4a20931aade60f0e99df0819fbd1c007a705da Mon Sep 17 00:00:00 2001
From: Matthias Kramm <kramm@quiss.org>
Date: Mon, 14 Jun 2010 17:49:20 -0700
Subject: [PATCH] multiply overflow fixes

---
 lib/gfxdevice.h         |    4 ++--
 lib/h.263/swfvideo.c    |    4 ++--
 lib/jpeg.c              |   38 ++++++++++++++++++++---------------
 lib/jpeg.h              |   14 ++++++-------
 lib/modules/swfrender.c |    5 ++---
 lib/png.c               |   50 ++++++++++++++++++++++++-----------------------
 lib/png.h               |   14 ++++++-------
 lib/readers/image.c     |    4 ++--
 src/png2swf.c           |    2 +-
 src/swfc.c              |    2 +-
 src/swfextract.c        |    2 +-
 11 files changed, 73 insertions(+), 66 deletions(-)

diff --git a/lib/gfxdevice.h b/lib/gfxdevice.h
index 699e7f6..c73117d 100644
--- a/lib/gfxdevice.h
+++ b/lib/gfxdevice.h
@@ -69,8 +69,8 @@ typedef struct _gfximage
     /* if the data contains an alpha layer (a != 255), the
        r,g,b values will have to be premultiplied */
     gfxcolor_t*data;
-    int width;
-    int height;
+    unsigned width;
+    unsigned height;
 } gfximage_t;
 
 /* gradients: A radial gradient will start at 0,0 and have a radius of 1,0 
diff --git a/lib/h.263/swfvideo.c b/lib/h.263/swfvideo.c
index ae31564..2ca6bde 100644
--- a/lib/h.263/swfvideo.c
+++ b/lib/h.263/swfvideo.c
@@ -1514,8 +1514,8 @@ int main(int argn, char*argv[])
     TAG * tag;
     RGBA* pic, *pic2, rgb;
     SWFPLACEOBJECT obj;
-    int width = 0;
-    int height = 0;
+    unsigned width = 0;
+    unsigned height = 0;
     int frames = 10;
     int framerate = 29;
     unsigned char*data;
diff --git a/lib/jpeg.c b/lib/jpeg.c
index 21cc0f7..03a2737 100644
--- a/lib/jpeg.c
+++ b/lib/jpeg.c
@@ -76,7 +76,7 @@ static void mem_term_destination(j_compress_ptr cinfo)
   dmgr->free_in_buffer = 0;
 }
 
-int jpeg_save(unsigned char*data, int width, int height, int quality, const char*filename)
+int jpeg_save(unsigned char*data, unsigned width, unsigned height, int quality, const char*filename)
 {
   struct jpeg_destination_mgr mgr;
   struct jpeg_compress_struct cinfo;
@@ -124,7 +124,7 @@ int jpeg_save(unsigned char*data, int width, int height, int quality, const char
   return 1;
 }
 
-int jpeg_save_gray(unsigned char*data, int width, int height, int quality, const char*filename)
+int jpeg_save_gray(unsigned char*data, unsigned width, unsigned height, int quality, const char*filename)
 {
   struct jpeg_destination_mgr mgr;
   struct jpeg_compress_struct cinfo;
@@ -163,7 +163,7 @@ int jpeg_save_gray(unsigned char*data, int width, int height, int quality, const
 }
 
 
-int jpeg_save_to_file(unsigned char*data, int width, int height, int quality, FILE*_fi)
+int jpeg_save_to_file(unsigned char*data, unsigned width, unsigned height, int quality, FILE*_fi)
 {
   struct jpeg_destination_mgr mgr;
   struct jpeg_compress_struct cinfo;
@@ -206,7 +206,7 @@ int jpeg_save_to_file(unsigned char*data, int width, int height, int quality, FI
   return 1;
 }
 
-int jpeg_save_to_mem(unsigned char*data, int width, int height, int quality, unsigned char*_dest, int _destlen)
+int jpeg_save_to_mem(unsigned char*data, unsigned width, unsigned height, int quality, unsigned char*_dest, int _destlen)
 {
   struct jpeg_destination_mgr mgr;
   struct jpeg_compress_struct cinfo;
@@ -288,7 +288,7 @@ void mem_term_source (j_decompress_ptr cinfo)
     //printf("term %d\n", size - mgr->bytes_in_buffer);
 }
 
-int jpeg_load_from_mem(unsigned char*_data, int _size, unsigned char**dest, int*width, int*height)
+int jpeg_load_from_mem(unsigned char*_data, int _size, unsigned char**dest, unsigned*width, unsigned*height)
 {
     struct jpeg_decompress_struct cinfo;
     struct jpeg_error_mgr jerr;
@@ -344,7 +344,7 @@ typedef struct _RGBA {
 
 typedef unsigned char U8;
 
-int jpeg_load(const char*filename, unsigned char**dest, int*_width, int*_height)
+int jpeg_load(const char*filename, unsigned char**dest, unsigned*_width, unsigned*_height)
 {
     struct jpeg_decompress_struct cinfo;
     struct jpeg_error_mgr jerr;
@@ -364,9 +364,15 @@ int jpeg_load(const char*filename, unsigned char**dest, int*_width, int*_height)
     
     U8*scanline = (U8 *)malloc(4 * cinfo.output_width);
 
-    int width = *_width = cinfo.output_width;
-    int height = *_height = cinfo.output_height;
-    *dest = (unsigned char*)malloc(width*height*4);
+    unsigned int width = *_width = cinfo.output_width;
+    unsigned int height = *_height = cinfo.output_height;
+    unsigned long long int image_size = (unsigned long long)width * height * 4;
+    if(image_size > 0xffffffff) {
+	*_width = 0;
+	*_height = 0;
+	return 0;
+    }
+    *dest = (unsigned char*)malloc(image_size);
 
     int y;
     for (y=0;y<height;y++) {
@@ -419,7 +425,7 @@ int jpeg_load(const char*filename, unsigned char**dest, int*_width, int*_height)
     return 1;
 }
 
-void jpeg_get_size(const char *filename, int *width, int *height)
+void jpeg_get_size(const char *filename, unsigned *width, unsigned*height)
 {
     struct jpeg_decompress_struct cinfo;
     struct jpeg_error_mgr jerr;
@@ -444,32 +450,32 @@ void jpeg_get_size(const char *filename, int *width, int *height)
 
 #else
 
-int jpeg_save(unsigned char*data, int width, int height, int quality, const char*filename)
+int jpeg_save(unsigned char*data, unsigned width, unsigned height, int quality, const char*filename)
 {
     fprintf(stderr, "jpeg_save: No JPEG support compiled in\n");
     return 0;
 }
-int jpeg_save_to_file(unsigned char*data, int width, int height, int quality, FILE*fi)
+int jpeg_save_to_file(unsigned char*data, unsigned width, unsigned height, int quality, FILE*fi)
 {
     fprintf(stderr, "jpeg_save_to_file: No JPEG support compiled in\n");
     return 0;
 }
-int jpeg_save_to_mem(unsigned char*data, int width, int height, int quality, unsigned char*dest, int destsize)
+int jpeg_save_to_mem(unsigned char*data, unsigned width, unsigned height, int quality, unsigned char*dest, int destsize)
 {
     fprintf(stderr, "jpeg_save_tomem: No JPEG support compiled in\n");
     return 0;
 }
-int jpeg_load_from_mem(unsigned char*_data, int size, unsigned char*dest, int width, int height)
+int jpeg_load_from_mem(unsigned char*_data, int size, unsigned char*dest, unsigned width, unsigned height)
 {
     fprintf(stderr, "jpeg_load_from_mem: No JPEG support compiled in\n");
     return 0;
 }
-int jpeg_load(const char*filename, unsigned char**dest, int*_width, int*_height)
+int jpeg_load(const char*filename, unsigned char**dest, unsigned*_width, unsigned*_height)
 {
     fprintf(stderr, "jpeg_load: No JPEG support compiled in\n");
     return 0;
 }
-void jpeg_get_size(const char *fname, int *width, int *height)
+void jpeg_get_size(const char *fname, unsigned *width, unsigned *height)
 {
     *width = 0;
     *height = 0;
diff --git a/lib/jpeg.h b/lib/jpeg.h
index 90f9fea..c6ef30b 100644
--- a/lib/jpeg.h
+++ b/lib/jpeg.h
@@ -7,13 +7,13 @@
 extern "C" {
 #endif
 
-int jpeg_save(unsigned char*data, int width, int height, int quality, const char*filename);
-int jpeg_save_gray(unsigned char*data, int width, int height, int quality, const char*filename);
-int jpeg_save_to_file(unsigned char*data, int width, int height, int quality, FILE*fi);
-int jpeg_save_to_mem(unsigned char*data, int width, int height, int quality, unsigned char*dest, int destsize);
-int jpeg_load(const char*filename, unsigned char**dest, int*width, int*height);
-int jpeg_load_from_mem(unsigned char*_data, int _size, unsigned char**dest, int*width, int*height);
-void jpeg_get_size(const char *fname, int *width, int *height);
+int jpeg_save(unsigned char*data, unsigned int width, unsigned int height, int quality, const char*filename);
+int jpeg_save_gray(unsigned char*data, unsigned int width, unsigned int height, int quality, const char*filename);
+int jpeg_save_to_file(unsigned char*data, unsigned int width, unsigned int height, int quality, FILE*fi);
+int jpeg_save_to_mem(unsigned char*data, unsigned int width, unsigned int height, int quality, unsigned char*dest, int destsize);
+int jpeg_load(const char*filename, unsigned char**dest, unsigned int*width, unsigned int*height);
+int jpeg_load_from_mem(unsigned char*_data, int _size, unsigned char**dest, unsigned int*width, unsigned int*height);
+void jpeg_get_size(const char *fname, unsigned int *width, unsigned int *height);
 
 #ifdef __cplusplus
 }
diff --git a/lib/modules/swfrender.c b/lib/modules/swfrender.c
index 9df996f..6c5262a 100644
--- a/lib/modules/swfrender.c
+++ b/lib/modules/swfrender.c
@@ -680,8 +680,6 @@ static void fill_gradient(RGBA*line, int*z, int y, int x1, int x2, MATRIX*m, GRA
 	    RGBA col;
 	    double xx = (  (x - rx) * m22 - (y - ry) * m21)*det;
 	    double yy = (- (x - rx) * m12 + (y - ry) * m11)*det;
-	    int ainv;
-	    ainv = 255-col.a;
 
 	    if(type == FILL_LINEAR) {
 		int xr = xx*256;
@@ -698,7 +696,8 @@ static void fill_gradient(RGBA*line, int*z, int y, int x1, int x2, MATRIX*m, GRA
 		    xr = 511;
 		col = palette[xr];
 	    }
-
+	    int ainv;
+	    ainv = 255-col.a;
 	    line[x].r = clamp(((line[x].r*ainv)>>8)+col.r);
 	    line[x].g = clamp(((line[x].g*ainv)>>8)+col.g);
 	    line[x].b = clamp(((line[x].b*ainv)>>8)+col.b);
diff --git a/lib/png.c b/lib/png.c
index 14de7f6..4aa4ddb 100644
--- a/lib/png.c
+++ b/lib/png.c
@@ -85,8 +85,8 @@ static unsigned int png_get_dword(FILE*fi)
 
 struct png_header
 {
-    int width;
-    int height;
+    unsigned width;
+    unsigned height;
     int bpp;
     int mode;
 };
@@ -167,7 +167,7 @@ static inline byte PaethPredictor (byte a,byte b,byte c)
         else return c;
 }
 
-static void applyfilter1(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, int width)
+static void applyfilter1(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, unsigned width)
 {
     int x;
     unsigned char last=0;
@@ -218,7 +218,7 @@ static void applyfilter1(int mode, unsigned char*src, unsigned char*old, unsigne
 
 }
 
-static void applyfilter2(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, int width)
+static void applyfilter2(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, unsigned width)
 {
     int x;
     unsigned char lasta=0;
@@ -281,7 +281,7 @@ static void applyfilter2(int mode, unsigned char*src, unsigned char*old, unsigne
 
 
 /* also performs 24 bit conversion! */
-static void applyfilter3(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, int width)
+static void applyfilter3(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, unsigned width)
 {
     int x;
     unsigned char lastr=0;
@@ -358,7 +358,7 @@ static void applyfilter3(int mode, unsigned char*src, unsigned char*old, unsigne
     }    
 }
 
-void png_inverse_filter_32(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, int width)
+void png_inverse_filter_32(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, unsigned width)
 {
     int x;
     unsigned char lastr=0;
@@ -441,7 +441,7 @@ void png_inverse_filter_32(int mode, unsigned char*src, unsigned char*old, unsig
     }    
 }
 
-EXPORT int getPNGdimensions(const char*sname, int*destwidth, int*destheight)
+EXPORT int getPNGdimensions(const char*sname, unsigned*destwidth, unsigned*destheight)
 {
     FILE*fi;
     struct png_header header;
@@ -459,14 +459,13 @@ EXPORT int getPNGdimensions(const char*sname, int*destwidth, int*destheight)
     return 1;
 }
 
-EXPORT int getPNG(const char*sname, int*destwidth, int*destheight, unsigned char**destdata)
+EXPORT int getPNG(const char*sname, unsigned*destwidth, unsigned*destheight, unsigned char**destdata)
 {
     char tagid[4];
     int len;
     unsigned char*data;
     unsigned char*imagedata;
     unsigned char*zimagedata=0;
-    unsigned long int imagedatalen;
     unsigned long int zimagedatalen=0;
     unsigned char*palette = 0;
     int palettelen = 0;
@@ -500,7 +499,10 @@ EXPORT int getPNG(const char*sname, int*destwidth, int*destheight, unsigned char
 	return 0;
     }
 
-    imagedatalen = bypp * header.width * header.height + 65536;
+    unsigned long long imagedatalen_64 = ((unsigned long long)header.width + 1) * header.height * bypp;
+    if(imagedatalen_64 > 0xffffffff)
+	return 0;
+    unsigned long imagedatalen = (unsigned long)imagedatalen_64;
     imagedata = (unsigned char*)malloc(imagedatalen);
 
     fseek(fi,8,SEEK_SET);
@@ -1151,7 +1153,7 @@ static inline u32 color_hash(COL*col)
     return hash;
 }
 
-static int png_get_number_of_palette_entries(COL*img, int width, int height, COL*palette, char*has_alpha)
+static int png_get_number_of_palette_entries(COL*img, unsigned width, unsigned height, COL*palette, char*has_alpha)
 {
     int len = width*height;
     int t;
@@ -1266,11 +1268,11 @@ static void png_map_to_palette(COL*src, unsigned char*dest, int size, COL*palett
     }
 }
 
-static int png_apply_specific_filter_8(int filtermode, unsigned char*dest, unsigned char*src, int width)
+static int png_apply_specific_filter_8(int filtermode, unsigned char*dest, unsigned char*src, unsigned width)
 {
     int pos2 = 0;
     int pos = 0;
-    int srcwidth = width;
+    unsigned srcwidth = width;
     int x;
     if(filtermode == 0) {
 	for(x=0;x<width;x++) {
@@ -1309,11 +1311,11 @@ static int png_apply_specific_filter_8(int filtermode, unsigned char*dest, unsig
     return filtermode;
 }
 
-static int png_apply_specific_filter_32(int filtermode, unsigned char*dest, unsigned char*src, int width)
+static int png_apply_specific_filter_32(int filtermode, unsigned char*dest, unsigned char*src, unsigned width)
 {
     int pos2 = 0;
     int pos = 0;
-    int srcwidth = width*4;
+    unsigned srcwidth = width*4;
     int x;
     if(filtermode == 0) {
 	for(x=0;x<width;x++) {
@@ -1396,7 +1398,7 @@ static void make_num_bits_table()
     }
 }
 
-static int png_find_best_filter(unsigned char*src, int width, int bpp, int y)
+static int png_find_best_filter(unsigned char*src, unsigned width, int bpp, int y)
 {
     make_num_bits_table();
     
@@ -1464,7 +1466,7 @@ static int png_find_best_filter(unsigned char*src, int width, int bpp, int y)
 }
     
     
-static int png_apply_filter(unsigned char*dest, unsigned char*src, int width, int y, int bpp)
+static int png_apply_filter(unsigned char*dest, unsigned char*src, unsigned width, int y, int bpp)
 {
     int best_nr = 0;
 #if 0
@@ -1512,16 +1514,16 @@ static int png_apply_filter(unsigned char*dest, unsigned char*src, int width, in
     return best_nr;
 }
 
-int png_apply_filter_8(unsigned char*dest, unsigned char*src, int width, int y)
+int png_apply_filter_8(unsigned char*dest, unsigned char*src, unsigned width, int y)
 {
     return png_apply_filter(dest, src, width, y, 8);
 }
-int png_apply_filter_32(unsigned char*dest, unsigned char*src, int width, int y)
+int png_apply_filter_32(unsigned char*dest, unsigned char*src, unsigned width, int y)
 {
     return png_apply_filter(dest, src, width, y, 32);
 }
 
-EXPORT void savePNG(const char*filename, unsigned char*data, int width, int height, int numcolors)
+EXPORT void savePNG(const char*filename, unsigned char*data, unsigned width, unsigned height, int numcolors)
 {
     FILE*fi;
     int crc;
@@ -1626,8 +1628,8 @@ EXPORT void savePNG(const char*filename, unsigned char*data, int width, int heig
     {
 	int x,y;
         int bypp = bpp/8;
-	int srcwidth = width * bypp;
-	int linelen = 1 + srcwidth;
+	unsigned srcwidth = width * bypp;
+	unsigned linelen = 1 + srcwidth;
         if(bypp==2) 
             linelen = 1 + ((srcwidth+1)&~1);
         else if(bypp==3) 
@@ -1687,11 +1689,11 @@ EXPORT void savePNG(const char*filename, unsigned char*data, int width, int heig
     fclose(fi);
 }
 
-EXPORT void writePNG(const char*filename, unsigned char*data, int width, int height)
+EXPORT void writePNG(const char*filename, unsigned char*data, unsigned width, unsigned height)
 {
     savePNG(filename, data, width, height, 0);
 }
-EXPORT void writePalettePNG(const char*filename, unsigned char*data, int width, int height)
+EXPORT void writePalettePNG(const char*filename, unsigned char*data, unsigned width, unsigned height)
 {
     savePNG(filename, data, width, height, 256);
 }
diff --git a/lib/png.h b/lib/png.h
index aa4b9c5..b97a85b 100644
--- a/lib/png.h
+++ b/lib/png.h
@@ -23,16 +23,16 @@
 extern "C" {
 #endif
 
-int png_apply_filter_32(unsigned char*dest, unsigned char*src, int width, int y);
-void png_inverse_filter_32(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, int width);
+int png_apply_filter_32(unsigned char*dest, unsigned char*src, unsigned width, int y);
+void png_inverse_filter_32(int mode, unsigned char*src, unsigned char*old, unsigned char*dest, unsigned width);
 
-int getPNG(const char*sname, int*destwidth, int*destheight, unsigned char**destdata);
-int getPNGdimensions(const char*sname, int*destwidth, int*destheight);
+int getPNG(const char*sname, unsigned*destwidth, unsigned*destheight, unsigned char**destdata);
+int getPNGdimensions(const char*sname, unsigned*destwidth, unsigned*destheight);
 
-void savePNG(const char*filename, unsigned char*data, int width, int height, int numcolors);
+void savePNG(const char*filename, unsigned char*data, unsigned width, unsigned height, int numcolors);
 
-void writePNG(const char*filename, unsigned char*data, int width, int height);
-void writePalettePNG(const char*filename, unsigned char*data, int width, int height);
+void writePNG(const char*filename, unsigned char*data, unsigned width, unsigned height);
+void writePalettePNG(const char*filename, unsigned char*data, unsigned width, unsigned height);
 
 #ifdef __cplusplus
 }
diff --git a/lib/readers/image.c b/lib/readers/image.c
index 7457a4e..b8f9a35 100644
--- a/lib/readers/image.c
+++ b/lib/readers/image.c
@@ -126,8 +126,8 @@ static gfxdocument_t*image_open(gfxsource_t*src, const char*filename)
     memset(i, 0, sizeof(image_doc_internal_t));
 
     gfxcolor_t*data = 0;
-    int width = 0;
-    int height = 0;
+    unsigned width = 0;
+    unsigned height = 0;
 
     if(!getPNG(filename, &width, &height, (unsigned char**)&data)) {
 	if(!jpeg_load(filename, (unsigned char**)&data, &width, &height)) {
diff --git a/src/png2swf.c b/src/png2swf.c
index 05366fc..8d1c476 100644
--- a/src/png2swf.c
+++ b/src/png2swf.c
@@ -749,7 +749,7 @@ TAG *MovieAddFrame(SWF * swf, TAG * t, char *sname, int id)
     MATRIX m;
     int fs;
 
-    int width=0, height=0;
+    unsigned width=0, height=0;
 
 #ifndef HAVE_JPEGLIB
     if(global.mkjpeg) {
diff --git a/src/swfc.c b/src/swfc.c
index 9163534..03a923a 100644
--- a/src/swfc.c
+++ b/src/swfc.c
@@ -1465,7 +1465,7 @@ void s_image(const char*name, const char*type, const char*filename, int quality)
     /* step 1: the bitmap */
     SRECT r;
     int imageID = id;
-    int width, height;
+    unsigned width, height;
     if(!strcmp(type,"jpeg")) {
 #ifndef HAVE_JPEGLIB
 	warning("no jpeg support compiled in");
diff --git a/src/swfextract.c b/src/swfextract.c
index 47bcbe7..c341f5f 100644
--- a/src/swfextract.c
+++ b/src/swfextract.c
@@ -741,7 +741,7 @@ int handlejpeg(TAG*tag)
 	    memmove(&tag->data[pos], &tag->data[pos+4], end-(pos+4));
 	}
 	unsigned char*image;
-	int width=0, height=0;
+	unsigned width=0, height=0;
 	jpeg_load_from_mem(&tag->data[6], end-6, &image, &width, &height);
 
 	uLongf datalen = width*height;
-- 
1.7.10.4