vnc: split encoding in specific files

This will allow to implement new encodings (tight, zrle, ..)
in a cleaner way. This may hurt performances, because some
functions like vnc_convert_pixel are not static anymore, but
should not be a problem with gcc 4.5 and the new -flto.

Signed-off-by: Corentin Chary <corentincj@iksaif.net>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>

6 files changed, 300 insertions, 211 deletions

diff --git a/Makefile b/Makefile
index 18e736813f..eb9e02b7a0 100644
--- a/Makefile
+++ b/Makefile
@@ -120,6 +120,10 @@ vnc-auth-vencrypt.o: vnc-auth-vencrypt.c vnc.h
 
 vnc-auth-sasl.o: vnc-auth-sasl.c vnc.h
 
+vnc-encoding-zlib.o: vnc.h
+
+vnc-encoding-hextile.o: vnc.h
+
 curses.o: curses.c keymaps.h curses_keys.h
 
 bt-host.o: QEMU_CFLAGS += $(BLUEZ_CFLAGS)
diff --git a/Makefile.objs b/Makefile.objs
index 4f65bfbae5..603a8d7909 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -100,6 +100,7 @@ common-obj-y += keymaps.o
 common-obj-$(CONFIG_SDL) += sdl.o sdl_zoom.o x_keymap.o
 common-obj-$(CONFIG_CURSES) += curses.o
 common-obj-y += vnc.o acl.o d3des.o
+common-obj-y += vnc-encoding-zlib.o vnc-encoding-hextile.o
 common-obj-y += iov.o
 common-obj-$(CONFIG_VNC_TLS) += vnc-tls.o vnc-auth-vencrypt.o
 common-obj-$(CONFIG_VNC_SASL) += vnc-auth-sasl.o
diff --git a/vnc-encoding-hextile.c b/vnc-encoding-hextile.c
new file mode 100644
index 0000000000..a01c5e260f
--- /dev/null
+++ b/vnc-encoding-hextile.c
@@ -0,0 +1,115 @@
+/*
+ * QEMU VNC display driver: hextile encoding
+ *
+ * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
+ * Copyright (C) 2006 Fabrice Bellard
+ * Copyright (C) 2009 Red Hat, Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "vnc.h"
+
+static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
+{
+    ptr[0] = ((x & 0x0F) << 4) | (y & 0x0F);
+    ptr[1] = (((w - 1) & 0x0F) << 4) | ((h - 1) & 0x0F);
+}
+
+#define BPP 8
+#include "vnchextile.h"
+#undef BPP
+
+#define BPP 16
+#include "vnchextile.h"
+#undef BPP
+
+#define BPP 32
+#include "vnchextile.h"
+#undef BPP
+
+#define GENERIC
+#define BPP 8
+#include "vnchextile.h"
+#undef BPP
+#undef GENERIC
+
+#define GENERIC
+#define BPP 16
+#include "vnchextile.h"
+#undef BPP
+#undef GENERIC
+
+#define GENERIC
+#define BPP 32
+#include "vnchextile.h"
+#undef BPP
+#undef GENERIC
+
+void vnc_hextile_send_framebuffer_update(VncState *vs, int x,
+                                         int y, int w, int h)
+{
+    int i, j;
+    int has_fg, has_bg;
+    uint8_t *last_fg, *last_bg;
+    VncDisplay *vd = vs->vd;
+
+    last_fg = (uint8_t *) qemu_malloc(vd->server->pf.bytes_per_pixel);
+    last_bg = (uint8_t *) qemu_malloc(vd->server->pf.bytes_per_pixel);
+    has_fg = has_bg = 0;
+    for (j = y; j < (y + h); j += 16) {
+        for (i = x; i < (x + w); i += 16) {
+            vs->send_hextile_tile(vs, i, j,
+                                  MIN(16, x + w - i), MIN(16, y + h - j),
+                                  last_bg, last_fg, &has_bg, &has_fg);
+        }
+    }
+    free(last_fg);
+    free(last_bg);
+
+}
+
+void vnc_hextile_set_pixel_conversion(VncState *vs, int generic)
+{
+    if (!generic) {
+        switch (vs->ds->surface->pf.bits_per_pixel) {
+            case 8:
+                vs->send_hextile_tile = send_hextile_tile_8;
+                break;
+            case 16:
+                vs->send_hextile_tile = send_hextile_tile_16;
+                break;
+            case 32:
+                vs->send_hextile_tile = send_hextile_tile_32;
+                break;
+        }
+    } else {
+        switch (vs->ds->surface->pf.bits_per_pixel) {
+            case 8:
+                vs->send_hextile_tile = send_hextile_tile_generic_8;
+                break;
+            case 16:
+                vs->send_hextile_tile = send_hextile_tile_generic_16;
+                break;
+            case 32:
+                vs->send_hextile_tile = send_hextile_tile_generic_32;
+                break;
+        }
+    }
+}
diff --git a/vnc-encoding-zlib.c b/vnc-encoding-zlib.c
new file mode 100644
index 0000000000..4a495adf94
--- /dev/null
+++ b/vnc-encoding-zlib.c
@@ -0,0 +1,142 @@
+/*
+ * QEMU VNC display driver: zlib encoding
+ *
+ * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
+ * Copyright (C) 2006 Fabrice Bellard
+ * Copyright (C) 2009 Red Hat, Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "vnc.h"
+
+#define ZALLOC_ALIGNMENT 16
+
+static void *zalloc(void *x, unsigned items, unsigned size)
+{
+    void *p;
+
+    size *= items;
+    size = (size + ZALLOC_ALIGNMENT - 1) & ~(ZALLOC_ALIGNMENT - 1);
+
+    p = qemu_mallocz(size);
+
+    return (p);
+}
+
+static void zfree(void *x, void *addr)
+{
+    qemu_free(addr);
+}
+
+static void vnc_zlib_start(VncState *vs)
+{
+    buffer_reset(&vs->zlib);
+
+    // make the output buffer be the zlib buffer, so we can compress it later
+    vs->zlib_tmp = vs->output;
+    vs->output = vs->zlib;
+}
+
+static int vnc_zlib_stop(VncState *vs, int stream_id)
+{
+    z_streamp zstream = &vs->zlib_stream[stream_id];
+    int previous_out;
+
+    // switch back to normal output/zlib buffers
+    vs->zlib = vs->output;
+    vs->output = vs->zlib_tmp;
+
+    // compress the zlib buffer
+
+    // initialize the stream
+    // XXX need one stream per session
+    if (zstream->opaque != vs) {
+        int err;
+
+        VNC_DEBUG("VNC: initializing zlib stream %d\n", stream_id);
+        VNC_DEBUG("VNC: opaque = %p | vs = %p\n", zstream->opaque, vs);
+        zstream->zalloc = zalloc;
+        zstream->zfree = zfree;
+
+        err = deflateInit2(zstream, vs->tight_compression, Z_DEFLATED, MAX_WBITS,
+                           MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
+
+        if (err != Z_OK) {
+            fprintf(stderr, "VNC: error initializing zlib\n");
+            return -1;
+        }
+
+        zstream->opaque = vs;
+    }
+
+    // XXX what to do if tight_compression changed in between?
+
+    // reserve memory in output buffer
+    buffer_reserve(&vs->output, vs->zlib.offset + 64);
+
+    // set pointers
+    zstream->next_in = vs->zlib.buffer;
+    zstream->avail_in = vs->zlib.offset;
+    zstream->next_out = vs->output.buffer + vs->output.offset;
+    zstream->avail_out = vs->output.capacity - vs->output.offset;
+    zstream->data_type = Z_BINARY;
+    previous_out = zstream->total_out;
+
+    // start encoding
+    if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) {
+        fprintf(stderr, "VNC: error during zlib compression\n");
+        return -1;
+    }
+
+    vs->output.offset = vs->output.capacity - zstream->avail_out;
+    return zstream->total_out - previous_out;
+}
+
+void vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
+{
+    int old_offset, new_offset, bytes_written;
+
+    vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_ZLIB);
+
+    // remember where we put in the follow-up size
+    old_offset = vs->output.offset;
+    vnc_write_s32(vs, 0);
+
+    // compress the stream
+    vnc_zlib_start(vs);
+    vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+    bytes_written = vnc_zlib_stop(vs, 0);
+
+    if (bytes_written == -1)
+        return;
+
+    // hack in the size
+    new_offset = vs->output.offset;
+    vs->output.offset = old_offset;
+    vnc_write_u32(vs, bytes_written);
+    vs->output.offset = new_offset;
+}
+
+void vnc_zlib_init(VncState *vs)
+{
+    int i;
+    for (i=0; i<(sizeof(vs->zlib_stream) / sizeof(z_stream)); i++)
+        vs->zlib_stream[i].opaque = NULL;
+}
diff --git a/vnc.c b/vnc.c
index d332099ffd..5241a6aa69 100644
--- a/vnc.c
+++ b/vnc.c
@@ -468,8 +468,8 @@ static void vnc_dpy_update(DisplayState *ds, int x, int y, int w, int h)
             vnc_set_bit(s->dirty[y], (x + i) / 16);
 }
 
-static void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
-                                   int32_t encoding)
+void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
+                            int32_t encoding)
 {
     vnc_write_u16(vs, x);
     vnc_write_u16(vs, y);
@@ -560,7 +560,7 @@ static void vnc_write_pixels_copy(VncState *vs, void *pixels, int size)
 }
 
 /* slowest but generic code. */
-static void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
+void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
 {
     uint8_t r, g, b;
     VncDisplay *vd = vs->vd;
@@ -638,7 +638,7 @@ static void vnc_write_pixels_generic(VncState *vs, void *pixels1, int size)
     }
 }
 
-static void send_framebuffer_update_raw(VncState *vs, int x, int y, int w, int h)
+void vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     int i;
     uint8_t *row;
@@ -651,192 +651,19 @@ static void send_framebuffer_update_raw(VncState *vs, int x, int y, int w, int h
     }
 }
 
-static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
-{
-    ptr[0] = ((x & 0x0F) << 4) | (y & 0x0F);
-    ptr[1] = (((w - 1) & 0x0F) << 4) | ((h - 1) & 0x0F);
-}
-
-#define BPP 8
-#include "vnchextile.h"
-#undef BPP
-
-#define BPP 16
-#include "vnchextile.h"
-#undef BPP
-
-#define BPP 32
-#include "vnchextile.h"
-#undef BPP
-
-#define GENERIC
-#define BPP 8
-#include "vnchextile.h"
-#undef BPP
-#undef GENERIC
-
-#define GENERIC
-#define BPP 16
-#include "vnchextile.h"
-#undef BPP
-#undef GENERIC
-
-#define GENERIC
-#define BPP 32
-#include "vnchextile.h"
-#undef BPP
-#undef GENERIC
-
-static void send_framebuffer_update_hextile(VncState *vs, int x, int y, int w, int h)
-{
-    int i, j;
-    int has_fg, has_bg;
-    uint8_t *last_fg, *last_bg;
-    VncDisplay *vd = vs->vd;
-
-    last_fg = (uint8_t *) qemu_malloc(vd->server->pf.bytes_per_pixel);
-    last_bg = (uint8_t *) qemu_malloc(vd->server->pf.bytes_per_pixel);
-    has_fg = has_bg = 0;
-    for (j = y; j < (y + h); j += 16) {
-        for (i = x; i < (x + w); i += 16) {
-            vs->send_hextile_tile(vs, i, j,
-                                  MIN(16, x + w - i), MIN(16, y + h - j),
-                                  last_bg, last_fg, &has_bg, &has_fg);
-        }
-    }
-    free(last_fg);
-    free(last_bg);
-
-}
-
-#define ZALLOC_ALIGNMENT 16
-
-static void *zalloc(void *x, unsigned items, unsigned size)
-{
-    void *p;
-
-    size *= items;
-    size = (size + ZALLOC_ALIGNMENT - 1) & ~(ZALLOC_ALIGNMENT - 1);
-
-    p = qemu_mallocz(size);
-
-    return (p);
-}
-
-static void zfree(void *x, void *addr)
-{
-    qemu_free(addr);
-}
-
-static void vnc_zlib_init(VncState *vs)
-{
-    int i;
-    for (i=0; i<(sizeof(vs->zlib_stream) / sizeof(z_stream)); i++)
-        vs->zlib_stream[i].opaque = NULL;
-}
-
-static void vnc_zlib_start(VncState *vs)
-{
-    buffer_reset(&vs->zlib);
-
-    // make the output buffer be the zlib buffer, so we can compress it later
-    vs->zlib_tmp = vs->output;
-    vs->output = vs->zlib;
-}
-
-static int vnc_zlib_stop(VncState *vs, int stream_id)
-{
-    z_streamp zstream = &vs->zlib_stream[stream_id];
-    int previous_out;
-
-    // switch back to normal output/zlib buffers
-    vs->zlib = vs->output;
-    vs->output = vs->zlib_tmp;
-
-    // compress the zlib buffer
-
-    // initialize the stream
-    // XXX need one stream per session
-    if (zstream->opaque != vs) {
-        int err;
-
-        VNC_DEBUG("VNC: initializing zlib stream %d\n", stream_id);
-        VNC_DEBUG("VNC: opaque = %p | vs = %p\n", zstream->opaque, vs);
-        zstream->zalloc = zalloc;
-        zstream->zfree = zfree;
-
-        err = deflateInit2(zstream, vs->tight_compression, Z_DEFLATED, MAX_WBITS,
-                           MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);
-
-        if (err != Z_OK) {
-            fprintf(stderr, "VNC: error initializing zlib\n");
-            return -1;
-        }
-
-        zstream->opaque = vs;
-    }
-
-    // XXX what to do if tight_compression changed in between?
-
-    // reserve memory in output buffer
-    buffer_reserve(&vs->output, vs->zlib.offset + 64);
-
-    // set pointers
-    zstream->next_in = vs->zlib.buffer;
-    zstream->avail_in = vs->zlib.offset;
-    zstream->next_out = vs->output.buffer + vs->output.offset;
-    zstream->avail_out = vs->output.capacity - vs->output.offset;
-    zstream->data_type = Z_BINARY;
-    previous_out = zstream->total_out;
-
-    // start encoding
-    if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) {
-        fprintf(stderr, "VNC: error during zlib compression\n");
-        return -1;
-    }
-
-    vs->output.offset = vs->output.capacity - zstream->avail_out;
-    return zstream->total_out - previous_out;
-}
-
-static void send_framebuffer_update_zlib(VncState *vs, int x, int y, int w, int h)
-{
-    int old_offset, new_offset, bytes_written;
-
-    vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_ZLIB);
-
-    // remember where we put in the follow-up size
-    old_offset = vs->output.offset;
-    vnc_write_s32(vs, 0);
-
-    // compress the stream
-    vnc_zlib_start(vs);
-    send_framebuffer_update_raw(vs, x, y, w, h);
-    bytes_written = vnc_zlib_stop(vs, 0);
-
-    if (bytes_written == -1)
-        return;
-
-    // hack in the size
-    new_offset = vs->output.offset;
-    vs->output.offset = old_offset;
-    vnc_write_u32(vs, bytes_written);
-    vs->output.offset = new_offset;
-}
-
 static void send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     switch(vs->vnc_encoding) {
         case VNC_ENCODING_ZLIB:
-            send_framebuffer_update_zlib(vs, x, y, w, h);
+            vnc_hextile_send_framebuffer_update(vs, x, y, w, h);
             break;
         case VNC_ENCODING_HEXTILE:
             vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_HEXTILE);
-            send_framebuffer_update_hextile(vs, x, y, w, h);
+            vnc_hextile_send_framebuffer_update(vs, x, y, w, h);
             break;
         default:
             vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
-            send_framebuffer_update_raw(vs, x, y, w, h);
+            vnc_raw_send_framebuffer_update(vs, x, y, w, h);
             break;
     }
 }
@@ -1823,30 +1650,10 @@ static void set_pixel_conversion(VncState *vs)
         (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) && 
         !memcmp(&(vs->clientds.pf), &(vs->ds->surface->pf), sizeof(PixelFormat))) {
         vs->write_pixels = vnc_write_pixels_copy;
-        switch (vs->ds->surface->pf.bits_per_pixel) {
-            case 8:
-                vs->send_hextile_tile = send_hextile_tile_8;
-                break;
-            case 16:
-                vs->send_hextile_tile = send_hextile_tile_16;
-                break;
-            case 32:
-                vs->send_hextile_tile = send_hextile_tile_32;
-                break;
-        }
+        vnc_hextile_set_pixel_conversion(vs, 0);
     } else {
         vs->write_pixels = vnc_write_pixels_generic;
-        switch (vs->ds->surface->pf.bits_per_pixel) {
-            case 8:
-                vs->send_hextile_tile = send_hextile_tile_generic_8;
-                break;
-            case 16:
-                vs->send_hextile_tile = send_hextile_tile_generic_16;
-                break;
-            case 32:
-                vs->send_hextile_tile = send_hextile_tile_generic_32;
-                break;
-        }
+        vnc_hextile_set_pixel_conversion(vs, 1);
     }
 }
 
@@ -1903,12 +1710,9 @@ static void pixel_format_message (VncState *vs) {
     vnc_write_u8(vs, vs->ds->surface->pf.rshift);    /* red-shift */
     vnc_write_u8(vs, vs->ds->surface->pf.gshift);    /* green-shift */
     vnc_write_u8(vs, vs->ds->surface->pf.bshift);    /* blue-shift */
-    if (vs->ds->surface->pf.bits_per_pixel == 32)
-        vs->send_hextile_tile = send_hextile_tile_32;
-    else if (vs->ds->surface->pf.bits_per_pixel == 16)
-        vs->send_hextile_tile = send_hextile_tile_16;
-    else if (vs->ds->surface->pf.bits_per_pixel == 8)
-        vs->send_hextile_tile = send_hextile_tile_8;
+
+    vnc_hextile_set_pixel_conversion(vs, 0);
+
     vs->clientds = *(vs->ds->surface);
     vs->clientds.flags &= ~QEMU_ALLOCATED_FLAG;
     vs->write_pixels = vnc_write_pixels_copy;
diff --git a/vnc.h b/vnc.h
index b593608066..1aa71b0085 100644
--- a/vnc.h
+++ b/vnc.h
@@ -132,8 +132,6 @@ struct VncState
     int last_y;
 
     uint32_t vnc_encoding;
-    uint8_t tight_quality;
-    uint8_t tight_compression;
 
     int major;
     int minor;
@@ -152,7 +150,6 @@ struct VncState
     Buffer input;
     /* current output mode information */
     VncWritePixels *write_pixels;
-    VncSendHextileTile *send_hextile_tile;
     DisplaySurface clientds;
 
     CaptureVoiceOut *audio_cap;
@@ -164,6 +161,16 @@ struct VncState
     uint8_t modifiers_state[256];
     QEMUPutLEDEntry *led;
 
+    /* Encoding specific */
+
+    /* Tight */
+    uint8_t tight_quality;
+    uint8_t tight_compression;
+
+    /* Hextile */
+    VncSendHextileTile *send_hextile_tile;
+
+    /* Zlib */
     Buffer zlib;
     Buffer zlib_tmp;
     z_stream zlib_stream[4];
@@ -376,4 +383,20 @@ void buffer_append(Buffer *buffer, const void *data, size_t len);
 char *vnc_socket_local_addr(const char *format, int fd);
 char *vnc_socket_remote_addr(const char *format, int fd);
 
+/* Framebuffer */
+void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
+                            int32_t encoding);
+
+void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v);
+
+/* Encodings */
+void vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h);
+
+void vnc_hextile_send_framebuffer_update(VncState *vs, int x,
+                                         int y, int w, int h);
+void vnc_hextile_set_pixel_conversion(VncState *vs, int generic);
+
+void vnc_zlib_init(VncState *vs);
+void vnc_zlib_send_framebuffer_update(VncState *vs, int x, int y, int w, int h);
+
 #endif /* __QEMU_VNC_H */