ui: refactor method for setting up VncDisplay auth types

There is a lot of repeated code in the auth type setup method,
particularly around checking TLS credential types. Refactor
it to reduce duplication and instead of having one method
do both plain and websockets at once, call it separately
for each.

Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
Message-id: 1475163940-26094-6-git-send-email-berrange@redhat.com
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>

2 files changed, 53 insertions, 72 deletions

diff --git a/ui/vnc.c b/ui/vnc.c
index 1104697e64..2f3ebdcd37 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -3325,7 +3325,9 @@ static QemuOptsList qemu_vnc_opts = {
 
 
 static int
-vnc_display_setup_auth(VncDisplay *vd,
+vnc_display_setup_auth(int *auth,
+                       int *subauth,
+                       QCryptoTLSCreds *tlscreds,
                        bool password,
                        bool sasl,
                        bool websocket,
@@ -3378,86 +3380,56 @@ vnc_display_setup_auth(VncDisplay *vd,
      * VNC auth mechs for plain VNC vs websockets VNC, the end
      * result has the same security characteristics.
      */
-    if (password) {
-        if (vd->tlscreds) {
-            vd->auth = VNC_AUTH_VENCRYPT;
-            if (object_dynamic_cast(OBJECT(vd->tlscreds),
-                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
-                VNC_DEBUG("Initializing VNC server with x509 password auth\n");
-                vd->subauth = VNC_AUTH_VENCRYPT_X509VNC;
-            } else if (object_dynamic_cast(OBJECT(vd->tlscreds),
-                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
-                VNC_DEBUG("Initializing VNC server with TLS password auth\n");
-                vd->subauth = VNC_AUTH_VENCRYPT_TLSVNC;
-            } else {
-                error_setg(errp,
-                           "Unsupported TLS cred type %s",
-                           object_get_typename(OBJECT(vd->tlscreds)));
-                return -1;
-            }
-        } else {
+    if (websocket || !tlscreds) {
+        if (password) {
             VNC_DEBUG("Initializing VNC server with password auth\n");
-            vd->auth = VNC_AUTH_VNC;
-            vd->subauth = VNC_AUTH_INVALID;
-        }
-        if (websocket) {
-            vd->ws_auth = VNC_AUTH_VNC;
+            *auth = VNC_AUTH_VNC;
+        } else if (sasl) {
+            VNC_DEBUG("Initializing VNC server with SASL auth\n");
+            *auth = VNC_AUTH_SASL;
         } else {
-            vd->ws_auth = VNC_AUTH_INVALID;
+            VNC_DEBUG("Initializing VNC server with no auth\n");
+            *auth = VNC_AUTH_NONE;
         }
-    } else if (sasl) {
-        if (vd->tlscreds) {
-            vd->auth = VNC_AUTH_VENCRYPT;
-            if (object_dynamic_cast(OBJECT(vd->tlscreds),
-                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
+        *subauth = VNC_AUTH_INVALID;
+    } else {
+        bool is_x509 = object_dynamic_cast(OBJECT(tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_X509) != NULL;
+        bool is_anon = object_dynamic_cast(OBJECT(tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON) != NULL;
+
+        if (!is_x509 && !is_anon) {
+            error_setg(errp,
+                       "Unsupported TLS cred type %s",
+                       object_get_typename(OBJECT(tlscreds)));
+            return -1;
+        }
+        *auth = VNC_AUTH_VENCRYPT;
+        if (password) {
+            if (is_x509) {
+                VNC_DEBUG("Initializing VNC server with x509 password auth\n");
+                *subauth = VNC_AUTH_VENCRYPT_X509VNC;
+            } else {
+                VNC_DEBUG("Initializing VNC server with TLS password auth\n");
+                *subauth = VNC_AUTH_VENCRYPT_TLSVNC;
+            }
+
+        } else if (sasl) {
+            if (is_x509) {
                 VNC_DEBUG("Initializing VNC server with x509 SASL auth\n");
-                vd->subauth = VNC_AUTH_VENCRYPT_X509SASL;
-            } else if (object_dynamic_cast(OBJECT(vd->tlscreds),
-                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
-                VNC_DEBUG("Initializing VNC server with TLS SASL auth\n");
-                vd->subauth = VNC_AUTH_VENCRYPT_TLSSASL;
+                *subauth = VNC_AUTH_VENCRYPT_X509SASL;
             } else {
-                error_setg(errp,
-                           "Unsupported TLS cred type %s",
-                           object_get_typename(OBJECT(vd->tlscreds)));
-                return -1;
+                VNC_DEBUG("Initializing VNC server with TLS SASL auth\n");
+                *subauth = VNC_AUTH_VENCRYPT_TLSSASL;
             }
         } else {
-            VNC_DEBUG("Initializing VNC server with SASL auth\n");
-            vd->auth = VNC_AUTH_SASL;
-            vd->subauth = VNC_AUTH_INVALID;
-        }
-        if (websocket) {
-            vd->ws_auth = VNC_AUTH_SASL;
-        } else {
-            vd->ws_auth = VNC_AUTH_INVALID;
-        }
-    } else {
-        if (vd->tlscreds) {
-            vd->auth = VNC_AUTH_VENCRYPT;
-            if (object_dynamic_cast(OBJECT(vd->tlscreds),
-                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
+            if (is_x509) {
                 VNC_DEBUG("Initializing VNC server with x509 no auth\n");
-                vd->subauth = VNC_AUTH_VENCRYPT_X509NONE;
-            } else if (object_dynamic_cast(OBJECT(vd->tlscreds),
-                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
-                VNC_DEBUG("Initializing VNC server with TLS no auth\n");
-                vd->subauth = VNC_AUTH_VENCRYPT_TLSNONE;
+                *subauth = VNC_AUTH_VENCRYPT_X509NONE;
             } else {
-                error_setg(errp,
-                           "Unsupported TLS cred type %s",
-                           object_get_typename(OBJECT(vd->tlscreds)));
-                return -1;
+                VNC_DEBUG("Initializing VNC server with TLS no auth\n");
+                *subauth = VNC_AUTH_VENCRYPT_TLSNONE;
             }
-        } else {
-            VNC_DEBUG("Initializing VNC server with no auth\n");
-            vd->auth = VNC_AUTH_NONE;
-            vd->subauth = VNC_AUTH_INVALID;
-        }
-        if (websocket) {
-            vd->ws_auth = VNC_AUTH_NONE;
-        } else {
-            vd->ws_auth = VNC_AUTH_INVALID;
         }
     }
     return 0;
@@ -3769,7 +3741,15 @@ void vnc_display_open(const char *id, Error **errp)
     }
 #endif
 
-    if (vnc_display_setup_auth(vd, password, sasl, ws_enabled, errp) < 0) {
+    if (vnc_display_setup_auth(&vd->auth, &vd->subauth,
+                               vd->tlscreds, password,
+                               sasl, false, errp) < 0) {
+        goto fail;
+    }
+
+    if (vnc_display_setup_auth(&vd->ws_auth, &vd->ws_subauth,
+                               vd->tlscreds, password,
+                               sasl, true, errp) < 0) {
         goto fail;
     }
 
diff --git a/ui/vnc.h b/ui/vnc.h
index 223af38515..d191d88356 100644
--- a/ui/vnc.h
+++ b/ui/vnc.h
@@ -172,6 +172,7 @@ struct VncDisplay
     int auth;
     int subauth; /* Used by VeNCrypt */
     int ws_auth; /* Used by websockets */
+    int ws_subauth; /* Used by websockets */
     bool lossy;
     bool non_adaptive;
     QCryptoTLSCreds *tlscreds;