ARM TCG conversion 15/16.

git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@4152 c046a42c-6fe2-441c-8c8c-71466251a162

8 files changed, 1124 insertions, 812 deletions

diff --git a/Makefile.target b/Makefile.target
index fef43646f3..1aecec5d30 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -211,7 +211,7 @@ LIBOBJS+= op_helper.o helper.o
 endif
 
 ifeq ($(TARGET_BASE_ARCH), arm)
-LIBOBJS+= op_helper.o helper.o neon_helper.o
+LIBOBJS+= op_helper.o helper.o neon_helper.o iwmmxt_helper.o
 endif
 
 ifeq ($(TARGET_BASE_ARCH), sh4)
diff --git a/target-arm/helpers.h b/target-arm/helpers.h
index 1ae36eef4a..8b3137204c 100644
--- a/target-arm/helpers.h
+++ b/target-arm/helpers.h
@@ -431,6 +431,103 @@ DEF_HELPER_1_2(neon_cgt_f32, uint32_t, (uint32_t, uint32_t))
 DEF_HELPER_1_2(neon_acge_f32, uint32_t, (uint32_t, uint32_t))
 DEF_HELPER_1_2(neon_acgt_f32, uint32_t, (uint32_t, uint32_t))
 
+/* iwmmxt_helper.c */
+DEF_HELPER_1_2(iwmmxt_maddsq, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_madduq, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_sadb, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_sadw, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_mulslw, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_mulshw, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_mululw, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_muluhw, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_macsw, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_2(iwmmxt_macuw, uint64_t, (uint64_t, uint64_t))
+DEF_HELPER_1_1(iwmmxt_setpsr_nz, uint32_t, (uint64_t))
+
+#define DEF_IWMMXT_HELPER_SIZE_ENV(name) \
+DEF_HELPER_1_3(iwmmxt_##name##b, uint64_t, (CPUState *, uint64_t, uint64_t)) \
+DEF_HELPER_1_3(iwmmxt_##name##w, uint64_t, (CPUState *, uint64_t, uint64_t)) \
+DEF_HELPER_1_3(iwmmxt_##name##l, uint64_t, (CPUState *, uint64_t, uint64_t)) \
+
+DEF_IWMMXT_HELPER_SIZE_ENV(unpackl)
+DEF_IWMMXT_HELPER_SIZE_ENV(unpackh)
+
+DEF_HELPER_1_2(iwmmxt_unpacklub, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpackluw, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpacklul, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpackhub, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpackhuw, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpackhul, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpacklsb, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpacklsw, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpacklsl, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpackhsb, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpackhsw, uint64_t, (CPUState *, uint64_t))
+DEF_HELPER_1_2(iwmmxt_unpackhsl, uint64_t, (CPUState *, uint64_t))
+
+DEF_IWMMXT_HELPER_SIZE_ENV(cmpeq)
+DEF_IWMMXT_HELPER_SIZE_ENV(cmpgtu)
+DEF_IWMMXT_HELPER_SIZE_ENV(cmpgts)
+
+DEF_IWMMXT_HELPER_SIZE_ENV(mins)
+DEF_IWMMXT_HELPER_SIZE_ENV(minu)
+DEF_IWMMXT_HELPER_SIZE_ENV(maxs)
+DEF_IWMMXT_HELPER_SIZE_ENV(maxu)
+
+DEF_IWMMXT_HELPER_SIZE_ENV(subn)
+DEF_IWMMXT_HELPER_SIZE_ENV(addn)
+DEF_IWMMXT_HELPER_SIZE_ENV(subu)
+DEF_IWMMXT_HELPER_SIZE_ENV(addu)
+DEF_IWMMXT_HELPER_SIZE_ENV(subs)
+DEF_IWMMXT_HELPER_SIZE_ENV(adds)
+
+DEF_HELPER_1_3(iwmmxt_avgb0, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_avgb1, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_avgw0, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_avgw1, uint64_t, (CPUState *, uint64_t, uint64_t))
+
+DEF_HELPER_1_2(iwmmxt_msadb, uint64_t, (uint64_t, uint64_t))
+
+DEF_HELPER_1_3(iwmmxt_align, uint64_t, (uint64_t, uint64_t, uint32_t))
+DEF_HELPER_1_4(iwmmxt_insr, uint64_t, (uint64_t, uint32_t, uint32_t, uint32_t))
+
+DEF_HELPER_1_1(iwmmxt_bcstb, uint64_t, (uint32_t))
+DEF_HELPER_1_1(iwmmxt_bcstw, uint64_t, (uint32_t))
+DEF_HELPER_1_1(iwmmxt_bcstl, uint64_t, (uint32_t))
+
+DEF_HELPER_1_1(iwmmxt_addcb, uint64_t, (uint64_t))
+DEF_HELPER_1_1(iwmmxt_addcw, uint64_t, (uint64_t))
+DEF_HELPER_1_1(iwmmxt_addcl, uint64_t, (uint64_t))
+
+DEF_HELPER_1_1(iwmmxt_msbb, uint32_t, (uint64_t))
+DEF_HELPER_1_1(iwmmxt_msbw, uint32_t, (uint64_t))
+DEF_HELPER_1_1(iwmmxt_msbl, uint32_t, (uint64_t))
+
+DEF_HELPER_1_3(iwmmxt_srlw, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_srll, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_srlq, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_sllw, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_slll, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_sllq, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_sraw, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_sral, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_sraq, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_rorw, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_rorl, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_rorq, uint64_t, (CPUState *, uint64_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_shufh, uint64_t, (CPUState *, uint64_t, uint32_t))
+
+DEF_HELPER_1_3(iwmmxt_packuw, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_packul, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_packuq, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_packsw, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_packsl, uint64_t, (CPUState *, uint64_t, uint64_t))
+DEF_HELPER_1_3(iwmmxt_packsq, uint64_t, (CPUState *, uint64_t, uint64_t))
+
+DEF_HELPER_1_3(iwmmxt_muladdsl, uint64_t, (uint64_t, uint32_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_muladdsw, uint64_t, (uint64_t, uint32_t, uint32_t))
+DEF_HELPER_1_3(iwmmxt_muladdswl, uint64_t, (uint64_t, uint32_t, uint32_t))
+
 #undef DEF_HELPER
 #undef DEF_HELPER_0_0
 #undef DEF_HELPER_0_1
diff --git a/target-arm/iwmmxt_helper.c b/target-arm/iwmmxt_helper.c
new file mode 100644
index 0000000000..6e801c863d
--- /dev/null
+++ b/target-arm/iwmmxt_helper.c
@@ -0,0 +1,682 @@
+/*
+ * iwMMXt micro operations for XScale.
+ *
+ * Copyright (c) 2007 OpenedHand, Ltd.
+ * Written by Andrzej Zaborowski <andrew@openedhand.com>
+ * Copyright (c) 2008 CodeSourcery
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "cpu.h"
+#include "exec-all.h"
+#include "helpers.h"
+
+/* iwMMXt macros extracted from GNU gdb.  */
+
+/* Set the SIMD wCASF flags for 8, 16, 32 or 64-bit operations.  */
+#define SIMD8_SET( v, n, b)	((v != 0) << ((((b) + 1) * 4) + (n)))
+#define SIMD16_SET(v, n, h)	((v != 0) << ((((h) + 1) * 8) + (n)))
+#define SIMD32_SET(v, n, w)	((v != 0) << ((((w) + 1) * 16) + (n)))
+#define SIMD64_SET(v, n)	((v != 0) << (32 + (n)))
+/* Flags to pass as "n" above.  */
+#define SIMD_NBIT	-1
+#define SIMD_ZBIT	-2
+#define SIMD_CBIT	-3
+#define SIMD_VBIT	-4
+/* Various status bit macros.  */
+#define NBIT8(x)	((x) & 0x80)
+#define NBIT16(x)	((x) & 0x8000)
+#define NBIT32(x)	((x) & 0x80000000)
+#define NBIT64(x)	((x) & 0x8000000000000000ULL)
+#define ZBIT8(x)	(((x) & 0xff) == 0)
+#define ZBIT16(x)	(((x) & 0xffff) == 0)
+#define ZBIT32(x)	(((x) & 0xffffffff) == 0)
+#define ZBIT64(x)	(x == 0)
+/* Sign extension macros.  */
+#define EXTEND8H(a)	((uint16_t) (int8_t) (a))
+#define EXTEND8(a)	((uint32_t) (int8_t) (a))
+#define EXTEND16(a)	((uint32_t) (int16_t) (a))
+#define EXTEND16S(a)	((int32_t) (int16_t) (a))
+#define EXTEND32(a)	((uint64_t) (int32_t) (a))
+
+uint64_t HELPER(iwmmxt_maddsq)(uint64_t a, uint64_t b)
+{
+    a = ((
+            EXTEND16S((a >> 0) & 0xffff) * EXTEND16S((b >> 0) & 0xffff) +
+            EXTEND16S((a >> 16) & 0xffff) * EXTEND16S((b >> 16) & 0xffff)
+        ) & 0xffffffff) | ((uint64_t) (
+            EXTEND16S((a >> 32) & 0xffff) * EXTEND16S((b >> 32) & 0xffff) +
+            EXTEND16S((a >> 48) & 0xffff) * EXTEND16S((b >> 48) & 0xffff)
+        ) << 32);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_madduq)(uint64_t a, uint64_t b)
+{
+    a = ((
+            ((a >> 0) & 0xffff) * ((b >> 0) & 0xffff) +
+            ((a >> 16) & 0xffff) * ((b >> 16) & 0xffff)
+        ) & 0xffffffff) | ((
+            ((a >> 32) & 0xffff) * ((b >> 32) & 0xffff) +
+            ((a >> 48) & 0xffff) * ((b >> 48) & 0xffff)
+        ) << 32);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_sadb)(uint64_t a, uint64_t b)
+{
+#define abs(x) (((x) >= 0) ? x : -x)
+#define SADB(SHR) abs((int) ((a >> SHR) & 0xff) - (int) ((b >> SHR) & 0xff))
+    return
+        SADB(0) + SADB(8) + SADB(16) + SADB(24) +
+        SADB(32) + SADB(40) + SADB(48) + SADB(56);
+#undef SADB
+}
+
+uint64_t HELPER(iwmmxt_sadw)(uint64_t a, uint64_t b)
+{
+#define SADW(SHR) \
+    abs((int) ((a >> SHR) & 0xffff) - (int) ((b >> SHR) & 0xffff))
+    return SADW(0) + SADW(16) + SADW(32) + SADW(48);
+#undef SADW
+}
+
+uint64_t HELPER(iwmmxt_mulslw)(uint64_t a, uint64_t b)
+{
+#define MULS(SHR) ((uint64_t) ((( \
+        EXTEND16S((a >> SHR) & 0xffff) * EXTEND16S((b >> SHR) & 0xffff) \
+    ) >> 0) & 0xffff) << SHR)
+    return MULS(0) | MULS(16) | MULS(32) | MULS(48);
+#undef MULS
+}
+
+uint64_t HELPER(iwmmxt_mulshw)(uint64_t a, uint64_t b)
+{
+#define MULS(SHR) ((uint64_t) ((( \
+        EXTEND16S((a >> SHR) & 0xffff) * EXTEND16S((b >> SHR) & 0xffff) \
+    ) >> 16) & 0xffff) << SHR)
+    return MULS(0) | MULS(16) | MULS(32) | MULS(48);
+#undef MULS
+}
+
+uint64_t HELPER(iwmmxt_mululw)(uint64_t a, uint64_t b)
+{
+#define MULU(SHR) ((uint64_t) ((( \
+        ((a >> SHR) & 0xffff) * ((b >> SHR) & 0xffff) \
+    ) >> 0) & 0xffff) << SHR)
+    return MULU(0) | MULU(16) | MULU(32) | MULU(48);
+#undef MULU
+}
+
+uint64_t HELPER(iwmmxt_muluhw)(uint64_t a, uint64_t b)
+{
+#define MULU(SHR) ((uint64_t) ((( \
+        ((a >> SHR) & 0xffff) * ((b >> SHR) & 0xffff) \
+    ) >> 16) & 0xffff) << SHR)
+    return MULU(0) | MULU(16) | MULU(32) | MULU(48);
+#undef MULU
+}
+
+uint64_t HELPER(iwmmxt_macsw)(uint64_t a, uint64_t b)
+{
+#define MACS(SHR) ( \
+        EXTEND16((a >> SHR) & 0xffff) * EXTEND16S((b >> SHR) & 0xffff))
+    return (int64_t) (MACS(0) + MACS(16) + MACS(32) + MACS(48));
+#undef MACS
+}
+
+uint64_t HELPER(iwmmxt_macuw)(uint64_t a, uint64_t b)
+{
+#define MACU(SHR) ( \
+        (uint32_t) ((a >> SHR) & 0xffff) * \
+        (uint32_t) ((b >> SHR) & 0xffff))
+    return MACU(0) + MACU(16) + MACU(32) + MACU(48);
+#undef MACU
+}
+
+#define NZBIT8(x, i) \
+    SIMD8_SET(NBIT8((x) & 0xff), SIMD_NBIT, i) | \
+    SIMD8_SET(ZBIT8((x) & 0xff), SIMD_ZBIT, i)
+#define NZBIT16(x, i) \
+    SIMD16_SET(NBIT16((x) & 0xffff), SIMD_NBIT, i) | \
+    SIMD16_SET(ZBIT16((x) & 0xffff), SIMD_ZBIT, i)
+#define NZBIT32(x, i) \
+    SIMD32_SET(NBIT32((x) & 0xffffffff), SIMD_NBIT, i) | \
+    SIMD32_SET(ZBIT32((x) & 0xffffffff), SIMD_ZBIT, i)
+#define NZBIT64(x) \
+    SIMD64_SET(NBIT64(x), SIMD_NBIT) | \
+    SIMD64_SET(ZBIT64(x), SIMD_ZBIT)
+#define IWMMXT_OP_UNPACK(S, SH0, SH1, SH2, SH3)			\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, b)))(CPUState *env, \
+                                                 uint64_t a, uint64_t b) \
+{								\
+    a =							        \
+        (((a >> SH0) & 0xff) << 0) | (((b >> SH0) & 0xff) << 8) |	\
+        (((a >> SH1) & 0xff) << 16) | (((b >> SH1) & 0xff) << 24) |	\
+        (((a >> SH2) & 0xff) << 32) | (((b >> SH2) & 0xff) << 40) |	\
+        (((a >> SH3) & 0xff) << 48) | (((b >> SH3) & 0xff) << 56);	\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT8(a >> 0, 0) | NZBIT8(a >> 8, 1) |		        \
+        NZBIT8(a >> 16, 2) | NZBIT8(a >> 24, 3) |		\
+        NZBIT8(a >> 32, 4) | NZBIT8(a >> 40, 5) |		\
+        NZBIT8(a >> 48, 6) | NZBIT8(a >> 56, 7);		\
+    return a;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, w)))(CPUState *env, \
+                                        uint64_t a, uint64_t b) \
+{								\
+    a =							        \
+        (((a >> SH0) & 0xffff) << 0) |				\
+        (((b >> SH0) & 0xffff) << 16) |			        \
+        (((a >> SH2) & 0xffff) << 32) |			        \
+        (((b >> SH2) & 0xffff) << 48);				\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT8(a >> 0, 0) | NZBIT8(a >> 16, 1) |		\
+        NZBIT8(a >> 32, 2) | NZBIT8(a >> 48, 3);		\
+    return a;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, l)))(CPUState *env, \
+                                        uint64_t a, uint64_t b) \
+{								\
+    a =							        \
+        (((a >> SH0) & 0xffffffff) << 0) |			\
+        (((b >> SH0) & 0xffffffff) << 32);			\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT32(a >> 0, 0) | NZBIT32(a >> 32, 1);		\
+    return a;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, ub)))(CPUState *env, \
+                                                  uint64_t x)   \
+{								\
+    x =							        \
+        (((x >> SH0) & 0xff) << 0) |				\
+        (((x >> SH1) & 0xff) << 16) |				\
+        (((x >> SH2) & 0xff) << 32) |				\
+        (((x >> SH3) & 0xff) << 48);				\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT16(x >> 0, 0) | NZBIT16(x >> 16, 1) |		\
+        NZBIT16(x >> 32, 2) | NZBIT16(x >> 48, 3);		\
+    return x;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, uw)))(CPUState *env, \
+                                                  uint64_t x)   \
+{								\
+    x =							        \
+        (((x >> SH0) & 0xffff) << 0) |				\
+        (((x >> SH2) & 0xffff) << 32);				\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT32(x >> 0, 0) | NZBIT32(x >> 32, 1);		\
+    return x;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, ul)))(CPUState *env, \
+                                                  uint64_t x)   \
+{								\
+    x = (((x >> SH0) & 0xffffffff) << 0);			\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(x >> 0);	\
+    return x;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, sb)))(CPUState *env, \
+                                                  uint64_t x)   \
+{								\
+    x =							        \
+        ((uint64_t) EXTEND8H((x >> SH0) & 0xff) << 0) |	        \
+        ((uint64_t) EXTEND8H((x >> SH1) & 0xff) << 16) |	\
+        ((uint64_t) EXTEND8H((x >> SH2) & 0xff) << 32) |	\
+        ((uint64_t) EXTEND8H((x >> SH3) & 0xff) << 48);	        \
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT16(x >> 0, 0) | NZBIT16(x >> 16, 1) |		\
+        NZBIT16(x >> 32, 2) | NZBIT16(x >> 48, 3);		\
+    return x;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, sw)))(CPUState *env, \
+                                                  uint64_t x)   \
+{								\
+    x =							        \
+        ((uint64_t) EXTEND16((x >> SH0) & 0xffff) << 0) |	\
+        ((uint64_t) EXTEND16((x >> SH2) & 0xffff) << 32);	\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT32(x >> 0, 0) | NZBIT32(x >> 32, 1);		\
+    return x;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_unpack, glue(S, sl)))(CPUState *env, \
+                                                  uint64_t x)   \
+{								\
+    x = EXTEND32((x >> SH0) & 0xffffffff);			\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(x >> 0);	\
+    return x;                                                   \
+}
+IWMMXT_OP_UNPACK(l, 0, 8, 16, 24)
+IWMMXT_OP_UNPACK(h, 32, 40, 48, 56)
+
+#define IWMMXT_OP_CMP(SUFF, Tb, Tw, Tl, O)			\
+uint64_t HELPER(glue(iwmmxt_, glue(SUFF, b)))(CPUState *env,    \
+                                        uint64_t a, uint64_t b) \
+{								\
+    a =							        \
+        CMP(0, Tb, O, 0xff) | CMP(8, Tb, O, 0xff) |		\
+        CMP(16, Tb, O, 0xff) | CMP(24, Tb, O, 0xff) |		\
+        CMP(32, Tb, O, 0xff) | CMP(40, Tb, O, 0xff) |		\
+        CMP(48, Tb, O, 0xff) | CMP(56, Tb, O, 0xff);		\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT8(a >> 0, 0) | NZBIT8(a >> 8, 1) |		        \
+        NZBIT8(a >> 16, 2) | NZBIT8(a >> 24, 3) |		\
+        NZBIT8(a >> 32, 4) | NZBIT8(a >> 40, 5) |		\
+        NZBIT8(a >> 48, 6) | NZBIT8(a >> 56, 7);		\
+    return a;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_, glue(SUFF, w)))(CPUState *env,    \
+                                        uint64_t a, uint64_t b) \
+{								\
+    a = CMP(0, Tw, O, 0xffff) | CMP(16, Tw, O, 0xffff) |	\
+        CMP(32, Tw, O, 0xffff) | CMP(48, Tw, O, 0xffff);	\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT16(a >> 0, 0) | NZBIT16(a >> 16, 1) |		\
+        NZBIT16(a >> 32, 2) | NZBIT16(a >> 48, 3);		\
+    return a;                                                   \
+}								\
+uint64_t HELPER(glue(iwmmxt_, glue(SUFF, l)))(CPUState *env,    \
+                                        uint64_t a, uint64_t b) \
+{								\
+    a = CMP(0, Tl, O, 0xffffffff) |				\
+        CMP(32, Tl, O, 0xffffffff);				\
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
+        NZBIT32(a >> 0, 0) | NZBIT32(a >> 32, 1);		\
+    return a;                                                   \
+}
+#define CMP(SHR, TYPE, OPER, MASK) ((((TYPE) ((a >> SHR) & MASK) OPER \
+            (TYPE) ((b >> SHR) & MASK)) ? (uint64_t) MASK : 0) << SHR)
+IWMMXT_OP_CMP(cmpeq, uint8_t, uint16_t, uint32_t, ==)
+IWMMXT_OP_CMP(cmpgts, int8_t, int16_t, int32_t, >)
+IWMMXT_OP_CMP(cmpgtu, uint8_t, uint16_t, uint32_t, >)
+#undef CMP
+#define CMP(SHR, TYPE, OPER, MASK) ((((TYPE) ((a >> SHR) & MASK) OPER \
+            (TYPE) ((b >> SHR) & MASK)) ? a : b) & ((uint64_t) MASK << SHR))
+IWMMXT_OP_CMP(mins, int8_t, int16_t, int32_t, <)
+IWMMXT_OP_CMP(minu, uint8_t, uint16_t, uint32_t, <)
+IWMMXT_OP_CMP(maxs, int8_t, int16_t, int32_t, >)
+IWMMXT_OP_CMP(maxu, uint8_t, uint16_t, uint32_t, >)
+#undef CMP
+#define CMP(SHR, TYPE, OPER, MASK) ((uint64_t) (((TYPE) ((a >> SHR) & MASK) \
+            OPER (TYPE) ((b >> SHR) & MASK)) & MASK) << SHR)
+IWMMXT_OP_CMP(subn, uint8_t, uint16_t, uint32_t, -)
+IWMMXT_OP_CMP(addn, uint8_t, uint16_t, uint32_t, +)
+#undef CMP
+/* TODO Signed- and Unsigned-Saturation */
+#define CMP(SHR, TYPE, OPER, MASK) ((uint64_t) (((TYPE) ((a >> SHR) & MASK) \
+            OPER (TYPE) ((b >> SHR) & MASK)) & MASK) << SHR)
+IWMMXT_OP_CMP(subu, uint8_t, uint16_t, uint32_t, -)
+IWMMXT_OP_CMP(addu, uint8_t, uint16_t, uint32_t, +)
+IWMMXT_OP_CMP(subs, int8_t, int16_t, int32_t, -)
+IWMMXT_OP_CMP(adds, int8_t, int16_t, int32_t, +)
+#undef CMP
+#undef IWMMXT_OP_CMP
+
+#define AVGB(SHR) ((( \
+        ((a >> SHR) & 0xff) + ((b >> SHR) & 0xff) + round) >> 1) << SHR)
+#define IWMMXT_OP_AVGB(r)                                                 \
+uint64_t HELPER(iwmmxt_avgb##r)(CPUState *env, uint64_t a, uint64_t b)    \
+{                                                                         \
+    const int round = r;                                                  \
+    a = AVGB(0) | AVGB(8) | AVGB(16) | AVGB(24) |                         \
+        AVGB(32) | AVGB(40) | AVGB(48) | AVGB(56);                        \
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =                                 \
+        SIMD8_SET(ZBIT8((a >> 0) & 0xff), SIMD_ZBIT, 0) |                 \
+        SIMD8_SET(ZBIT8((a >> 8) & 0xff), SIMD_ZBIT, 1) |                 \
+        SIMD8_SET(ZBIT8((a >> 16) & 0xff), SIMD_ZBIT, 2) |                \
+        SIMD8_SET(ZBIT8((a >> 24) & 0xff), SIMD_ZBIT, 3) |                \
+        SIMD8_SET(ZBIT8((a >> 32) & 0xff), SIMD_ZBIT, 4) |                \
+        SIMD8_SET(ZBIT8((a >> 40) & 0xff), SIMD_ZBIT, 5) |                \
+        SIMD8_SET(ZBIT8((a >> 48) & 0xff), SIMD_ZBIT, 6) |                \
+        SIMD8_SET(ZBIT8((a >> 56) & 0xff), SIMD_ZBIT, 7);                 \
+    return a;                                                             \
+}
+IWMMXT_OP_AVGB(0)
+IWMMXT_OP_AVGB(1)
+#undef IWMMXT_OP_AVGB
+#undef AVGB
+
+#define AVGW(SHR) ((( \
+        ((a >> SHR) & 0xffff) + ((b >> SHR) & 0xffff) + round) >> 1) << SHR)
+#define IWMMXT_OP_AVGW(r)                                               \
+uint64_t HELPER(iwmmxt_avgw##r)(CPUState *env, uint64_t a, uint64_t b)  \
+{                                                                       \
+    const int round = r;                                                \
+    a = AVGW(0) | AVGW(16) | AVGW(32) | AVGW(48);                       \
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =                               \
+        SIMD16_SET(ZBIT16((a >> 0) & 0xffff), SIMD_ZBIT, 0) |           \
+        SIMD16_SET(ZBIT16((a >> 16) & 0xffff), SIMD_ZBIT, 1) |          \
+        SIMD16_SET(ZBIT16((a >> 32) & 0xffff), SIMD_ZBIT, 2) |          \
+        SIMD16_SET(ZBIT16((a >> 48) & 0xffff), SIMD_ZBIT, 3);           \
+    return a;                                                           \
+}
+IWMMXT_OP_AVGW(0)
+IWMMXT_OP_AVGW(1)
+#undef IWMMXT_OP_AVGW
+#undef AVGW
+
+uint64_t HELPER(iwmmxt_msadb)(uint64_t a, uint64_t b)
+{
+    a =  ((((a >> 0 ) & 0xffff) * ((b >> 0) & 0xffff) +
+           ((a >> 16) & 0xffff) * ((b >> 16) & 0xffff)) & 0xffffffff) |
+         ((((a >> 32) & 0xffff) * ((b >> 32) & 0xffff) +
+           ((a >> 48) & 0xffff) * ((b >> 48) & 0xffff)) << 32);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_align)(uint64_t a, uint64_t b, uint32_t n)
+{
+    a >>= n << 3;
+    a |= b << (64 - (n << 3));
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_insr)(uint64_t x, uint32_t a, uint32_t b, uint32_t n)
+{
+    x &= ~((uint64_t) b << n);
+    x |= (uint64_t) (a & b) << n;
+    return x;
+}
+
+uint32_t HELPER(iwmmxt_setpsr_nz)(uint64_t x)
+{
+    return SIMD64_SET((x == 0), SIMD_ZBIT) |
+           SIMD64_SET((x & (1ULL << 63)), SIMD_NBIT);
+}
+
+uint64_t HELPER(iwmmxt_bcstb)(uint32_t arg)
+{
+    arg &= 0xff;
+    return
+        ((uint64_t) arg << 0 ) | ((uint64_t) arg << 8 ) |
+        ((uint64_t) arg << 16) | ((uint64_t) arg << 24) |
+        ((uint64_t) arg << 32) | ((uint64_t) arg << 40) |
+        ((uint64_t) arg << 48) | ((uint64_t) arg << 56);
+}
+
+uint64_t HELPER(iwmmxt_bcstw)(uint32_t arg)
+{
+    arg &= 0xffff;
+    return
+        ((uint64_t) arg << 0 ) | ((uint64_t) arg << 16) |
+        ((uint64_t) arg << 32) | ((uint64_t) arg << 48);
+}
+
+uint64_t HELPER(iwmmxt_bcstl)(uint32_t arg)
+{
+    return arg | ((uint64_t) arg << 32);
+}
+
+uint64_t HELPER(iwmmxt_addcb)(uint64_t x)
+{
+    return
+        ((x >> 0) & 0xff) + ((x >> 8) & 0xff) +
+        ((x >> 16) & 0xff) + ((x >> 24) & 0xff) +
+        ((x >> 32) & 0xff) + ((x >> 40) & 0xff) +
+        ((x >> 48) & 0xff) + ((x >> 56) & 0xff);
+}
+
+uint64_t HELPER(iwmmxt_addcw)(uint64_t x)
+{
+    return
+        ((x >> 0) & 0xffff) + ((x >> 16) & 0xffff) +
+        ((x >> 32) & 0xffff) + ((x >> 48) & 0xffff);
+}
+
+uint64_t HELPER(iwmmxt_addcl)(uint64_t x)
+{
+    return (x & 0xffffffff) + (x >> 32);
+}
+
+uint32_t HELPER(iwmmxt_msbb)(uint64_t x)
+{
+    return
+        ((x >> 7) & 0x01) | ((x >> 14) & 0x02) |
+        ((x >> 21) & 0x04) | ((x >> 28) & 0x08) |
+        ((x >> 35) & 0x10) | ((x >> 42) & 0x20) |
+        ((x >> 49) & 0x40) | ((x >> 56) & 0x80);
+}
+
+uint32_t HELPER(iwmmxt_msbw)(uint64_t x)
+{
+    return
+        ((x >> 15) & 0x01) | ((x >> 30) & 0x02) |
+        ((x >> 45) & 0x04) | ((x >> 52) & 0x08);
+}
+
+uint32_t HELPER(iwmmxt_msbl)(uint64_t x)
+{
+    return ((x >> 31) & 0x01) | ((x >> 62) & 0x02);
+}
+
+/* FIXME: Split wCASF setting into a separate op to avoid env use.  */
+uint64_t HELPER(iwmmxt_srlw)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = (((x & (0xffffll << 0)) >> n) & (0xffffll << 0)) |
+        (((x & (0xffffll << 16)) >> n) & (0xffffll << 16)) |
+        (((x & (0xffffll << 32)) >> n) & (0xffffll << 32)) |
+        (((x & (0xffffll << 48)) >> n) & (0xffffll << 48));
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT16(x >> 0, 0) | NZBIT16(x >> 16, 1) |
+        NZBIT16(x >> 32, 2) | NZBIT16(x >> 48, 3);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_srll)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = ((x & (0xffffffffll << 0)) >> n) |
+        ((x >> n) & (0xffffffffll << 32));
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT32(x >> 0, 0) | NZBIT32(x >> 32, 1);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_srlq)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x >>= n;
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(x);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_sllw)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = (((x & (0xffffll << 0)) << n) & (0xffffll << 0)) |
+        (((x & (0xffffll << 16)) << n) & (0xffffll << 16)) |
+        (((x & (0xffffll << 32)) << n) & (0xffffll << 32)) |
+        (((x & (0xffffll << 48)) << n) & (0xffffll << 48));
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT16(x >> 0, 0) | NZBIT16(x >> 16, 1) |
+        NZBIT16(x >> 32, 2) | NZBIT16(x >> 48, 3);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_slll)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = ((x << n) & (0xffffffffll << 0)) |
+        ((x & (0xffffffffll << 32)) << n);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT32(x >> 0, 0) | NZBIT32(x >> 32, 1);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_sllq)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x <<= n;
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(x);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_sraw)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = ((uint64_t) ((EXTEND16(x >> 0) >> n) & 0xffff) << 0) |
+        ((uint64_t) ((EXTEND16(x >> 16) >> n) & 0xffff) << 16) |
+        ((uint64_t) ((EXTEND16(x >> 32) >> n) & 0xffff) << 32) |
+        ((uint64_t) ((EXTEND16(x >> 48) >> n) & 0xffff) << 48);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT16(x >> 0, 0) | NZBIT16(x >> 16, 1) |
+        NZBIT16(x >> 32, 2) | NZBIT16(x >> 48, 3);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_sral)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = (((EXTEND32(x >> 0) >> n) & 0xffffffff) << 0) |
+        (((EXTEND32(x >> 32) >> n) & 0xffffffff) << 32);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT32(x >> 0, 0) | NZBIT32(x >> 32, 1);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_sraq)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = (int64_t) x >> n;
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(x);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_rorw)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = ((((x & (0xffffll << 0)) >> n) |
+          ((x & (0xffffll << 0)) << (16 - n))) & (0xffffll << 0)) |
+        ((((x & (0xffffll << 16)) >> n) |
+          ((x & (0xffffll << 16)) << (16 - n))) & (0xffffll << 16)) |
+        ((((x & (0xffffll << 32)) >> n) |
+          ((x & (0xffffll << 32)) << (16 - n))) & (0xffffll << 32)) |
+        ((((x & (0xffffll << 48)) >> n) |
+          ((x & (0xffffll << 48)) << (16 - n))) & (0xffffll << 48));
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT16(x >> 0, 0) | NZBIT16(x >> 16, 1) |
+        NZBIT16(x >> 32, 2) | NZBIT16(x >> 48, 3);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_rorl)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = ((x & (0xffffffffll << 0)) >> n) |
+        ((x >> n) & (0xffffffffll << 32)) |
+        ((x << (32 - n)) & (0xffffffffll << 0)) |
+        ((x & (0xffffffffll << 32)) << (32 - n));
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT32(x >> 0, 0) | NZBIT32(x >> 32, 1);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_rorq)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = (x >> n) | (x << (64 - n));
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(x);
+    return x;
+}
+
+uint64_t HELPER(iwmmxt_shufh)(CPUState *env, uint64_t x, uint32_t n)
+{
+    x = (((x >> ((n << 4) & 0x30)) & 0xffff) << 0) |
+        (((x >> ((n << 2) & 0x30)) & 0xffff) << 16) |
+        (((x >> ((n << 0) & 0x30)) & 0xffff) << 32) |
+        (((x >> ((n >> 2) & 0x30)) & 0xffff) << 48);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT16(x >> 0, 0) | NZBIT16(x >> 16, 1) |
+        NZBIT16(x >> 32, 2) | NZBIT16(x >> 48, 3);
+    return x;
+}
+
+/* TODO: Unsigned-Saturation */
+uint64_t HELPER(iwmmxt_packuw)(CPUState *env, uint64_t a, uint64_t b)
+{
+    a = (((a >> 0) & 0xff) << 0) | (((a >> 16) & 0xff) << 8) |
+        (((a >> 32) & 0xff) << 16) | (((a >> 48) & 0xff) << 24) |
+        (((b >> 0) & 0xff) << 32) | (((b >> 16) & 0xff) << 40) |
+        (((b >> 32) & 0xff) << 48) | (((b >> 48) & 0xff) << 56);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT8(a >> 0, 0) | NZBIT8(a >> 8, 1) |
+        NZBIT8(a >> 16, 2) | NZBIT8(a >> 24, 3) |
+        NZBIT8(a >> 32, 4) | NZBIT8(a >> 40, 5) |
+        NZBIT8(a >> 48, 6) | NZBIT8(a >> 56, 7);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_packul)(CPUState *env, uint64_t a, uint64_t b)
+{
+    a = (((a >> 0) & 0xffff) << 0) | (((a >> 32) & 0xffff) << 16) |
+        (((b >> 0) & 0xffff) << 32) | (((b >> 32) & 0xffff) << 48);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT16(a >> 0, 0) | NZBIT16(a >> 16, 1) |
+        NZBIT16(a >> 32, 2) | NZBIT16(a >> 48, 3);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_packuq)(CPUState *env, uint64_t a, uint64_t b)
+{
+    a = (a & 0xffffffff) | ((b & 0xffffffff) << 32);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT32(a >> 0, 0) | NZBIT32(a >> 32, 1);
+    return a;
+}
+
+/* TODO: Signed-Saturation */
+uint64_t HELPER(iwmmxt_packsw)(CPUState *env, uint64_t a, uint64_t b)
+{
+    a = (((a >> 0) & 0xff) << 0) | (((a >> 16) & 0xff) << 8) |
+        (((a >> 32) & 0xff) << 16) | (((a >> 48) & 0xff) << 24) |
+        (((b >> 0) & 0xff) << 32) | (((b >> 16) & 0xff) << 40) |
+        (((b >> 32) & 0xff) << 48) | (((b >> 48) & 0xff) << 56);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT8(a >> 0, 0) | NZBIT8(a >> 8, 1) |
+        NZBIT8(a >> 16, 2) | NZBIT8(a >> 24, 3) |
+        NZBIT8(a >> 32, 4) | NZBIT8(a >> 40, 5) |
+        NZBIT8(a >> 48, 6) | NZBIT8(a >> 56, 7);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_packsl)(CPUState *env, uint64_t a, uint64_t b)
+{
+    a = (((a >> 0) & 0xffff) << 0) | (((a >> 32) & 0xffff) << 16) |
+        (((b >> 0) & 0xffff) << 32) | (((b >> 32) & 0xffff) << 48);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT16(a >> 0, 0) | NZBIT16(a >> 16, 1) |
+        NZBIT16(a >> 32, 2) | NZBIT16(a >> 48, 3);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_packsq)(CPUState *env, uint64_t a, uint64_t b)
+{
+    a = (a & 0xffffffff) | ((b & 0xffffffff) << 32);
+    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
+        NZBIT32(a >> 0, 0) | NZBIT32(a >> 32, 1);
+    return a;
+}
+
+uint64_t HELPER(iwmmxt_muladdsl)(uint64_t c, uint32_t a, uint32_t b)
+{
+    return c + ((int32_t) EXTEND32(a) * (int32_t) EXTEND32(b));
+}
+
+uint64_t HELPER(iwmmxt_muladdsw)(uint64_t c, uint32_t a, uint32_t b)
+{
+    c += EXTEND32(EXTEND16S((a >> 0) & 0xffff) *
+                  EXTEND16S((b >> 0) & 0xffff));
+    c += EXTEND32(EXTEND16S((a >> 16) & 0xffff) *
+                  EXTEND16S((b >> 16) & 0xffff));
+    return c;
+}
+
+uint64_t HELPER(iwmmxt_muladdswl)(uint64_t c, uint32_t a, uint32_t b)
+{
+    return c + (EXTEND32(EXTEND16S(a & 0xffff) *
+                         EXTEND16S(b & 0xffff)));
+}
diff --git a/target-arm/neon_helper.c b/target-arm/neon_helper.c
index c8ab2b4ac2..edc6e11ee2 100644
--- a/target-arm/neon_helper.c
+++ b/target-arm/neon_helper.c
@@ -1,3 +1,11 @@
+/*
+ * ARM NEON vector operations.
+ *
+ * Copyright (c) 2007, 2008 CodeSourcery.
+ * Written by Paul Brook
+ *
+ * This code is licenced under the GNU GPL v2.
+ */
 #include <stdlib.h>
 #include <stdio.h>
 
diff --git a/target-arm/op.c b/target-arm/op.c
index 3937de9a43..1175cf081a 100644
--- a/target-arm/op.c
+++ b/target-arm/op.c
@@ -20,17 +20,6 @@
  */
 #include "exec.h"
 
-/* memory access */
-
-#define MEMSUFFIX _raw
-#include "op_mem.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#define MEMSUFFIX _user
-#include "op_mem.h"
-#define MEMSUFFIX _kernel
-#include "op_mem.h"
-#endif
-
-/* iwMMXt support */
-#include "op_iwmmxt.c"
+void OPPROTO op_dummy(void)
+{
+}
diff --git a/target-arm/op_iwmmxt.c b/target-arm/op_iwmmxt.c
deleted file mode 100644
index 40ae10d297..0000000000
--- a/target-arm/op_iwmmxt.c
+++ /dev/null
@@ -1,707 +0,0 @@
-/*
- * iwMMXt micro operations for XScale.
- *
- * Copyright (c) 2007 OpenedHand, Ltd.
- * Written by Andrzej Zaborowski <andrew@openedhand.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-#define M1	env->iwmmxt.regs[PARAM1]
-
-/* iwMMXt macros extracted from GNU gdb.  */
-
-/* Set the SIMD wCASF flags for 8, 16, 32 or 64-bit operations.  */
-#define SIMD8_SET( v, n, b)	((v != 0) << ((((b) + 1) * 4) + (n)))
-#define SIMD16_SET(v, n, h)	((v != 0) << ((((h) + 1) * 8) + (n)))
-#define SIMD32_SET(v, n, w)	((v != 0) << ((((w) + 1) * 16) + (n)))
-#define SIMD64_SET(v, n)	((v != 0) << (32 + (n)))
-/* Flags to pass as "n" above.  */
-#define SIMD_NBIT	-1
-#define SIMD_ZBIT	-2
-#define SIMD_CBIT	-3
-#define SIMD_VBIT	-4
-/* Various status bit macros.  */
-#define NBIT8(x)	((x) & 0x80)
-#define NBIT16(x)	((x) & 0x8000)
-#define NBIT32(x)	((x) & 0x80000000)
-#define NBIT64(x)	((x) & 0x8000000000000000ULL)
-#define ZBIT8(x)	(((x) & 0xff) == 0)
-#define ZBIT16(x)	(((x) & 0xffff) == 0)
-#define ZBIT32(x)	(((x) & 0xffffffff) == 0)
-#define ZBIT64(x)	(x == 0)
-/* Sign extension macros.  */
-#define EXTEND8H(a)	((uint16_t) (int8_t) (a))
-#define EXTEND8(a)	((uint32_t) (int8_t) (a))
-#define EXTEND16(a)	((uint32_t) (int16_t) (a))
-#define EXTEND16S(a)	((int32_t) (int16_t) (a))
-#define EXTEND32(a)	((uint64_t) (int32_t) (a))
-
-void OPPROTO op_iwmmxt_movl_T0_T1_wRn(void)
-{
-    T0 = M1 & ~(uint32_t) 0;
-    T1 = M1 >> 32;
-}
-
-void OPPROTO op_iwmmxt_movl_wRn_T0_T1(void)
-{
-    M1 = ((uint64_t) T1 << 32) | T0;
-}
-
-void OPPROTO op_iwmmxt_movq_M0_wRn(void)
-{
-    M0 = M1;
-}
-
-void OPPROTO op_iwmmxt_orq_M0_wRn(void)
-{
-    M0 |= M1;
-}
-
-void OPPROTO op_iwmmxt_andq_M0_wRn(void)
-{
-    M0 &= M1;
-}
-
-void OPPROTO op_iwmmxt_xorq_M0_wRn(void)
-{
-    M0 ^= M1;
-}
-
-void OPPROTO op_iwmmxt_maddsq_M0_wRn(void)
-{
-    M0 = ((
-            EXTEND16S((M0 >> 0) & 0xffff) * EXTEND16S((M1 >> 0) & 0xffff) +
-            EXTEND16S((M0 >> 16) & 0xffff) * EXTEND16S((M1 >> 16) & 0xffff)
-        ) & 0xffffffff) | ((uint64_t) (
-            EXTEND16S((M0 >> 32) & 0xffff) * EXTEND16S((M1 >> 32) & 0xffff) +
-            EXTEND16S((M0 >> 48) & 0xffff) * EXTEND16S((M1 >> 48) & 0xffff)
-        ) << 32);
-}
-
-void OPPROTO op_iwmmxt_madduq_M0_wRn(void)
-{
-    M0 = ((
-            ((M0 >> 0) & 0xffff) * ((M1 >> 0) & 0xffff) +
-            ((M0 >> 16) & 0xffff) * ((M1 >> 16) & 0xffff)
-        ) & 0xffffffff) | ((
-            ((M0 >> 32) & 0xffff) * ((M1 >> 32) & 0xffff) +
-            ((M0 >> 48) & 0xffff) * ((M1 >> 48) & 0xffff)
-        ) << 32);
-}
-
-void OPPROTO op_iwmmxt_sadb_M0_wRn(void)
-{
-#define abs(x) (((x) >= 0) ? x : -x)
-#define SADB(SHR) abs((int) ((M0 >> SHR) & 0xff) - (int) ((M1 >> SHR) & 0xff))
-    M0 =
-        SADB(0) + SADB(8) + SADB(16) + SADB(24) +
-        SADB(32) + SADB(40) + SADB(48) + SADB(56);
-#undef SADB
-}
-
-void OPPROTO op_iwmmxt_sadw_M0_wRn(void)
-{
-#define SADW(SHR) \
-    abs((int) ((M0 >> SHR) & 0xffff) - (int) ((M1 >> SHR) & 0xffff))
-    M0 = SADW(0) + SADW(16) + SADW(32) + SADW(48);
-#undef SADW
-}
-
-void OPPROTO op_iwmmxt_addl_M0_wRn(void)
-{
-    M0 += env->iwmmxt.regs[PARAM1] & 0xffffffff;
-}
-
-void OPPROTO op_iwmmxt_mulsw_M0_wRn(void)
-{
-#define MULS(SHR) ((uint64_t) ((( \
-        EXTEND16S((M0 >> SHR) & 0xffff) * EXTEND16S((M1 >> SHR) & 0xffff) \
-    ) >> PARAM2) & 0xffff) << SHR)
-    M0 = MULS(0) | MULS(16) | MULS(32) | MULS(48);
-#undef MULS
-}
-
-void OPPROTO op_iwmmxt_muluw_M0_wRn(void)
-{
-#define MULU(SHR) ((uint64_t) ((( \
-        ((M0 >> SHR) & 0xffff) * ((M1 >> SHR) & 0xffff) \
-    ) >> PARAM2) & 0xffff) << SHR)
-    M0 = MULU(0) | MULU(16) | MULU(32) | MULU(48);
-#undef MULU
-}
-
-void OPPROTO op_iwmmxt_macsw_M0_wRn(void)
-{
-#define MACS(SHR) ( \
-        EXTEND16((M0 >> SHR) & 0xffff) * EXTEND16S((M1 >> SHR) & 0xffff))
-    M0 = (int64_t) (MACS(0) + MACS(16) + MACS(32) + MACS(48));
-#undef MACS
-}
-
-void OPPROTO op_iwmmxt_macuw_M0_wRn(void)
-{
-#define MACU(SHR) ( \
-        (uint32_t) ((M0 >> SHR) & 0xffff) * \
-        (uint32_t) ((M1 >> SHR) & 0xffff))
-    M0 = MACU(0) + MACU(16) + MACU(32) + MACU(48);
-#undef MACU
-}
-
-void OPPROTO op_iwmmxt_addsq_M0_wRn(void)
-{
-    M0 = (int64_t) M0 + (int64_t) M1;
-}
-
-void OPPROTO op_iwmmxt_adduq_M0_wRn(void)
-{
-    M0 += M1;
-}
-
-void OPPROTO op_iwmmxt_movq_wRn_M0(void)
-{
-    M1 = M0;
-}
-
-void OPPROTO op_iwmmxt_movl_wCx_T0(void)
-{
-    env->iwmmxt.cregs[PARAM1] = T0;
-}
-
-void OPPROTO op_iwmmxt_movl_T0_wCx(void)
-{
-    T0 = env->iwmmxt.cregs[PARAM1];
-}
-
-void OPPROTO op_iwmmxt_movl_T1_wCx(void)
-{
-    T1 = env->iwmmxt.cregs[PARAM1];
-}
-
-void OPPROTO op_iwmmxt_set_mup(void)
-{
-    env->iwmmxt.cregs[ARM_IWMMXT_wCon] |= 2;
-}
-
-void OPPROTO op_iwmmxt_set_cup(void)
-{
-    env->iwmmxt.cregs[ARM_IWMMXT_wCon] |= 1;
-}
-
-void OPPROTO op_iwmmxt_setpsr_nz(void)
-{
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        SIMD64_SET((M0 == 0), SIMD_ZBIT) |
-        SIMD64_SET((M0 & (1ULL << 63)), SIMD_NBIT);
-}
-
-void OPPROTO op_iwmmxt_negq_M0(void)
-{
-    M0 = ~M0;
-}
-
-#define NZBIT8(x, i) \
-    SIMD8_SET(NBIT8((x) & 0xff), SIMD_NBIT, i) | \
-    SIMD8_SET(ZBIT8((x) & 0xff), SIMD_ZBIT, i)
-#define NZBIT16(x, i) \
-    SIMD16_SET(NBIT16((x) & 0xffff), SIMD_NBIT, i) | \
-    SIMD16_SET(ZBIT16((x) & 0xffff), SIMD_ZBIT, i)
-#define NZBIT32(x, i) \
-    SIMD32_SET(NBIT32((x) & 0xffffffff), SIMD_NBIT, i) | \
-    SIMD32_SET(ZBIT32((x) & 0xffffffff), SIMD_ZBIT, i)
-#define NZBIT64(x) \
-    SIMD64_SET(NBIT64(x), SIMD_NBIT) | \
-    SIMD64_SET(ZBIT64(x), SIMD_ZBIT)
-#define IWMMXT_OP_UNPACK(S, SH0, SH1, SH2, SH3)			\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, b_M0_wRn))(void)	\
-{								\
-    M0 =							\
-        (((M0 >> SH0) & 0xff) << 0) | (((M1 >> SH0) & 0xff) << 8) |	\
-        (((M0 >> SH1) & 0xff) << 16) | (((M1 >> SH1) & 0xff) << 24) |	\
-        (((M0 >> SH2) & 0xff) << 32) | (((M1 >> SH2) & 0xff) << 40) |	\
-        (((M0 >> SH3) & 0xff) << 48) | (((M1 >> SH3) & 0xff) << 56);	\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT8(M0 >> 0, 0) | NZBIT8(M0 >> 8, 1) |		\
-        NZBIT8(M0 >> 16, 2) | NZBIT8(M0 >> 24, 3) |		\
-        NZBIT8(M0 >> 32, 4) | NZBIT8(M0 >> 40, 5) |		\
-        NZBIT8(M0 >> 48, 6) | NZBIT8(M0 >> 56, 7);		\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, w_M0_wRn))(void)	\
-{								\
-    M0 =							\
-        (((M0 >> SH0) & 0xffff) << 0) |				\
-        (((M1 >> SH0) & 0xffff) << 16) |			\
-        (((M0 >> SH2) & 0xffff) << 32) |			\
-        (((M1 >> SH2) & 0xffff) << 48);				\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT8(M0 >> 0, 0) | NZBIT8(M0 >> 16, 1) |		\
-        NZBIT8(M0 >> 32, 2) | NZBIT8(M0 >> 48, 3);		\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, l_M0_wRn))(void)	\
-{								\
-    M0 =							\
-        (((M0 >> SH0) & 0xffffffff) << 0) |			\
-        (((M1 >> SH0) & 0xffffffff) << 32);			\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);		\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, ub_M0))(void)	\
-{								\
-    M0 =							\
-        (((M0 >> SH0) & 0xff) << 0) |				\
-        (((M0 >> SH1) & 0xff) << 16) |				\
-        (((M0 >> SH2) & 0xff) << 32) |				\
-        (((M0 >> SH3) & 0xff) << 48);				\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |		\
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);		\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, uw_M0))(void)	\
-{								\
-    M0 =							\
-        (((M0 >> SH0) & 0xffff) << 0) |				\
-        (((M0 >> SH2) & 0xffff) << 32);				\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);		\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, ul_M0))(void)	\
-{								\
-    M0 = (((M0 >> SH0) & 0xffffffff) << 0);			\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(M0 >> 0);	\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, sb_M0))(void)	\
-{								\
-    M0 =							\
-        ((uint64_t) EXTEND8H((M0 >> SH0) & 0xff) << 0) |	\
-        ((uint64_t) EXTEND8H((M0 >> SH1) & 0xff) << 16) |	\
-        ((uint64_t) EXTEND8H((M0 >> SH2) & 0xff) << 32) |	\
-        ((uint64_t) EXTEND8H((M0 >> SH3) & 0xff) << 48);	\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |		\
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);		\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, sw_M0))(void)	\
-{								\
-    M0 =							\
-        ((uint64_t) EXTEND16((M0 >> SH0) & 0xffff) << 0) |	\
-        ((uint64_t) EXTEND16((M0 >> SH2) & 0xffff) << 32);	\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);		\
-}								\
-void OPPROTO glue(op_iwmmxt_unpack, glue(S, sl_M0))(void)	\
-{								\
-    M0 = EXTEND32((M0 >> SH0) & 0xffffffff);			\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(M0 >> 0);	\
-}
-IWMMXT_OP_UNPACK(l, 0, 8, 16, 24)
-IWMMXT_OP_UNPACK(h, 32, 40, 48, 56)
-
-#define IWMMXT_OP_CMP(SUFF, Tb, Tw, Tl, O)				\
-void OPPROTO glue(op_iwmmxt_, glue(SUFF, b_M0_wRn))(void)	\
-{								\
-    M0 =							\
-        CMP(0, Tb, O, 0xff) | CMP(8, Tb, O, 0xff) |		\
-        CMP(16, Tb, O, 0xff) | CMP(24, Tb, O, 0xff) |		\
-        CMP(32, Tb, O, 0xff) | CMP(40, Tb, O, 0xff) |		\
-        CMP(48, Tb, O, 0xff) | CMP(56, Tb, O, 0xff);		\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT8(M0 >> 0, 0) | NZBIT8(M0 >> 8, 1) |		\
-        NZBIT8(M0 >> 16, 2) | NZBIT8(M0 >> 24, 3) |		\
-        NZBIT8(M0 >> 32, 4) | NZBIT8(M0 >> 40, 5) |		\
-        NZBIT8(M0 >> 48, 6) | NZBIT8(M0 >> 56, 7);		\
-}								\
-void OPPROTO glue(op_iwmmxt_, glue(SUFF, w_M0_wRn))(void)	\
-{								\
-    M0 = CMP(0, Tw, O, 0xffff) | CMP(16, Tw, O, 0xffff) |	\
-        CMP(32, Tw, O, 0xffff) | CMP(48, Tw, O, 0xffff);	\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |		\
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);		\
-}								\
-void OPPROTO glue(op_iwmmxt_, glue(SUFF, l_M0_wRn))(void)	\
-{								\
-    M0 = CMP(0, Tl, O, 0xffffffff) |				\
-        CMP(32, Tl, O, 0xffffffff);				\
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =			\
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);		\
-}
-#define CMP(SHR, TYPE, OPER, MASK) ((((TYPE) ((M0 >> SHR) & MASK) OPER \
-            (TYPE) ((M1 >> SHR) & MASK)) ? (uint64_t) MASK : 0) << SHR)
-IWMMXT_OP_CMP(cmpeq, uint8_t, uint16_t, uint32_t, ==)
-IWMMXT_OP_CMP(cmpgts, int8_t, int16_t, int32_t, >)
-IWMMXT_OP_CMP(cmpgtu, uint8_t, uint16_t, uint32_t, >)
-#undef CMP
-#define CMP(SHR, TYPE, OPER, MASK) ((((TYPE) ((M0 >> SHR) & MASK) OPER \
-            (TYPE) ((M1 >> SHR) & MASK)) ? M0 : M1) & ((uint64_t) MASK << SHR))
-IWMMXT_OP_CMP(mins, int8_t, int16_t, int32_t, <)
-IWMMXT_OP_CMP(minu, uint8_t, uint16_t, uint32_t, <)
-IWMMXT_OP_CMP(maxs, int8_t, int16_t, int32_t, >)
-IWMMXT_OP_CMP(maxu, uint8_t, uint16_t, uint32_t, >)
-#undef CMP
-#define CMP(SHR, TYPE, OPER, MASK) ((uint64_t) (((TYPE) ((M0 >> SHR) & MASK) \
-            OPER (TYPE) ((M1 >> SHR) & MASK)) & MASK) << SHR)
-IWMMXT_OP_CMP(subn, uint8_t, uint16_t, uint32_t, -)
-IWMMXT_OP_CMP(addn, uint8_t, uint16_t, uint32_t, +)
-#undef CMP
-/* TODO Signed- and Unsigned-Saturation */
-#define CMP(SHR, TYPE, OPER, MASK) ((uint64_t) (((TYPE) ((M0 >> SHR) & MASK) \
-            OPER (TYPE) ((M1 >> SHR) & MASK)) & MASK) << SHR)
-IWMMXT_OP_CMP(subu, uint8_t, uint16_t, uint32_t, -)
-IWMMXT_OP_CMP(addu, uint8_t, uint16_t, uint32_t, +)
-IWMMXT_OP_CMP(subs, int8_t, int16_t, int32_t, -)
-IWMMXT_OP_CMP(adds, int8_t, int16_t, int32_t, +)
-#undef CMP
-#undef IWMMXT_OP_CMP
-
-void OPPROTO op_iwmmxt_avgb_M0_wRn(void)
-{
-#define AVGB(SHR) ((( \
-        ((M0 >> SHR) & 0xff) + ((M1 >> SHR) & 0xff) + PARAM2) >> 1) << SHR)
-    M0 =
-        AVGB(0) | AVGB(8) | AVGB(16) | AVGB(24) |
-        AVGB(32) | AVGB(40) | AVGB(48) | AVGB(56);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        SIMD8_SET(ZBIT8((M0 >> 0) & 0xff), SIMD_ZBIT, 0) |
-        SIMD8_SET(ZBIT8((M0 >> 8) & 0xff), SIMD_ZBIT, 1) |
-        SIMD8_SET(ZBIT8((M0 >> 16) & 0xff), SIMD_ZBIT, 2) |
-        SIMD8_SET(ZBIT8((M0 >> 24) & 0xff), SIMD_ZBIT, 3) |
-        SIMD8_SET(ZBIT8((M0 >> 32) & 0xff), SIMD_ZBIT, 4) |
-        SIMD8_SET(ZBIT8((M0 >> 40) & 0xff), SIMD_ZBIT, 5) |
-        SIMD8_SET(ZBIT8((M0 >> 48) & 0xff), SIMD_ZBIT, 6) |
-        SIMD8_SET(ZBIT8((M0 >> 56) & 0xff), SIMD_ZBIT, 7);
-#undef AVGB
-}
-
-void OPPROTO op_iwmmxt_avgw_M0_wRn(void)
-{
-#define AVGW(SHR) ((( \
-        ((M0 >> SHR) & 0xffff) + ((M1 >> SHR) & 0xffff) + PARAM2) >> 1) << SHR)
-    M0 = AVGW(0) | AVGW(16) | AVGW(32) | AVGW(48);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        SIMD16_SET(ZBIT16((M0 >> 0) & 0xffff), SIMD_ZBIT, 0) |
-        SIMD16_SET(ZBIT16((M0 >> 16) & 0xffff), SIMD_ZBIT, 1) |
-        SIMD16_SET(ZBIT16((M0 >> 32) & 0xffff), SIMD_ZBIT, 2) |
-        SIMD16_SET(ZBIT16((M0 >> 48) & 0xffff), SIMD_ZBIT, 3);
-#undef AVGW
-}
-
-void OPPROTO op_iwmmxt_msadb_M0_wRn(void)
-{
-    M0 = ((((M0 >> 0) & 0xffff) * ((M1 >> 0) & 0xffff) +
-           ((M0 >> 16) & 0xffff) * ((M1 >> 16) & 0xffff)) & 0xffffffff) |
-         ((((M0 >> 32) & 0xffff) * ((M1 >> 32) & 0xffff) +
-           ((M0 >> 48) & 0xffff) * ((M1 >> 48) & 0xffff)) << 32);
-}
-
-void OPPROTO op_iwmmxt_align_M0_T0_wRn(void)
-{
-    M0 >>= T0 << 3;
-    M0 |= M1 << (64 - (T0 << 3));
-}
-
-void OPPROTO op_iwmmxt_insr_M0_T0_T1(void)
-{
-    M0 &= ~((uint64_t) T1 << PARAM1);
-    M0 |= (uint64_t) (T0 & T1) << PARAM1;
-}
-
-void OPPROTO op_iwmmxt_extrsb_T0_M0(void)
-{
-    T0 = EXTEND8((M0 >> PARAM1) & 0xff);
-}
-
-void OPPROTO op_iwmmxt_extrsw_T0_M0(void)
-{
-    T0 = EXTEND16((M0 >> PARAM1) & 0xffff);
-}
-
-void OPPROTO op_iwmmxt_extru_T0_M0_T1(void)
-{
-    T0 = (M0 >> PARAM1) & T1;
-}
-
-void OPPROTO op_iwmmxt_bcstb_M0_T0(void)
-{
-    T0 &= 0xff;
-    M0 =
-        ((uint64_t) T0 << 0) | ((uint64_t) T0 << 8) |
-        ((uint64_t) T0 << 16) | ((uint64_t) T0 << 24) |
-        ((uint64_t) T0 << 32) | ((uint64_t) T0 << 40) |
-        ((uint64_t) T0 << 48) | ((uint64_t) T0 << 56);
-}
-
-void OPPROTO op_iwmmxt_bcstw_M0_T0(void)
-{
-    T0 &= 0xffff;
-    M0 =
-        ((uint64_t) T0 << 0) | ((uint64_t) T0 << 16) |
-        ((uint64_t) T0 << 32) | ((uint64_t) T0 << 48);
-}
-
-void OPPROTO op_iwmmxt_bcstl_M0_T0(void)
-{
-    M0 = ((uint64_t) T0 << 0) | ((uint64_t) T0 << 32);
-}
-
-void OPPROTO op_iwmmxt_addcb_M0(void)
-{
-    M0 =
-        ((M0 >> 0) & 0xff) + ((M0 >> 8) & 0xff) +
-        ((M0 >> 16) & 0xff) + ((M0 >> 24) & 0xff) +
-        ((M0 >> 32) & 0xff) + ((M0 >> 40) & 0xff) +
-        ((M0 >> 48) & 0xff) + ((M0 >> 56) & 0xff);
-}
-
-void OPPROTO op_iwmmxt_addcw_M0(void)
-{
-    M0 =
-        ((M0 >> 0) & 0xffff) + ((M0 >> 16) & 0xffff) +
-        ((M0 >> 32) & 0xffff) + ((M0 >> 48) & 0xffff);
-}
-
-void OPPROTO op_iwmmxt_addcl_M0(void)
-{
-    M0 = (M0 & 0xffffffff) + (M0 >> 32);
-}
-
-void OPPROTO op_iwmmxt_msbb_T0_M0(void)
-{
-    T0 =
-        ((M0 >> 7) & 0x01) | ((M0 >> 14) & 0x02) |
-        ((M0 >> 21) & 0x04) | ((M0 >> 28) & 0x08) |
-        ((M0 >> 35) & 0x10) | ((M0 >> 42) & 0x20) |
-        ((M0 >> 49) & 0x40) | ((M0 >> 56) & 0x80);
-}
-
-void OPPROTO op_iwmmxt_msbw_T0_M0(void)
-{
-    T0 =
-        ((M0 >> 15) & 0x01) | ((M0 >> 30) & 0x02) |
-        ((M0 >> 45) & 0x04) | ((M0 >> 52) & 0x08);
-}
-
-void OPPROTO op_iwmmxt_msbl_T0_M0(void)
-{
-    T0 = ((M0 >> 31) & 0x01) | ((M0 >> 62) & 0x02);
-}
-
-void OPPROTO op_iwmmxt_srlw_M0_T0(void)
-{
-    M0 =
-        (((M0 & (0xffffll << 0)) >> T0) & (0xffffll << 0)) |
-        (((M0 & (0xffffll << 16)) >> T0) & (0xffffll << 16)) |
-        (((M0 & (0xffffll << 32)) >> T0) & (0xffffll << 32)) |
-        (((M0 & (0xffffll << 48)) >> T0) & (0xffffll << 48));
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);
-}
-
-void OPPROTO op_iwmmxt_srll_M0_T0(void)
-{
-    M0 =
-        ((M0 & (0xffffffffll << 0)) >> T0) |
-        ((M0 >> T0) & (0xffffffffll << 32));
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);
-}
-
-void OPPROTO op_iwmmxt_srlq_M0_T0(void)
-{
-    M0 >>= T0;
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(M0);
-}
-
-void OPPROTO op_iwmmxt_sllw_M0_T0(void)
-{
-    M0 =
-        (((M0 & (0xffffll << 0)) << T0) & (0xffffll << 0)) |
-        (((M0 & (0xffffll << 16)) << T0) & (0xffffll << 16)) |
-        (((M0 & (0xffffll << 32)) << T0) & (0xffffll << 32)) |
-        (((M0 & (0xffffll << 48)) << T0) & (0xffffll << 48));
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);
-}
-
-void OPPROTO op_iwmmxt_slll_M0_T0(void)
-{
-    M0 =
-        ((M0 << T0) & (0xffffffffll << 0)) |
-        ((M0 & (0xffffffffll << 32)) << T0);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);
-}
-
-void OPPROTO op_iwmmxt_sllq_M0_T0(void)
-{
-    M0 <<= T0;
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(M0);
-}
-
-void OPPROTO op_iwmmxt_sraw_M0_T0(void)
-{
-    M0 =
-        ((uint64_t) ((EXTEND16(M0 >> 0) >> T0) & 0xffff) << 0) |
-        ((uint64_t) ((EXTEND16(M0 >> 16) >> T0) & 0xffff) << 16) |
-        ((uint64_t) ((EXTEND16(M0 >> 32) >> T0) & 0xffff) << 32) |
-        ((uint64_t) ((EXTEND16(M0 >> 48) >> T0) & 0xffff) << 48);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);
-}
-
-void OPPROTO op_iwmmxt_sral_M0_T0(void)
-{
-    M0 =
-        (((EXTEND32(M0 >> 0) >> T0) & 0xffffffff) << 0) |
-        (((EXTEND32(M0 >> 32) >> T0) & 0xffffffff) << 32);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);
-}
-
-void OPPROTO op_iwmmxt_sraq_M0_T0(void)
-{
-    M0 = (int64_t) M0 >> T0;
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(M0);
-}
-
-void OPPROTO op_iwmmxt_rorw_M0_T0(void)
-{
-    M0 =
-        ((((M0 & (0xffffll << 0)) >> T0) |
-          ((M0 & (0xffffll << 0)) << (16 - T0))) & (0xffffll << 0)) |
-        ((((M0 & (0xffffll << 16)) >> T0) |
-          ((M0 & (0xffffll << 16)) << (16 - T0))) & (0xffffll << 16)) |
-        ((((M0 & (0xffffll << 32)) >> T0) |
-          ((M0 & (0xffffll << 32)) << (16 - T0))) & (0xffffll << 32)) |
-        ((((M0 & (0xffffll << 48)) >> T0) |
-          ((M0 & (0xffffll << 48)) << (16 - T0))) & (0xffffll << 48));
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);
-}
-
-void OPPROTO op_iwmmxt_rorl_M0_T0(void)
-{
-    M0 =
-        ((M0 & (0xffffffffll << 0)) >> T0) |
-        ((M0 >> T0) & (0xffffffffll << 32)) |
-        ((M0 << (32 - T0)) & (0xffffffffll << 0)) |
-        ((M0 & (0xffffffffll << 32)) << (32 - T0));
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);
-}
-
-void OPPROTO op_iwmmxt_rorq_M0_T0(void)
-{
-    M0 = (M0 >> T0) | (M0 << (64 - T0));
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] = NZBIT64(M0);
-}
-
-void OPPROTO op_iwmmxt_shufh_M0_T0(void)
-{
-    M0 =
-        (((M0 >> ((T0 << 4) & 0x30)) & 0xffff) << 0) |
-        (((M0 >> ((T0 << 2) & 0x30)) & 0xffff) << 16) |
-        (((M0 >> ((T0 << 0) & 0x30)) & 0xffff) << 32) |
-        (((M0 >> ((T0 >> 2) & 0x30)) & 0xffff) << 48);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);
-}
-
-/* TODO: Unsigned-Saturation */
-void OPPROTO op_iwmmxt_packuw_M0_wRn(void)
-{
-    M0 =
-        (((M0 >> 0) & 0xff) << 0) | (((M0 >> 16) & 0xff) << 8) |
-        (((M0 >> 32) & 0xff) << 16) | (((M0 >> 48) & 0xff) << 24) |
-        (((M1 >> 0) & 0xff) << 32) | (((M1 >> 16) & 0xff) << 40) |
-        (((M1 >> 32) & 0xff) << 48) | (((M1 >> 48) & 0xff) << 56);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT8(M0 >> 0, 0) | NZBIT8(M0 >> 8, 1) |
-        NZBIT8(M0 >> 16, 2) | NZBIT8(M0 >> 24, 3) |
-        NZBIT8(M0 >> 32, 4) | NZBIT8(M0 >> 40, 5) |
-        NZBIT8(M0 >> 48, 6) | NZBIT8(M0 >> 56, 7);
-}
-
-void OPPROTO op_iwmmxt_packul_M0_wRn(void)
-{
-    M0 =
-        (((M0 >> 0) & 0xffff) << 0) | (((M0 >> 32) & 0xffff) << 16) |
-        (((M1 >> 0) & 0xffff) << 32) | (((M1 >> 32) & 0xffff) << 48);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);
-}
-
-void OPPROTO op_iwmmxt_packuq_M0_wRn(void)
-{
-    M0 = (M0 & 0xffffffff) | ((M1 & 0xffffffff) << 32);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);
-}
-
-/* TODO: Signed-Saturation */
-void OPPROTO op_iwmmxt_packsw_M0_wRn(void)
-{
-    M0 =
-        (((M0 >> 0) & 0xff) << 0) | (((M0 >> 16) & 0xff) << 8) |
-        (((M0 >> 32) & 0xff) << 16) | (((M0 >> 48) & 0xff) << 24) |
-        (((M1 >> 0) & 0xff) << 32) | (((M1 >> 16) & 0xff) << 40) |
-        (((M1 >> 32) & 0xff) << 48) | (((M1 >> 48) & 0xff) << 56);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT8(M0 >> 0, 0) | NZBIT8(M0 >> 8, 1) |
-        NZBIT8(M0 >> 16, 2) | NZBIT8(M0 >> 24, 3) |
-        NZBIT8(M0 >> 32, 4) | NZBIT8(M0 >> 40, 5) |
-        NZBIT8(M0 >> 48, 6) | NZBIT8(M0 >> 56, 7);
-}
-
-void OPPROTO op_iwmmxt_packsl_M0_wRn(void)
-{
-    M0 =
-        (((M0 >> 0) & 0xffff) << 0) | (((M0 >> 32) & 0xffff) << 16) |
-        (((M1 >> 0) & 0xffff) << 32) | (((M1 >> 32) & 0xffff) << 48);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT16(M0 >> 0, 0) | NZBIT16(M0 >> 16, 1) |
-        NZBIT16(M0 >> 32, 2) | NZBIT16(M0 >> 48, 3);
-}
-
-void OPPROTO op_iwmmxt_packsq_M0_wRn(void)
-{
-    M0 = (M0 & 0xffffffff) | ((M1 & 0xffffffff) << 32);
-    env->iwmmxt.cregs[ARM_IWMMXT_wCASF] =
-        NZBIT32(M0 >> 0, 0) | NZBIT32(M0 >> 32, 1);
-}
-
-void OPPROTO op_iwmmxt_muladdsl_M0_T0_T1(void)
-{
-    M0 += (int32_t) EXTEND32(T0) * (int32_t) EXTEND32(T1);
-}
-
-void OPPROTO op_iwmmxt_muladdsw_M0_T0_T1(void)
-{
-    M0 += EXTEND32(EXTEND16S((T0 >> 0) & 0xffff) *
-                   EXTEND16S((T1 >> 0) & 0xffff));
-    M0 += EXTEND32(EXTEND16S((T0 >> 16) & 0xffff) *
-                   EXTEND16S((T1 >> 16) & 0xffff));
-}
-
-void OPPROTO op_iwmmxt_muladdswl_M0_T0_T1(void)
-{
-    M0 += EXTEND32(EXTEND16S(T0 & 0xffff) *
-                   EXTEND16S(T1 & 0xffff));
-}
diff --git a/target-arm/op_mem.h b/target-arm/op_mem.h
deleted file mode 100644
index daaab51f86..0000000000
--- a/target-arm/op_mem.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* ARM memory operations.  */
-
-/* iwMMXt load/store.  Address is in T1 */
-#define MMX_MEM_OP(name, ldname) \
-void OPPROTO glue(op_iwmmxt_ld##name,MEMSUFFIX)(void) \
-{ \
-    M0 = glue(ld##ldname,MEMSUFFIX)(T1); \
-    FORCE_RET(); \
-} \
-void OPPROTO glue(op_iwmmxt_st##name,MEMSUFFIX)(void) \
-{ \
-    glue(st##name,MEMSUFFIX)(T1, M0); \
-    FORCE_RET(); \
-}
-
-MMX_MEM_OP(b, ub)
-MMX_MEM_OP(w, uw)
-MMX_MEM_OP(l, l)
-MMX_MEM_OP(q, q)
-
-#undef MMX_MEM_OP
-
-#undef MEMSUFFIX
diff --git a/target-arm/translate.c b/target-arm/translate.c
index 582dfa73be..fdba7ddccb 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -78,7 +78,7 @@ extern int loglevel;
 
 static TCGv cpu_env;
 /* We reuse the same 64-bit temporaries for efficiency.  */
-static TCGv cpu_V0, cpu_V1;
+static TCGv cpu_V0, cpu_V1, cpu_M0;
 
 /* FIXME:  These should be removed.  */
 static TCGv cpu_T[2];
@@ -456,6 +456,12 @@ static inline void tcg_gen_not_i32(TCGv t0, TCGv t1)
     tcg_gen_xori_i32(t0, t1, ~0);
 }
 
+/* FIXME:  Implement this natively.  */
+static inline void tcg_gen_neg_i64(TCGv dest, TCGv src)
+{
+    tcg_gen_sub_i64(dest, tcg_const_i64(0), src);
+}
+
 /* T0 &= ~T1.  Clobbers T1.  */
 /* FIXME: Implement bic natively.  */
 static inline void tcg_gen_bic_i32(TCGv dest, TCGv t0, TCGv t1)
@@ -1234,6 +1240,238 @@ static inline void gen_mov_vreg_F0(int dp, int reg)
 
 #define ARM_CP_RW_BIT	(1 << 20)
 
+static inline void iwmmxt_load_reg(TCGv var, int reg)
+{
+    tcg_gen_ld_i64(var, cpu_env, offsetof(CPUState, iwmmxt.regs[reg]));
+}
+
+static inline void iwmmxt_store_reg(TCGv var, int reg)
+{
+    tcg_gen_st_i64(var, cpu_env, offsetof(CPUState, iwmmxt.regs[reg]));
+}
+
+static inline void gen_op_iwmmxt_movl_wCx_T0(int reg)
+{
+    tcg_gen_st_i32(cpu_T[0], cpu_env, offsetof(CPUState, iwmmxt.cregs[reg]));
+}
+
+static inline void gen_op_iwmmxt_movl_T0_wCx(int reg)
+{
+    tcg_gen_ld_i32(cpu_T[0], cpu_env, offsetof(CPUState, iwmmxt.cregs[reg]));
+}
+
+static inline void gen_op_iwmmxt_movl_T1_wCx(int reg)
+{
+    tcg_gen_ld_i32(cpu_T[1], cpu_env, offsetof(CPUState, iwmmxt.cregs[reg]));
+}
+
+static inline void gen_op_iwmmxt_movq_wRn_M0(int rn)
+{
+    iwmmxt_store_reg(cpu_M0, rn);
+}
+
+static inline void gen_op_iwmmxt_movq_M0_wRn(int rn)
+{
+    iwmmxt_load_reg(cpu_M0, rn);
+}
+
+static inline void gen_op_iwmmxt_orq_M0_wRn(int rn)
+{
+    iwmmxt_load_reg(cpu_V1, rn);
+    tcg_gen_or_i64(cpu_M0, cpu_M0, cpu_V1);
+}
+
+static inline void gen_op_iwmmxt_andq_M0_wRn(int rn)
+{
+    iwmmxt_load_reg(cpu_V1, rn);
+    tcg_gen_and_i64(cpu_M0, cpu_M0, cpu_V1);
+}
+
+static inline void gen_op_iwmmxt_xorq_M0_wRn(int rn)
+{
+    iwmmxt_load_reg(cpu_V1, rn);
+    tcg_gen_xor_i64(cpu_M0, cpu_M0, cpu_V1);
+}
+
+#define IWMMXT_OP(name) \
+static inline void gen_op_iwmmxt_##name##_M0_wRn(int rn) \
+{ \
+    iwmmxt_load_reg(cpu_V1, rn); \
+    gen_helper_iwmmxt_##name(cpu_M0, cpu_M0, cpu_V1); \
+}
+
+#define IWMMXT_OP_ENV(name) \
+static inline void gen_op_iwmmxt_##name##_M0_wRn(int rn) \
+{ \
+    iwmmxt_load_reg(cpu_V1, rn); \
+    gen_helper_iwmmxt_##name(cpu_M0, cpu_env, cpu_M0, cpu_V1); \
+}
+
+#define IWMMXT_OP_ENV_SIZE(name) \
+IWMMXT_OP_ENV(name##b) \
+IWMMXT_OP_ENV(name##w) \
+IWMMXT_OP_ENV(name##l)
+
+#define IWMMXT_OP_ENV1(name) \
+static inline void gen_op_iwmmxt_##name##_M0(void) \
+{ \
+    gen_helper_iwmmxt_##name(cpu_M0, cpu_env, cpu_M0); \
+}
+
+IWMMXT_OP(maddsq)
+IWMMXT_OP(madduq)
+IWMMXT_OP(sadb)
+IWMMXT_OP(sadw)
+IWMMXT_OP(mulslw)
+IWMMXT_OP(mulshw)
+IWMMXT_OP(mululw)
+IWMMXT_OP(muluhw)
+IWMMXT_OP(macsw)
+IWMMXT_OP(macuw)
+
+IWMMXT_OP_ENV_SIZE(unpackl)
+IWMMXT_OP_ENV_SIZE(unpackh)
+
+IWMMXT_OP_ENV1(unpacklub)
+IWMMXT_OP_ENV1(unpackluw)
+IWMMXT_OP_ENV1(unpacklul)
+IWMMXT_OP_ENV1(unpackhub)
+IWMMXT_OP_ENV1(unpackhuw)
+IWMMXT_OP_ENV1(unpackhul)
+IWMMXT_OP_ENV1(unpacklsb)
+IWMMXT_OP_ENV1(unpacklsw)
+IWMMXT_OP_ENV1(unpacklsl)
+IWMMXT_OP_ENV1(unpackhsb)
+IWMMXT_OP_ENV1(unpackhsw)
+IWMMXT_OP_ENV1(unpackhsl)
+
+IWMMXT_OP_ENV_SIZE(cmpeq)
+IWMMXT_OP_ENV_SIZE(cmpgtu)
+IWMMXT_OP_ENV_SIZE(cmpgts)
+
+IWMMXT_OP_ENV_SIZE(mins)
+IWMMXT_OP_ENV_SIZE(minu)
+IWMMXT_OP_ENV_SIZE(maxs)
+IWMMXT_OP_ENV_SIZE(maxu)
+
+IWMMXT_OP_ENV_SIZE(subn)
+IWMMXT_OP_ENV_SIZE(addn)
+IWMMXT_OP_ENV_SIZE(subu)
+IWMMXT_OP_ENV_SIZE(addu)
+IWMMXT_OP_ENV_SIZE(subs)
+IWMMXT_OP_ENV_SIZE(adds)
+
+IWMMXT_OP_ENV(avgb0)
+IWMMXT_OP_ENV(avgb1)
+IWMMXT_OP_ENV(avgw0)
+IWMMXT_OP_ENV(avgw1)
+
+IWMMXT_OP(msadb)
+
+IWMMXT_OP_ENV(packuw)
+IWMMXT_OP_ENV(packul)
+IWMMXT_OP_ENV(packuq)
+IWMMXT_OP_ENV(packsw)
+IWMMXT_OP_ENV(packsl)
+IWMMXT_OP_ENV(packsq)
+
+static inline void gen_op_iwmmxt_muladdsl_M0_T0_T1(void)
+{
+    gen_helper_iwmmxt_muladdsl(cpu_M0, cpu_M0, cpu_T[0], cpu_T[1]);
+}
+
+static inline void gen_op_iwmmxt_muladdsw_M0_T0_T1(void)
+{
+    gen_helper_iwmmxt_muladdsw(cpu_M0, cpu_M0, cpu_T[0], cpu_T[1]);
+}
+
+static inline void gen_op_iwmmxt_muladdswl_M0_T0_T1(void)
+{
+    gen_helper_iwmmxt_muladdswl(cpu_M0, cpu_M0, cpu_T[0], cpu_T[1]);
+}
+
+static inline void gen_op_iwmmxt_align_M0_T0_wRn(int rn)
+{
+    iwmmxt_load_reg(cpu_V1, rn);
+    gen_helper_iwmmxt_align(cpu_M0, cpu_M0, cpu_V1, cpu_T[0]);
+}
+
+static inline void gen_op_iwmmxt_insr_M0_T0_T1(int shift)
+{
+    TCGv tmp = tcg_const_i32(shift);
+    gen_helper_iwmmxt_insr(cpu_M0, cpu_M0, cpu_T[0], cpu_T[1], tmp);
+}
+
+static inline void gen_op_iwmmxt_extrsb_T0_M0(int shift)
+{
+    tcg_gen_shri_i64(cpu_M0, cpu_M0, shift);
+    tcg_gen_trunc_i64_i32(cpu_T[0], cpu_M0);
+    tcg_gen_ext8s_i32(cpu_T[0], cpu_T[0]);
+}
+
+static inline void gen_op_iwmmxt_extrsw_T0_M0(int shift)
+{
+    tcg_gen_shri_i64(cpu_M0, cpu_M0, shift);
+    tcg_gen_trunc_i64_i32(cpu_T[0], cpu_M0);
+    tcg_gen_ext16s_i32(cpu_T[0], cpu_T[0]);
+}
+
+static inline void gen_op_iwmmxt_extru_T0_M0(int shift, uint32_t mask)
+{
+    tcg_gen_shri_i64(cpu_M0, cpu_M0, shift);
+    tcg_gen_trunc_i64_i32(cpu_T[0], cpu_M0);
+    if (mask != ~0u)
+        tcg_gen_andi_i32(cpu_T[0], cpu_T[0], mask);
+}
+
+static void gen_op_iwmmxt_set_mup(void)
+{
+    TCGv tmp;
+    tmp = load_cpu_field(iwmmxt.cregs[ARM_IWMMXT_wCon]);
+    tcg_gen_ori_i32(tmp, tmp, 2);
+    store_cpu_field(tmp, iwmmxt.cregs[ARM_IWMMXT_wCon]);
+}
+
+static void gen_op_iwmmxt_set_cup(void)
+{
+    TCGv tmp;
+    tmp = load_cpu_field(iwmmxt.cregs[ARM_IWMMXT_wCon]);
+    tcg_gen_ori_i32(tmp, tmp, 1);
+    store_cpu_field(tmp, iwmmxt.cregs[ARM_IWMMXT_wCon]);
+}
+
+static void gen_op_iwmmxt_setpsr_nz(void)
+{
+    TCGv tmp = new_tmp();
+    gen_helper_iwmmxt_setpsr_nz(tmp, cpu_M0);
+    store_cpu_field(tmp, iwmmxt.cregs[ARM_IWMMXT_wCASF]);
+}
+
+static inline void gen_op_iwmmxt_addl_M0_wRn(int rn)
+{
+    iwmmxt_load_reg(cpu_V1, rn);
+    tcg_gen_andi_i64(cpu_V1, cpu_V1, 0xffffffffu);
+    tcg_gen_add_i64(cpu_M0, cpu_M0, cpu_V1);
+}
+
+
+static void gen_iwmmxt_movl_T0_T1_wRn(int rn)
+{
+    iwmmxt_load_reg(cpu_V0, rn);
+    tcg_gen_trunc_i64_i32(cpu_T[0], cpu_V0);
+    tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
+    tcg_gen_trunc_i64_i32(cpu_T[1], cpu_V0);
+}
+
+static void gen_iwmmxt_movl_wRn_T0_T1(int rn)
+{
+    tcg_gen_extu_i32_i64(cpu_V0, cpu_T[0]);
+    tcg_gen_extu_i32_i64(cpu_V1, cpu_T[0]);
+    tcg_gen_shli_i64(cpu_V1, cpu_V1, 32);
+    tcg_gen_or_i64(cpu_V0, cpu_V0, cpu_V1);
+    iwmmxt_store_reg(cpu_V0, rn);
+}
+
 static inline int gen_iwmmxt_address(DisasContext *s, uint32_t insn)
 {
     int rd;
@@ -1275,7 +1513,7 @@ static inline int gen_iwmmxt_shift(uint32_t insn, uint32_t mask)
         else
             gen_op_iwmmxt_movl_T0_wCx(rd);
     else
-        gen_op_iwmmxt_movl_T0_T1_wRn(rd);
+        gen_iwmmxt_movl_T0_T1_wRn(rd);
 
     gen_op_movl_T1_im(mask);
     gen_op_andl_T0_T1();
@@ -1296,13 +1534,13 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
             rdlo = (insn >> 12) & 0xf;
             rdhi = (insn >> 16) & 0xf;
             if (insn & ARM_CP_RW_BIT) {			/* TMRRC */
-                gen_op_iwmmxt_movl_T0_T1_wRn(wrd);
+                gen_iwmmxt_movl_T0_T1_wRn(wrd);
                 gen_movl_reg_T0(s, rdlo);
                 gen_movl_reg_T1(s, rdhi);
             } else {					/* TMCRR */
                 gen_movl_T0_reg(s, rdlo);
                 gen_movl_T1_reg(s, rdhi);
-                gen_op_iwmmxt_movl_wRn_T0_T1(wrd);
+                gen_iwmmxt_movl_wRn_T0_T1(wrd);
                 gen_op_iwmmxt_set_mup();
             }
             return 0;
@@ -1318,16 +1556,25 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
                 dead_tmp(tmp);
                 gen_op_iwmmxt_movl_wCx_T0(wrd);
             } else {
-                if (insn & (1 << 8))
-                    if (insn & (1 << 22))		/* WLDRD */
-                        gen_ldst(iwmmxt_ldq, s);
-                    else				/* WLDRW wRd */
-                        gen_ldst(iwmmxt_ldl, s);
-                else
-                    if (insn & (1 << 22))		/* WLDRH */
-                        gen_ldst(iwmmxt_ldw, s);
-                    else				/* WLDRB */
-                        gen_ldst(iwmmxt_ldb, s);
+                i = 1;
+                if (insn & (1 << 8)) {
+                    if (insn & (1 << 22)) {		/* WLDRD */
+                        tcg_gen_qemu_ld64(cpu_M0, cpu_T[1], IS_USER(s));
+                        i = 0;
+                    } else {				/* WLDRW wRd */
+                        tmp = gen_ld32(cpu_T[1], IS_USER(s));
+                    }
+                } else {
+                    if (insn & (1 << 22)) {		/* WLDRH */
+                        tmp = gen_ld16u(cpu_T[1], IS_USER(s));
+                    } else {				/* WLDRB */
+                        tmp = gen_ld8u(cpu_T[1], IS_USER(s));
+                    }
+                }
+                if (i) {
+                    tcg_gen_extu_i32_i64(cpu_M0, tmp);
+                    dead_tmp(tmp);
+                }
                 gen_op_iwmmxt_movq_wRn_M0(wrd);
             }
         } else {
@@ -1338,16 +1585,24 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
                 gen_st32(tmp, cpu_T[1], IS_USER(s));
             } else {
                 gen_op_iwmmxt_movq_M0_wRn(wrd);
-                if (insn & (1 << 8))
-                    if (insn & (1 << 22))		/* WSTRD */
-                        gen_ldst(iwmmxt_stq, s);
-                    else				/* WSTRW wRd */
-                        gen_ldst(iwmmxt_stl, s);
-                else
-                    if (insn & (1 << 22))		/* WSTRH */
-                        gen_ldst(iwmmxt_ldw, s);
-                    else				/* WSTRB */
-                        gen_ldst(iwmmxt_stb, s);
+                tmp = new_tmp();
+                if (insn & (1 << 8)) {
+                    if (insn & (1 << 22)) {		/* WSTRD */
+                        dead_tmp(tmp);
+                        tcg_gen_qemu_st64(cpu_M0, cpu_T[1], IS_USER(s));
+                    } else {				/* WSTRW wRd */
+                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+                        gen_st32(tmp, cpu_T[1], IS_USER(s));
+                    }
+                } else {
+                    if (insn & (1 << 22)) {		/* WSTRH */
+                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+                        gen_st16(tmp, cpu_T[1], IS_USER(s));
+                    } else {				/* WSTRB */
+                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+                        gen_st8(tmp, cpu_T[1], IS_USER(s));
+                    }
+                }
             }
         }
         return 0;
@@ -1422,7 +1677,7 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         rd0 = (insn >> 0) & 0xf;
         rd1 = (insn >> 16) & 0xf;
         gen_op_iwmmxt_movq_M0_wRn(rd0);
-        gen_op_iwmmxt_negq_M0();
+        tcg_gen_neg_i64(cpu_M0, cpu_M0);
         gen_op_iwmmxt_andq_M0_wRn(rd1);
         gen_op_iwmmxt_setpsr_nz();
         gen_op_iwmmxt_movq_wRn_M0(wrd);
@@ -1515,10 +1770,17 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         rd0 = (insn >> 16) & 0xf;
         rd1 = (insn >> 0) & 0xf;
         gen_op_iwmmxt_movq_M0_wRn(rd0);
-        if (insn & (1 << 21))
-            gen_op_iwmmxt_mulsw_M0_wRn(rd1, (insn & (1 << 20)) ? 16 : 0);
-        else
-            gen_op_iwmmxt_muluw_M0_wRn(rd1, (insn & (1 << 20)) ? 16 : 0);
+        if (insn & (1 << 21)) {
+            if (insn & (1 << 20))
+                gen_op_iwmmxt_mulshw_M0_wRn(rd1);
+            else
+                gen_op_iwmmxt_mulslw_M0_wRn(rd1);
+        } else {
+            if (insn & (1 << 20))
+                gen_op_iwmmxt_muluhw_M0_wRn(rd1);
+            else
+                gen_op_iwmmxt_mululw_M0_wRn(rd1);
+        }
         gen_op_iwmmxt_movq_wRn_M0(wrd);
         gen_op_iwmmxt_set_mup();
         break;
@@ -1532,10 +1794,8 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         else
             gen_op_iwmmxt_macuw_M0_wRn(rd1);
         if (!(insn & (1 << 20))) {
-            if (insn & (1 << 21))
-                gen_op_iwmmxt_addsq_M0_wRn(wrd);
-            else
-                gen_op_iwmmxt_adduq_M0_wRn(wrd);
+            iwmmxt_load_reg(cpu_V1, wrd);
+            tcg_gen_add_i64(cpu_M0, cpu_M0, cpu_V1);
         }
         gen_op_iwmmxt_movq_wRn_M0(wrd);
         gen_op_iwmmxt_set_mup();
@@ -1567,10 +1827,17 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         rd0 = (insn >> 16) & 0xf;
         rd1 = (insn >> 0) & 0xf;
         gen_op_iwmmxt_movq_M0_wRn(rd0);
-        if (insn & (1 << 22))
-            gen_op_iwmmxt_avgw_M0_wRn(rd1, (insn >> 20) & 1);
-        else
-            gen_op_iwmmxt_avgb_M0_wRn(rd1, (insn >> 20) & 1);
+        if (insn & (1 << 22)) {
+            if (insn & (1 << 20))
+                gen_op_iwmmxt_avgw1_M0_wRn(rd1);
+            else
+                gen_op_iwmmxt_avgw0_M0_wRn(rd1);
+        } else {
+            if (insn & (1 << 20))
+                gen_op_iwmmxt_avgb1_M0_wRn(rd1);
+            else
+                gen_op_iwmmxt_avgb0_M0_wRn(rd1);
+        }
         gen_op_iwmmxt_movq_wRn_M0(wrd);
         gen_op_iwmmxt_set_mup();
         gen_op_iwmmxt_set_cup();
@@ -1622,21 +1889,18 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
             if (insn & 8)
                 gen_op_iwmmxt_extrsb_T0_M0((insn & 7) << 3);
             else {
-                gen_op_movl_T1_im(0xff);
-                gen_op_iwmmxt_extru_T0_M0_T1((insn & 7) << 3);
+                gen_op_iwmmxt_extru_T0_M0((insn & 7) << 3, 0xff);
             }
             break;
         case 1:
             if (insn & 8)
                 gen_op_iwmmxt_extrsw_T0_M0((insn & 3) << 4);
             else {
-                gen_op_movl_T1_im(0xffff);
-                gen_op_iwmmxt_extru_T0_M0_T1((insn & 3) << 4);
+                gen_op_iwmmxt_extru_T0_M0((insn & 3) << 4, 0xffff);
             }
             break;
         case 2:
-            gen_op_movl_T1_im(0xffffffff);
-            gen_op_iwmmxt_extru_T0_M0_T1((insn & 1) << 5);
+            gen_op_iwmmxt_extru_T0_M0((insn & 1) << 5, ~0u);
             break;
         case 3:
             return 1;
@@ -1669,13 +1933,13 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         gen_movl_T0_reg(s, rd);
         switch ((insn >> 6) & 3) {
         case 0:
-            gen_op_iwmmxt_bcstb_M0_T0();
+            gen_helper_iwmmxt_bcstb(cpu_M0, cpu_T[0]);
             break;
         case 1:
-            gen_op_iwmmxt_bcstw_M0_T0();
+            gen_helper_iwmmxt_bcstw(cpu_M0, cpu_T[0]);
             break;
         case 2:
-            gen_op_iwmmxt_bcstl_M0_T0();
+            gen_helper_iwmmxt_bcstl(cpu_M0, cpu_T[0]);
             break;
         case 3:
             return 1;
@@ -1715,13 +1979,13 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         gen_op_iwmmxt_movq_M0_wRn(rd0);
         switch ((insn >> 22) & 3) {
         case 0:
-            gen_op_iwmmxt_addcb_M0();
+            gen_helper_iwmmxt_addcb(cpu_M0, cpu_M0);
             break;
         case 1:
-            gen_op_iwmmxt_addcw_M0();
+            gen_helper_iwmmxt_addcw(cpu_M0, cpu_M0);
             break;
         case 2:
-            gen_op_iwmmxt_addcl_M0();
+            gen_helper_iwmmxt_addcl(cpu_M0, cpu_M0);
             break;
         case 3:
             return 1;
@@ -1763,13 +2027,13 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         gen_op_iwmmxt_movq_M0_wRn(rd0);
         switch ((insn >> 22) & 3) {
         case 0:
-            gen_op_iwmmxt_msbb_T0_M0();
+            gen_helper_iwmmxt_msbb(cpu_T[0], cpu_M0);
             break;
         case 1:
-            gen_op_iwmmxt_msbw_T0_M0();
+            gen_helper_iwmmxt_msbw(cpu_T[0], cpu_M0);
             break;
         case 2:
-            gen_op_iwmmxt_msbl_T0_M0();
+            gen_helper_iwmmxt_msbl(cpu_T[0], cpu_M0);
             break;
         case 3:
             return 1;
@@ -1881,13 +2145,13 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         case 0:
             return 1;
         case 1:
-            gen_op_iwmmxt_srlw_M0_T0();
+            gen_helper_iwmmxt_srlw(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 2:
-            gen_op_iwmmxt_srll_M0_T0();
+            gen_helper_iwmmxt_srll(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 3:
-            gen_op_iwmmxt_srlq_M0_T0();
+            gen_helper_iwmmxt_srlq(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         }
         gen_op_iwmmxt_movq_wRn_M0(wrd);
@@ -1905,13 +2169,13 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         case 0:
             return 1;
         case 1:
-            gen_op_iwmmxt_sraw_M0_T0();
+            gen_helper_iwmmxt_sraw(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 2:
-            gen_op_iwmmxt_sral_M0_T0();
+            gen_helper_iwmmxt_sral(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 3:
-            gen_op_iwmmxt_sraq_M0_T0();
+            gen_helper_iwmmxt_sraq(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         }
         gen_op_iwmmxt_movq_wRn_M0(wrd);
@@ -1929,13 +2193,13 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         case 0:
             return 1;
         case 1:
-            gen_op_iwmmxt_sllw_M0_T0();
+            gen_helper_iwmmxt_sllw(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 2:
-            gen_op_iwmmxt_slll_M0_T0();
+            gen_helper_iwmmxt_slll(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 3:
-            gen_op_iwmmxt_sllq_M0_T0();
+            gen_helper_iwmmxt_sllq(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         }
         gen_op_iwmmxt_movq_wRn_M0(wrd);
@@ -1953,17 +2217,17 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         case 1:
             if (gen_iwmmxt_shift(insn, 0xf))
                 return 1;
-            gen_op_iwmmxt_rorw_M0_T0();
+            gen_helper_iwmmxt_rorw(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 2:
             if (gen_iwmmxt_shift(insn, 0x1f))
                 return 1;
-            gen_op_iwmmxt_rorl_M0_T0();
+            gen_helper_iwmmxt_rorl(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         case 3:
             if (gen_iwmmxt_shift(insn, 0x3f))
                 return 1;
-            gen_op_iwmmxt_rorq_M0_T0();
+            gen_helper_iwmmxt_rorq(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
             break;
         }
         gen_op_iwmmxt_movq_wRn_M0(wrd);
@@ -2094,7 +2358,7 @@ static int disas_iwmmxt_insn(CPUState *env, DisasContext *s, uint32_t insn)
         rd0 = (insn >> 16) & 0xf;
         gen_op_iwmmxt_movq_M0_wRn(rd0);
         gen_op_movl_T0_im(((insn >> 16) & 0xf0) | (insn & 0x0f));
-        gen_op_iwmmxt_shufh_M0_T0();
+        gen_helper_iwmmxt_shufh(cpu_M0, cpu_env, cpu_M0, cpu_T[0]);
         gen_op_iwmmxt_movq_wRn_M0(wrd);
         gen_op_iwmmxt_set_mup();
         gen_op_iwmmxt_set_cup();
@@ -2279,7 +2543,7 @@ static int disas_dsp_insn(CPUState *env, DisasContext *s, uint32_t insn)
             return 1;
 
         if (insn & ARM_CP_RW_BIT) {			/* MRA */
-            gen_op_iwmmxt_movl_T0_T1_wRn(acc);
+            gen_iwmmxt_movl_T0_T1_wRn(acc);
             gen_movl_reg_T0(s, rdlo);
             gen_op_movl_T0_im((1 << (40 - 32)) - 1);
             gen_op_andl_T0_T1();
@@ -2287,7 +2551,7 @@ static int disas_dsp_insn(CPUState *env, DisasContext *s, uint32_t insn)
         } else {					/* MAR */
             gen_movl_T0_reg(s, rdlo);
             gen_movl_T1_reg(s, rdhi);
-            gen_op_iwmmxt_movl_wRn_T0_T1(acc);
+            gen_iwmmxt_movl_wRn_T0_T1(acc);
         }
         return 0;
     }
@@ -8322,6 +8586,8 @@ static inline int gen_intermediate_code_internal(CPUState *env,
     cpu_F1d = tcg_temp_new(TCG_TYPE_I64);
     cpu_V0 = cpu_F0d;
     cpu_V1 = cpu_F1d;
+    /* FIXME: cpu_M0 can probably be the same as cpu_V0.  */
+    cpu_M0 = tcg_temp_new(TCG_TYPE_I64);
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
     lj = -1;
     /* Reset the conditional execution bits immediately. This avoids