x64: Add EVEX shifts to the new assembler (bytecodealliance#11270)

* x64: Add EVEX shifts to the new assembler

This commit adds bindings for the EVEX encodings of `vps{ll,ra,rl}{d,q}`
to the new assembler. Currently the 16-bit shifts `vps{ll,ra,rl}w` are
omitted due to the `avx512bw` feature not yet being bound in Cranelift.
In implementing these instructions a few refactorings/fixes were
necessary:

* Primarily all EVEX instructions now need to be defined not only with
  their vector length but also their "tuple type" found in encoding
  tables. This is required to correctly handle the 8-bit displacement
  scaling that happens with EVEX instructions.
* Some small helpers to the `Evex` structure were added such as
  `Evex::digit` and `Evex::ib`.
* The `evex_scaling` factor is now calculated in `generate_evex_prefix`
  according to the instruction format itself.
* The VEX and EVEX `generate_*_prefix` functions now delegate to a
  shared function to handle the same operand formats across both of
  them.
* Fuzz generation of `AmodeOffset` is now updated to bias to some
  "interesting" offsets that exercise the cases where EVEX scaling is
  necessary.
* The ISLE `XmmUnaryRmRImmEvex` instruction format was removed as it's
  no longer necessary.

* Fix emit tests

Author: alexcrichton

cranelift/assembler-x64/meta/src/dsl.rs

@@ -13,7 +13,7 @@ pub use custom::{Custom, Customization};
 pub use encoding::{Encoding, ModRmKind, OpcodeMod};
 pub use encoding::{Evex, Length, Vex, VexEscape, VexPrefix, evex, vex};
 pub use encoding::{
-    Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, rex,
+    Group1Prefix, Group2Prefix, Group3Prefix, Group4Prefix, Opcodes, Prefixes, Rex, TupleType, rex,
 };
 pub use features::{ALL_FEATURES, Feature, Features};
 pub use format::{Eflags, Extension, Format, Location, Mutability, Operand, OperandKind, RegClass};

cranelift/assembler-x64/meta/src/dsl/encoding.rs

@@ -47,7 +47,7 @@ pub fn vex(length: Length) -> Vex {
 
 /// An abbreviated constructor for EVEX-encoded instructions.
 #[must_use]
-pub fn evex(length: Length) -> Evex {
+pub fn evex(length: Length, tuple_type: TupleType) -> Evex {
     Evex {
         length,
         pp: None,
@@ -56,6 +56,7 @@ pub fn evex(length: Length) -> Evex {
         opcode: u8::MAX,
         modrm: None,
         imm: Imm::None,
+        tuple_type,
     }
 }
 
@@ -1243,6 +1244,9 @@ pub struct Evex {
     pub modrm: Option<ModRmKind>,
     /// See [`Rex.imm`](Rex.imm).
     pub imm: Imm,
+    /// The "Tuple Type" corresponding to scaling of the 8-bit displacement
+    /// parameter for memory operands. See [`TupleType`] for more information.
+    pub tuple_type: TupleType,
 }
 
 impl Evex {
@@ -1361,6 +1365,36 @@ impl Evex {
             _ => None,
         }
     }
+
+    /// Set the digit extending the opcode; equivalent to `/<digit>` in the
+    /// reference manual.
+    ///
+    /// # Panics
+    ///
+    /// Panics if `extension` is too large.
+    #[must_use]
+    pub fn digit(self, extension: u8) -> Self {
+        assert!(extension <= 0b111, "must fit in 3 bits");
+        Self {
+            modrm: Some(ModRmKind::Digit(extension)),
+            ..self
+        }
+    }
+
+    /// Append a byte-sized immediate operand (8-bit); equivalent to `ib` in the
+    /// reference manual.
+    ///
+    /// # Panics
+    ///
+    /// Panics if an immediate operand is already set.
+    #[must_use]
+    pub fn ib(self) -> Self {
+        assert_eq!(self.imm, Imm::None);
+        Self {
+            imm: Imm::ib,
+            ..self
+        }
+    }
 }
 
 impl From<Evex> for Encoding {
@@ -1388,3 +1422,25 @@ impl fmt::Display for Evex {
         Ok(())
     }
 }
+
+/// Tuple Type definitions used in EVEX encodings.
+///
+/// This enumeration corresponds to table 2-34 and 2-35 in the Intel manual.
+/// This is a property of all instruction formats listed in the encoding table
+/// for each instruction.
+#[expect(missing_docs, reason = "matching manual names")]
+pub enum TupleType {
+    Full,
+    Half,
+    FullMem,
+    Tuple1Scalar,
+    Tuple1Fixed,
+    Tuple2,
+    Tuple4,
+    Tuple8,
+    HalfMem,
+    QuarterMem,
+    EigthMem,
+    Mem128,
+    Movddup,
+}