Support CPU-local variables with user-defined types (#1024)

* The `cpu_local` macro now generates four functions for user-defined types:
```rust
pub fn replace(&self, value: #ty) -> #ty;

pub fn replace_guarded<G>(&self, mut value: #ty, guard: &G) -> #ty
where
    G: CpuAtomicGuard

pub fn set(&self, value: #ty);

pub fn set_guarded<G>(&self, mut value: #ty, guard: &G)
where
    G: CpuAtomicGuard
```

* The new `CpuAtomicGuard` trait is sealed, but implemented for
  `preemption::PreemptionGuard` and `irq_safety::HeldInterrupts`. 
  Passing a reference to one of those guard types into either `set_guarded()`
  or `replace_guarded()` can efficiently ensure that either preemption or
  interrupts are disabled, meaning that the CPU-local variable can be
  safely accessed in an atomic manner across multiple assembly instructions.

* However, this doesn't cover all use cases, so we also add two optional
  arguments to the `cpu_local` macro:
  1. `cls_dep`: a boolean that defaults to true, but can be set to false to
     prevent the macro from generating functions that depend on `cls`. 
     This is only used by the `preemption` crate to avoid a circular dependency.
  2. `stores_guard`: accepts a type denoting that the CLS variable stores an
     `Option<impl cls::CpuAtomicGuard>`, such as the task switch preemption 
     guard. This argument modifies the signatures of `replace` and `set` to accept
     the guard type by value, because the guard type obviates the need to pass in
     a reference to another redundant guard (normally used to ensure atomicity). 

Signed-off-by: Klimenty Tsoutsman <[email protected]>

Author: tsoutsman

Cargo.lock

@@ -7,3 +7,12 @@ edition = "2021"
 
 [dependencies]
 cls_macros = { path = "cls_macros" }
+irq_safety = { git = "https://github.com/theseus-os/irq_safety" }
+preemption = { path = "../preemption" }
+
+[target.'cfg(target_arch = "x86_64")'.dependencies]
+x86_64 = "0.14.8"
+
+[target.'cfg(target_arch = "aarch64")'.dependencies]
+tock-registers = "0.7.0"
+cortex-a = "7.5.0"

kernel/cls/Cargo.toml

@@ -0,0 +1,120 @@
+use proc_macro2::TokenStream;
+use quote::{quote, ToTokens};
+use syn::{LitInt, Type};
+
+pub(crate) fn int_functions(ty: Type, offset: LitInt) -> Option<TokenStream> {
+    let ((x64_asm_width, x64_reg_class), (aarch64_reg_modifier, aarch64_instr_width)) =
+        match ty.to_token_stream().to_string().as_ref() {
+            "u8" => (("byte", quote! { reg_byte }), (":w", "b")),
+            "u16" => (("word", quote! { reg }), (":w", "w")),
+            "u32" => (("dword", quote! { reg }), (":w", "")),
+            "u64" => (("qword", quote! { reg }), ("", "")),
+            _ => {
+                return None;
+            }
+        };
+    let x64_width_modifier = format!("{x64_asm_width} ptr ");
+    let x64_cls_location = format!("gs:[{offset}]");
+
+    Some(quote! {
+        #[inline]
+        pub fn load(&self) -> #ty {
+            #[cfg(target_arch = "x86_64")]
+            {
+                let ret;
+                unsafe {
+                    ::core::arch::asm!(
+                        ::core::concat!("mov {}, ", #x64_cls_location),
+                        out(#x64_reg_class) ret,
+                        options(preserves_flags, nostack),
+                    )
+                };
+                ret
+            }
+            #[cfg(target_arch = "aarch64")]
+            {
+                let ret;
+                unsafe {
+                    ::core::arch::asm!(
+                        "2:",
+                        // Load value.
+                        "mrs {tp_1}, tpidr_el1",
+                        concat!(
+                            "ldr", #aarch64_instr_width,
+                            " {ret", #aarch64_reg_modifier,"},",
+                            " [{tp_1},#", stringify!(#offset), "]",
+                        ),
+
+                        // Make sure task wasn't migrated between mrs and ldr.
+                        "mrs {tp_2}, tpidr_el1",
+                        "cmp {tp_1}, {tp_2}",
+                        "b.ne 2b",
+
+                        tp_1 = out(reg) _,
+                        ret = out(reg) ret,
+                        tp_2 = out(reg) _,
+
+                        options(nostack),
+                    )
+                };
+                ret
+            }
+        }
+
+        #[inline]
+        pub fn fetch_add(&self, mut operand: #ty) -> #ty {
+            #[cfg(target_arch = "x86_64")]
+            {
+                unsafe {
+                    ::core::arch::asm!(
+                        ::core::concat!("xadd ", #x64_width_modifier, #x64_cls_location, ", {}"),
+                        inout(#x64_reg_class) operand,
+                        options(nostack),
+                    )
+                };
+                operand
+            }
+            #[cfg(target_arch = "aarch64")]
+            {
+                let ret;
+                unsafe {
+                    ::core::arch::asm!(
+                        "2:",
+                        // Load value.
+                        "mrs {tp_1}, tpidr_el1",
+                        concat!("add {ptr}, {tp_1}, ", stringify!(#offset)),
+                        concat!("ldxr", #aarch64_instr_width, " {value", #aarch64_reg_modifier,"}, [{ptr}]"),
+
+                        // Make sure task wasn't migrated between msr and ldxr.
+                        "mrs {tp_2}, tpidr_el1",
+                        "cmp {tp_1}, {tp_2}",
+                        "b.ne 2b",
+
+                        // Compute and store value (reuse tp_1 register).
+                        "add {tp_1}, {value}, {operand}",
+                        concat!("stxr", #aarch64_instr_width, " {cond:w}, {tp_1", #aarch64_reg_modifier,"}, [{ptr}]"),
+
+                        // Make sure task wasn't migrated between ldxr and stxr.
+                        "cbnz {cond}, 2b",
+
+                        tp_1 = out(reg) ret,
+                        ptr = out(reg) _,
+                        value = out(reg) ret,
+                        tp_2 = out(reg) _,
+                        operand = in(reg) operand,
+                        cond = out(reg) _,
+
+                        options(nostack),
+                    )
+                };
+                ret
+            }
+        }
+
+        #[inline]
+        pub fn fetch_sub(&self, mut operand: #ty) -> #ty {
+            operand = operand.overflowing_neg().0;
+            self.fetch_add(operand)
+        }
+    })
+}