refactor(es): use wtf8

crates/hstr/Cargo.toml

@@ -18,6 +18,7 @@ rkyv          = ["dep:rkyv"]
 serde         = ["dep:serde"]
 
 [dependencies]
+arrayvec              = { workspace = true }
 hashbrown             = { workspace = true }
 new_debug_unreachable = { workspace = true }
 once_cell             = { workspace = true }

crates/hstr/src/dynamic.rs

@@ -70,7 +70,12 @@ impl Default for AtomStore {
 impl AtomStore {
     #[inline(always)]
     pub fn atom<'a>(&mut self, text: impl Into<Cow<'a, str>>) -> Atom {
-        atom_in(self, &text.into())
+        self.atom_raw(text.into().as_bytes())
+    }
+
+    #[inline(always)]
+    pub fn atom_raw(&mut self, text: &[u8]) -> Atom {
+        atom_in(self, text)
     }
 
     fn gc(&mut self) {
@@ -94,7 +99,7 @@ pub fn global_atom_store_gc() {
     });
 }
 
-pub(crate) fn global_atom(text: &str) -> Atom {
+pub(crate) fn global_atom(text: &[u8]) -> Atom {
     GLOBAL_DATA.with(|global| {
         let mut store = global.borrow_mut();
 
@@ -104,7 +109,7 @@ pub(crate) fn global_atom(text: &str) -> Atom {
 
 /// This can create any kind of [Atom], although this lives in the `dynamic`
 /// module.
-fn atom_in<S>(storage: S, text: &str) -> Atom
+fn atom_in<S>(storage: S, text: &[u8]) -> Atom
 where
     S: Storage,
 {
@@ -115,7 +120,7 @@ where
         let tag = INLINE_TAG_INIT | ((len as u8) << LEN_OFFSET);
         let mut unsafe_data = TaggedValue::new_tag(tag);
         unsafe {
-            unsafe_data.data_mut()[..len].copy_from_slice(text.as_bytes());
+            unsafe_data.data_mut()[..len].copy_from_slice(text);
         }
         return Atom { unsafe_data };
     }
@@ -159,31 +164,25 @@ pub(crate) const fn inline_atom(text: &str) -> Option<Atom> {
 }
 
 trait Storage {
-    fn insert_entry(self, text: &str, hash: u64) -> Item;
+    fn insert_entry(self, text: &[u8], hash: u64) -> Item;
 }
 
 impl Storage for &'_ mut AtomStore {
-    fn insert_entry(self, text: &str, hash: u64) -> Item {
+    fn insert_entry(self, text: &[u8], hash: u64) -> Item {
         // If the text is too long, interning is not worth it.
         if text.len() > 512 {
-            return Item(ThinArc::from_header_and_slice(
-                Metadata { hash },
-                text.as_bytes(),
-            ));
+            return Item(ThinArc::from_header_and_slice(Metadata { hash }, text));
         }
 
         let (entry, _) = self
             .data
             .raw_entry_mut()
             .from_hash(hash, |key| {
-                key.header.header.hash == hash && key.slice.eq(text.as_bytes())
+                key.header.header.hash == hash && key.slice.eq(text)
             })
             .or_insert_with(move || {
                 (
-                    Item(ThinArc::from_header_and_slice(
-                        Metadata { hash },
-                        text.as_bytes(),
-                    )),
+                    Item(ThinArc::from_header_and_slice(Metadata { hash }, text)),
                     (),
                 )
             });
@@ -192,7 +191,7 @@ impl Storage for &'_ mut AtomStore {
 }
 
 #[inline(always)]
-fn calc_hash(text: &str) -> u64 {
+fn calc_hash(text: &[u8]) -> u64 {
     let mut hasher = FxHasher::default();
     text.hash(&mut hasher);
     hasher.finish()

crates/hstr/src/global_store.rs

@@ -6,7 +6,7 @@ macro_rules! direct_from_impl {
     ($T:ty) => {
         impl From<$T> for Atom {
             fn from(s: $T) -> Self {
-                global_atom(&s)
+                global_atom(s.as_bytes())
             }
         }
     };
@@ -18,6 +18,6 @@ direct_from_impl!(String);
 
 impl From<Box<str>> for crate::Atom {
     fn from(s: Box<str>) -> Self {
-        global_atom(&s)
+        global_atom(s.as_bytes())
     }
 }

crates/hstr/src/lib.rs

@@ -22,6 +22,7 @@ mod global_store;
 mod tagged_value;
 #[cfg(test)]
 mod tests;
+pub mod wtf8;
 
 /// An immutable string which is cheap to clone, compare, hash, and has small
 /// size.
@@ -249,6 +250,21 @@ impl Atom {
         }
         self.clone()
     }
+
+    pub fn as_wtf8_str(&self) -> &wtf8::Wtf8 {
+        match self.tag() {
+            DYNAMIC_TAG => unsafe {
+                let item = crate::dynamic::deref_from(self.unsafe_data);
+                wtf8::Wtf8::from_bytes(transmute::<&[u8], &'static [u8]>(&item.slice))
+            },
+            INLINE_TAG => {
+                let len = (self.unsafe_data.tag() & LEN_MASK) >> LEN_OFFSET;
+                let src = self.unsafe_data.data();
+                wtf8::Wtf8::from_bytes(&src[..(len as usize)])
+            }
+            _ => unsafe { debug_unreachable!() },
+        }
+    }
 }
 
 impl Atom {