Context 0.1: Introducing SingleLookupHuffmanTable

This table provides best performance but may only be used reasonably for small bit
lengths due to its memory cost. We'll implement a more space-efficient (but not
as fast) MultiLookupHuffmanTable in a followup patch.

Author: Yoric

crates/binjs_io/src/context/huffman.rs renamed to crates/binjs_io/src/context/huffman/mod.rs

@@ -1,9 +1,13 @@
 use io::statistics::Instances;
 
+use std::borrow::Cow;
 use std::cmp::Ordering;
 use std::collections::{BinaryHeap, HashMap};
 use std::hash::Hash;
 
+/// Reading from bitstreams and decoding their contents using Huffman tables.
+pub mod read;
+
 /// A newtype for `u8` used to count the length of a key in bits.
 #[derive(
     Debug,
@@ -25,6 +29,11 @@ use std::hash::Hash;
     Eq,
 )]
 pub struct BitLen(u8);
+impl BitLen {
+    pub fn as_u8(&self) -> u8 {
+        self.0
+    }
+}
 
 /// Convenience implementation of operator `<<` in
 /// `bits << bit_len`
@@ -34,6 +43,12 @@ impl std::ops::Shl<BitLen> for u32 {
         self << Into::<u8>::into(rhs)
     }
 }
+impl std::ops::Shl<BitLen> for usize {
+    type Output = usize;
+    fn shl(self, rhs: BitLen) -> usize {
+        self << Into::<u8>::into(rhs)
+    }
+}
 
 /// Convenience implementation of operator `>>` in
 /// `bits >> bit_len`
@@ -43,56 +58,125 @@ impl std::ops::Shr<BitLen> for u32 {
         self >> Into::<u8>::into(rhs)
     }
 }
+impl std::ops::Shr<BitLen> for usize {
+    type Output = usize;
+    fn shr(self, rhs: BitLen) -> usize {
+        self >> Into::<u8>::into(rhs)
+    }
+}
 
 /// The largerst acceptable length for a key.
 ///
 /// Hardcoded in the format.
 const MAX_CODE_BIT_LENGTH: u8 = 20;
 
-// privacy barrier
-mod key {
-    use context::huffman::BitLen;
-
-/// A Huffman key
-#[derive(Debug)]
-pub struct Key {
-    /// The bits in the key.
-    ///
-    /// Note that we only use the `bit_len` lowest-weight bits.
-    /// Any other bit MUST BE 0.
+/// A sequence of bits, read from a bit stream.
+///
+/// Typically used for lookup of entries in Huffman tables.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct BitSequence {
     bits: u32,
-
-    /// The number of bits of `bits` to use.
     bit_len: BitLen,
 }
-impl Key {
-    /// Create a new Key.
+impl BitSequence {
     pub fn new(bits: u32, bit_len: BitLen) -> Self {
-        debug_assert!({let bit_len : u8 = bit_len.into(); bit_len <= 32});
-        debug_assert!({let bit_len : u8 = bit_len.into(); if bit_len < 32 { bits >> bit_len == 0 } else { true }});
-        Key {
-            bits,
-            bit_len,
+        Self { bits, bit_len }
+    }
+    pub fn bits(&self) -> u32 {
+        self.bits
+    }
+    /// The number of bits of `bits` to use.
+    pub fn bit_len(&self) -> BitLen {
+        self.bit_len
+    }
+    /// Split the bits into a prefix of `bit_len` bits and a suffix of `self.bit_len - bit_len`
+    /// bits.
+    ///
+    /// # Failure
+    ///
+    /// This function panics if `bit_len > self.bit_len`.
+    pub fn split(&self, bit_len: BitLen) -> (u32, u32) {
+        let shift = self.bit_len - bit_len;
+        match shift.into() {
+            0u8 => (self.bits, 0),  // Special case: cannot >> 32
+            32u8 => (0, self.bits), // Special case: cannot >> 32
+            shift => (
+                self.bits >> shift,
+                self.bits & (std::u32::MAX >> 32 - shift),
+            ),
         }
     }
+    pub fn pad_lowest_to(&self, total_bit_len: BitLen) -> Cow<BitSequence> {
+        assert!(total_bit_len.0 <= 32u8);
+        if total_bit_len <= self.bit_len {
+            return Cow::Borrowed(self);
+        }
+        let shift = total_bit_len - self.bit_len;
+        if shift.0 == 32u8 {
+            return Cow::Owned(BitSequence::new(0, BitLen(32)));
+        }
+        Cow::Owned(BitSequence::new(self.bits << shift, total_bit_len))
+    }
+}
 
-    /// The bits in the key.
+#[test]
+fn test_bit_sequence_split() {
+    let bits = 0b11111111_11111111_00000000_00000000;
+    let key = BitSequence::new(bits, BitLen(32));
+    assert_eq!(key.split(BitLen(0)), (0, bits));
+    assert_eq!(key.split(BitLen(32)), (bits, 0));
+    assert_eq!(key.split(BitLen(16)), (0b11111111_11111111, 0));
+
+    let bits = 0b00000000_00000000_00000000_11111111;
+    let key = BitSequence::new(bits, BitLen(16));
+    assert_eq!(key.split(BitLen(0)), (0, bits));
+    assert_eq!(key.split(BitLen(16)), (bits, 0));
+    assert_eq!(key.split(BitLen(8)), (0, 0b11111111));
+}
+
+/// A Huffman key
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct Key(BitSequence);
+
+impl Key {
+    /// Create a new Key.
     ///
     /// Note that we only use the `bit_len` lowest-weight bits.
-    /// Any other bit is guaranteed to be 0.
+    /// Any other bit MUST BE 0.
+    pub fn new(bits: u32, bit_len: BitLen) -> Self {
+        debug_assert!({
+            let bit_len: u8 = bit_len.into();
+            bit_len <= 32
+        });
+        debug_assert!({
+            let bit_len: u8 = bit_len.into();
+            if bit_len < 32 {
+                bits >> bit_len == 0
+            } else {
+                true
+            }
+        });
+        Key(BitSequence { bits, bit_len })
+    }
+
+    /// The bits in this Key.
+    ///
+    /// # Invariant
+    ///
+    /// Only the `self.bit_len()` lowest-weight bits may be non-0.
     pub fn bits(&self) -> u32 {
-        self.bits
+        self.0.bits
     }
 
     /// The number of bits of `bits` to use.
     pub fn bit_len(&self) -> BitLen {
-        self.bit_len
+        self.0.bit_len
     }
-}
-
-} // mod key
 
-use self::key::Key;
+    pub fn as_bit_sequence(&self) -> &BitSequence {
+        &self.0
+    }
+}
 
 /// A node in the Huffman tree.
 struct Node<T> {
@@ -136,17 +220,34 @@ impl<T> PartialEq for Node<T> {
 impl<T> Eq for Node<T> {}
 
 /// Keys associated to a sequence of values.
-#[derive(Debug)]
-pub struct Keys<T>
-where
-    T: Ord + Clone,
-{
+#[derive(Clone, Debug)]
+pub struct Keys<T> {
+    /// The longest bit length that actually appears in `keys`.
+    highest_bit_len: BitLen,
+
     /// The sequence of keys.
     ///
     /// Order is meaningful.
     keys: Vec<(T, Key)>,
 }
 
+impl<T> Keys<T> {
+    pub fn len(&self) -> usize {
+        self.keys.len()
+    }
+    pub fn highest_bit_len(&self) -> BitLen {
+        self.highest_bit_len
+    }
+}
+
+impl<T> IntoIterator for Keys<T> {
+    type Item = (T, Key);
+    type IntoIter = std::vec::IntoIter<(T, Key)>;
+    fn into_iter(self) -> Self::IntoIter {
+        self.keys.into_iter()
+    }
+}
+
 impl<T> Keys<T>
 where
     T: Ord + Clone,
@@ -155,12 +256,12 @@ where
     ///
     /// Optionally, `max_bit_len` may specify a largest acceptable bit length.
     /// If `Keys` may not be computed without exceeding this bit length,
-    /// fail with `Err(problemantic_bit_length)`.
+    /// fail with `Err(problemantic_bit_len)`.
     ///
     /// The current implementation only attempts to produce the best compression
-    /// level. This may cause us to exceed `max_bit_length` even though an
+    /// level. This may cause us to exceed `max_bit_len` even though an
     /// alternative table, with a lower compression level, would let us
-    /// proceed without exceeding `max_bit_length`.
+    /// proceed without exceeding `max_bit_len`.
     ///
     /// # Performance
     ///
@@ -185,9 +286,9 @@ where
     /// with a number of instances already attached.
     ///
     /// The current implementation only attempts to produce the best compression
-    /// level. This may cause us to exceed `max_bit_length` even though an
+    /// level. This may cause us to exceed `max_bit_len` even though an
     /// alternative table, with a lower compression level, would let us
-    /// proceed without exceeding `max_bit_length`.
+    /// proceed without exceeding `max_bit_len`.
     ///
     /// # Requirement
     ///
@@ -197,9 +298,9 @@ where
         S: IntoIterator<Item = (T, Instances)>,
     {
         let mut bit_lengths = Self::compute_bit_lengths(source, max_bit_len)?;
+        let mut highest_bit_len = BitLen(0);
 
         // Canonicalize order: (BitLen, T)
-        // As values of `T` are
         bit_lengths.sort_unstable_by_key(|&(ref value, ref bit_len)| (*bit_len, value.clone()));
 
         // The bits associated to the next value.
@@ -214,12 +315,18 @@ where
             );
             keys.push((symbol.clone(), Key::new(bits, bit_len)));
             bits = (bits + 1) << (next_bit_len - bit_len);
+            if bit_len > highest_bit_len {
+                highest_bit_len = bit_len;
+            }
         }
         // Handle the last element.
         let (ref symbol, bit_len) = bit_lengths[bit_lengths.len() - 1];
         keys.push((symbol.clone(), Key::new(bits, bit_len)));
 
-        return Ok(Self { keys });
+        return Ok(Self {
+            highest_bit_len,
+            keys,
+        });
     }
 
     /// Convert a sequence of values labelled by their number of instances