Support the free-threaded build

Author: ngoldbaum

.github/workflows/build_wheels.yml

@@ -17,12 +17,12 @@ jobs:
         # cibuildwheel builds linux wheels inside a manylinux container
         # it also takes care of procuring the correct python version for us
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: [39, 310, 311, 312, 313]
+        python-version: [39, 310, 311, 312, 313, 313t, 314, 314t]
 
     steps:
       - uses: actions/checkout@v4
 
-      - uses: pypa/cibuildwheel@v2.22.0
+      - uses: pypa/cibuildwheel@v3.1.4
         env:
           CIBW_BUILD: "cp${{ matrix.python-version}}-*"
 
@@ -39,7 +39,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04-arm]
-        python-version: [39, 310, 311, 312, 313]
+        python-version: [39, 310, 311, 312, 313, 313t, 314, 314t]
 
     steps:
       - uses: actions/checkout@v4

src/lib.rs

@@ -187,7 +187,7 @@ impl std::error::Error for EncodeError {}
 
 const MAX_NUM_THREADS: usize = 128;
 
-#[cfg_attr(feature = "python", pyclass)]
+#[cfg_attr(feature = "python", pyclass(frozen))]
 #[derive(Clone)]
 pub struct CoreBPE {
     encoder: HashMap<Vec<u8>, Rank>,

src/py.rs

@@ -183,7 +183,7 @@ impl CoreBPE {
     }
 }
 
-#[pyclass]
+#[pyclass(frozen)]
 struct TiktokenBuffer {
     tokens: Vec<Rank>,
 }
@@ -248,7 +248,7 @@ impl TiktokenBuffer {
     }
 }
 
-#[pymodule]
+#[pymodule(gil_used = false)]
 fn _tiktoken(_py: Python, m: &Bound<PyModule>) -> PyResult<()> {
     m.add_class::<CoreBPE>()?;
     Ok(())