chore(deps): update tokenizers requirement from 0.19.1 to 0.20.1

ModelTC · Oct 18, 2024 · a499b78 · a499b78
1 parent 214c86b
commit a499b78
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/Cargo.toml b/Cargo.toml
@@ -26,7 +26,7 @@ rand = { version = "0.8.5", optional = true }
 [dev-dependencies]
 criterion = "0.5.1"
 rand = "0.8.5"
-tokenizers = "0.19.1"
+tokenizers = "0.20.1"
 
 [[bench]]
 name = "tokenize"