Use pyo3 smd v0.21

bindings/python/Cargo.toml

@@ -19,15 +19,15 @@ ndarray = "0.15"
 onig = { version = "6.4", default-features = false }
 itertools = "0.12"
 derive_more = "0.99.17"
-pyo3 = { version = "0.22", features = ["multiple-pymethods"] }
-pyo3_special_method_derive =  "0.4"
+pyo3 = { version = "0.21", features = ["multiple-pymethods"] }
+pyo3_special_method_derive_0_21 =  "0.4"
 
 [dependencies.tokenizers]
 path = "../../tokenizers"
 
 [dev-dependencies]
 tempfile = "3.10"
-pyo3 = { version = "0.22", features = ["auto-initialize"] }
+pyo3 = { version = "0.21", features = ["auto-initialize"] }
 
 [features]
 defaut = ["pyo3/extension-module"]

bindings/python/src/decoders.rs

@@ -2,10 +2,12 @@ use std::sync::{Arc, RwLock};
 
 use crate::pre_tokenizers::from_string;
 use crate::utils::PyPattern;
-use pyo3_special_method_derive::AutoDisplay;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
+use pyo3_special_method_derive_0_21::AutoDisplay;
+use pyo3_special_method_derive_0_21::PyDebug;
+use pyo3_special_method_derive_0_21::PyDisplay;
 use serde::de::Error;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use tk::decoders::bpe::BPEDecoder;
@@ -487,11 +489,23 @@ impl PySequenceDecoder {
     }
 }
 
-#[derive(Clone, AutoDisplay)]
+#[derive(Clone)]
 pub(crate) struct CustomDecoder {
     pub inner: PyObject,
 }
 
+impl PyDisplay for CustomDecoder {
+    fn fmt_display(&self) -> String {
+        "CustomDecoder()".to_string()
+    }
+}
+
+impl PyDebug for CustomDecoder {
+    fn fmt_debug(&self) -> String {
+        "CustomDecoder()".to_string()
+    }
+}
+
 impl CustomDecoder {
     pub(crate) fn new(inner: PyObject) -> Self {
         CustomDecoder { inner }

bindings/python/src/models.rs

@@ -5,10 +5,10 @@ use std::sync::{Arc, RwLock};
 use super::error::{deprecation_warning, ToPyResult};
 use crate::token::PyToken;
 use crate::trainers::PyTrainer;
-use pyo3_special_method_derive::AutoDisplay;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 use tk::models::bpe::{BpeBuilder, Merges, Vocab, BPE};
 use tk::models::unigram::Unigram;

bindings/python/src/normalizers.rs

@@ -2,10 +2,10 @@ use std::sync::{Arc, RwLock};
 
 use crate::error::ToPyResult;
 use crate::utils::{PyNormalizedString, PyNormalizedStringRefMut, PyPattern};
-use pyo3_special_method_derive::AutoDisplay;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use tk::normalizers::{

bindings/python/src/pre_tokenizers.rs

@@ -23,7 +23,7 @@ use tokenizers as tk;
 
 use super::error::ToPyResult;
 use super::utils::*;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::{AutoDisplay, Dict, Dir, Repr, Str};
 /// Base class for all pre-tokenizers
 ///
 /// This class is not supposed to be instantiated directly. Instead, any implementation of a
@@ -181,14 +181,6 @@ impl PyPreTokenizer {
             .map(|(s, o, _)| (s.to_owned(), o))
             .collect())
     }
-
-    fn __str__(&self) -> PyResult<String> {
-        Ok(format!("{}", self.pretok))
-    }
-
-    fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("{}", self.pretok))
-    }
 }
 
 macro_rules! getter {

bindings/python/src/processors.rs

@@ -3,10 +3,10 @@ use std::sync::Arc;
 
 use crate::encoding::PyEncoding;
 use crate::error::ToPyResult;
-use pyo3_special_method_derive::AutoDisplay;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 use tk::processors::bert::BertProcessing;
 use tk::processors::byte_level::ByteLevel;

bindings/python/src/tokenizer.rs

@@ -10,13 +10,13 @@ use super::pre_tokenizers::PyPreTokenizer;
 use super::trainers::PyTrainer;
 use crate::processors::PyPostProcessor;
 use crate::utils::{MaybeSizedIterator, PyBufferedIterator};
-use pyo3_special_method_derive::AutoDisplay;
 use numpy::{npyffi, PyArray1};
 use pyo3::class::basic::CompareOp;
 use pyo3::exceptions;
 use pyo3::intern;
 use pyo3::prelude::*;
 use pyo3::types::*;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use std::collections::BTreeMap;
 use tk::models::bpe::BPE;
 use tk::tokenizer::{
@@ -1409,14 +1409,6 @@ impl PyTokenizer {
     fn set_decoder(&mut self, decoder: PyRef<PyDecoder>) {
         self.tokenizer.with_decoder(decoder.clone());
     }
-
-    fn __str__(&self) -> PyResult<String> {
-        Ok(format!("{}", self.tokenizer))
-    }
-
-    fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("{}", self.tokenizer))
-    }
 }
 
 #[cfg(test)]

tokenizers/Cargo.toml

@@ -63,7 +63,7 @@ fancy-regex = { version = "0.13", optional = true}
 getrandom = { version = "0.2.10" }
 esaxx-rs = { version = "0.1.10", default-features = false, features=[]}
 monostate = "0.1.12"
-pyo3_special_method_derive = "0.4"
+pyo3_special_method_derive_0_21 = "0.4"
 
 [features]
 default = ["progressbar", "onig", "esaxx_fast"]

tokenizers/src/decoders/bpe.rs

@@ -1,5 +1,5 @@
 use crate::tokenizer::{Decoder, Result};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 #[derive(Deserialize, Clone, Debug, Serialize, AutoDisplay)]
 /// Allows decoding Original BPE by joining all the tokens and then replacing

tokenizers/src/decoders/byte_fallback.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{Decoder, Result};
 use monostate::MustBe;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 #[derive(Deserialize, Clone, Debug, Serialize, Default, AutoDisplay)]
 /// ByteFallback is a simple trick which converts tokens looking like `<0x61>`

tokenizers/src/decoders/ctc.rs

@@ -1,7 +1,7 @@
 use crate::decoders::wordpiece;
 use crate::tokenizer::{Decoder, Result};
 use itertools::Itertools;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 #[derive(Debug, Clone, Serialize, Deserialize, AutoDisplay)]

tokenizers/src/decoders/fuse.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{Decoder, Result};
 use monostate::MustBe;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 #[derive(Clone, Debug, Serialize, Deserialize, Default, AutoDisplay)]
 /// Fuse simply fuses all tokens into one big string.

tokenizers/src/decoders/mod.rs

@@ -21,10 +21,10 @@
 use crate::pre_tokenizers::byte_level::ByteLevel;
 use crate::pre_tokenizers::metaspace::Metaspace;
 use crate::{Decoder, Result};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Clone, Debug, AutoDisplay)]
 #[auto_display(fmt="decoders.{}")]
 #[serde(untagged)]
 pub enum DecoderWrapper {

tokenizers/src/decoders/sequence.rs

@@ -1,7 +1,7 @@
 use crate::decoders::DecoderWrapper;
 use crate::tokenizer::{Decoder, Result};
 use crate::utils::macro_rules_attribute;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 #[macro_rules_attribute(impl_serde_type!)]

tokenizers/src/decoders/strip.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{Decoder, Result};
 
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 #[derive(Deserialize, Clone, Debug, Serialize, Default, AutoDisplay)]
 /// Strip is a simple trick which converts tokens looking like `<0x61>`

tokenizers/src/decoders/wordpiece.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{Decoder, Result};
 
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 #[derive(Deserialize, Clone, Debug, Serialize, AutoDisplay)]
 /// The WordPiece decoder takes care of decoding a list of wordpiece tokens

tokenizers/src/models/mod.rs

@@ -8,7 +8,7 @@
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize, Serializer};
 
 use crate::models::bpe::{BpeTrainer, BPE};
@@ -58,7 +58,7 @@
     }
 }
 
 #[derive(Deserialize, Serialize, Debug, PartialEq, Clone, AutoDisplay)]
 #[serde(untagged)]
 pub enum ModelWrapper {
     BPE(BPE),

tokenizers/src/models/unigram/model.rs

@@ -6,7 +6,7 @@ use super::{
 use crate::tokenizer::{Model, Result, Token};
 use crate::utils::cache::Cache;
 
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use std::collections::HashMap;
 use std::convert::TryInto;
 use std::fs::read_to_string;

tokenizers/src/models/wordlevel/mod.rs

@@ -1,6 +1,6 @@
 use super::OrderedVocabIter;
 use crate::tokenizer::{Model, Result, Token};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde_json::Value;
 use std::collections::HashMap;
 use std::fs::File;

tokenizers/src/models/wordpiece/mod.rs

@@ -3,7 +3,7 @@
 
 use crate::models::bpe::BPE;
 use crate::tokenizer::{Model, Result, Token};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use std::{
     borrow::Cow,
     collections::HashMap,

tokenizers/src/normalizers/bert.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
 
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 use unicode_categories::UnicodeCategories;
 /// Checks whether a character is whitespace

tokenizers/src/normalizers/mod.rs

@@ -17,10 +17,10 @@
 use serde::{Deserialize, Serialize};
 
 use crate::{NormalizedString, Normalizer};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 
 /// Wrapper for known Normalizers.
 #[derive(Clone, Debug, Deserialize, Serialize, AutoDisplay)]
 #[serde(untagged)]
 #[auto_display(fmt = "normalizers.{}")]
 pub enum NormalizerWrapper {

tokenizers/src/normalizers/prepend.rs

@@ -1,5 +1,5 @@
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Debug, Deserialize, Serialize, AutoDisplay)]

tokenizers/src/normalizers/replace.rs

@@ -2,7 +2,7 @@ use crate::tokenizer::pattern::Pattern;
 use crate::tokenizer::Decoder;
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
 use crate::utils::SysRegex;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 /// Represents the different patterns that `Replace` can use
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq)]

tokenizers/src/normalizers/strip.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
 use crate::utils::macro_rules_attribute;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 use unicode_normalization_alignments::char::is_combining_mark;
 #[derive(Copy, Clone, Debug, Deserialize, Serialize, AutoDisplay)]

tokenizers/src/normalizers/unicode.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
 use crate::utils::macro_rules_attribute;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 
 #[derive(Default, Copy, Clone, Debug, AutoDisplay)]
 #[macro_rules_attribute(impl_serde_type!)]

tokenizers/src/normalizers/utils.rs

@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
 use crate::normalizers::NormalizerWrapper;
 use crate::tokenizer::{NormalizedString, Normalizer, Result};
 use crate::utils::macro_rules_attribute;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 #[derive(Clone, Deserialize, Debug, Serialize, AutoDisplay)]
 #[serde(tag = "type")]
 /// Allows concatenating multiple other Normalizer as a Sequence.

tokenizers/src/pre_tokenizers/bert.rs

@@ -1,6 +1,6 @@
 use crate::tokenizer::{PreTokenizedString, PreTokenizer, Result, SplitDelimiterBehavior};
 use crate::utils::macro_rules_attribute;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use unicode_categories::UnicodeCategories;
 
 fn is_bert_punc(x: char) -> bool {

tokenizers/src/pre_tokenizers/byte_level.rs

@@ -6,7 +6,7 @@ use crate::tokenizer::{
 };
 use crate::utils::macro_rules_attribute;
 use crate::utils::SysRegex;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 /// Converts bytes to unicode characters.

tokenizers/src/pre_tokenizers/delimiter.rs

@@ -1,4 +1,4 @@
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 use crate::tokenizer::{PreTokenizedString, PreTokenizer, Result, SplitDelimiterBehavior};

tokenizers/src/pre_tokenizers/digits.rs

@@ -1,4 +1,4 @@
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 use crate::tokenizer::{PreTokenizedString, PreTokenizer, Result, SplitDelimiterBehavior};

tokenizers/src/pre_tokenizers/metaspace.rs

@@ -1,5 +1,5 @@
 use crate::tokenizer::{Decoder, PreTokenizedString, PreTokenizer, Result, SplitDelimiterBehavior};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{de, Deserialize, Deserializer, Serialize};
 /// Enum representing options for the metaspace prepending scheme.
 #[derive(Debug, Clone, PartialEq, Serialize, Eq, Deserialize, Copy, AutoDisplay)]

tokenizers/src/pre_tokenizers/mod.rs

@@ -22,9 +22,9 @@
 use crate::pre_tokenizers::unicode_scripts::UnicodeScripts;
 use crate::pre_tokenizers::whitespace::{Whitespace, WhitespaceSplit};
 use crate::{PreTokenizedString, PreTokenizer};
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 
 #[derive(Deserialize, Serialize, Clone, Debug, PartialEq, AutoDisplay)]
 #[auto_display(fmt="pre_tokenizers.{}")]
 #[serde(untagged)]
 pub enum PreTokenizerWrapper {

tokenizers/src/pre_tokenizers/punctuation.rs

@@ -1,4 +1,4 @@
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 use crate::tokenizer::{PreTokenizedString, PreTokenizer, Result, SplitDelimiterBehavior};

tokenizers/src/pre_tokenizers/sequence.rs

@@ -1,7 +1,7 @@
 use crate::pre_tokenizers::PreTokenizerWrapper;
 use crate::tokenizer::{PreTokenizedString, PreTokenizer, Result};
 use crate::utils::macro_rules_attribute;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Serialize};
 
 #[macro_rules_attribute(impl_serde_type!)]

tokenizers/src/pre_tokenizers/split.rs

@@ -2,11 +2,11 @@
     pattern::Invert, PreTokenizedString, PreTokenizer, Result, SplitDelimiterBehavior,
 };
 use crate::utils::SysRegex;
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use serde::{Deserialize, Deserializer, Serialize};
 
 /// Represents the different patterns that `Split` can use
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Eq, AutoDisplay)]
 pub enum SplitPattern {
     String(String),
     Regex(String),

tokenizers/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs

@@ -1,4 +1,4 @@
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 
 use crate::pre_tokenizers::unicode_scripts::scripts::{get_script, Script};
 use crate::tokenizer::{normalizer::Range, PreTokenizedString, PreTokenizer, Result};

tokenizers/src/pre_tokenizers/whitespace.rs

@@ -1,4 +1,4 @@
-use pyo3_special_method_derive::AutoDisplay;
+use pyo3_special_method_derive_0_21::AutoDisplay;
 use regex::Regex;
 
 use crate::tokenizer::{