wip

.gitignore

@@ -66,6 +66,4 @@ site/
 
 .idea/
 
-scripts/
-
 data/

expts/scripts/tokenizer_trainer.py

@@ -33,9 +33,7 @@ class TokenizerTrainingArguments:
     tokenizer_name: Optional[str] = field(default="safe", metadata={"help": "Name of new tokenizer."})
     outfile: Optional[str] = field(default=None, metadata={"help": "Path to the local save of the trained tokenizer"})
     all_split: Optional[bool] = field(default=False, metadata={"help": "Whether to use all the splits or just the train split if only that is available."})
-    push_to_hub: Optional[bool] = field(default=True, metadata={"help": "Whether to push saved tokenizer to the hub."})
-
-
+    push_to_hub: Optional[bool] = field(default=False, metadata={"help": "Whether to push saved tokenizer to the hub."})
 
 if __name__ == "__main__":
     # Configuration
@@ -61,6 +59,7 @@ class TokenizerTrainingArguments:
         if args.outfile is not None:
             tokenizer.save(args.outfile)
         tokenizer = tokenizer.get_pretrained()
+
     tokenizer.save_pretrained(tokenizer_name, push_to_hub=args.push_to_hub)
 
 

safe/tokenizer.py

@@ -155,13 +155,13 @@ def set_special_tokens(cls, tokenizer, bos_token=CLS_TOKEN, eos_token=SEP_TOKEN)
             bos_token: Optional bos token to use
             eos_token: Optional eos token to use
         """
-        tokenizer._pad_token = PADDING_TOKEN
-        tokenizer._cls_token = CLS_TOKEN
-        tokenizer._sep_token = SEP_TOKEN
-        tokenizer._mask_token = MASK_TOKEN
-        tokenizer._unk_token = UNK_TOKEN
-        tokenizer._eos_token = eos_token
-        tokenizer._bos_token = bos_token
+        tokenizer.pad_token = PADDING_TOKEN
+        tokenizer.cls_token = CLS_TOKEN
+        tokenizer.sep_token = SEP_TOKEN
+        tokenizer.mask_token = MASK_TOKEN
+        tokenizer.unk_token = UNK_TOKEN
+        tokenizer.eos_token = eos_token
+        tokenizer.bos_token = bos_token
         if isinstance(tokenizer, Tokenizer):
             tokenizer.add_special_tokens(
                 [
@@ -267,7 +267,6 @@ def save_pretrained(self, *args, **kwargs):
         """Save pretrained tokenizer"""
         self.tokenizer.save_pretrained(*args, **kwargs)
 
-
     def save(self, file_name=None):
         r"""
         Saves the :class:`~tokenizers.Tokenizer` to the file at the given path.