xlm
- modeling
- tokenizer
XLMTokenizer
XLMTokenizer.ja_tokenize()
XLMTokenizer.vocab_size
XLMTokenizer.get_vocab()
XLMTokenizer.tokenize()
XLMTokenizer.convert_tokens_to_string()
XLMTokenizer.build_inputs_with_special_tokens()
XLMTokenizer.create_token_type_ids_from_sequences()
XLMTokenizer.get_special_tokens_mask()
XLMTokenizer.save_resources()