unimo
- modeling
- tokenizer
UNIMOTokenizer
UNIMOTokenizer.vocab_size
UNIMOTokenizer.load_vocabulary()
UNIMOTokenizer.get_vocab()
UNIMOTokenizer.convert_tokens_to_string()
UNIMOTokenizer.num_special_tokens_to_add()
UNIMOTokenizer.build_inputs_with_special_tokens()
UNIMOTokenizer.merge_subword()
UNIMOTokenizer.build_offset_mapping_with_special_tokens()
UNIMOTokenizer.create_token_type_ids_from_sequences()
UNIMOTokenizer.gen_encode()