unimo#
- modeling
- tokenizer
UNIMOTokenizerUNIMOTokenizer.vocab_sizeUNIMOTokenizer.load_vocabulary()UNIMOTokenizer.get_vocab()UNIMOTokenizer.convert_tokens_to_string()UNIMOTokenizer.num_special_tokens_to_add()UNIMOTokenizer.build_inputs_with_special_tokens()UNIMOTokenizer.merge_subword()UNIMOTokenizer.build_offset_mapping_with_special_tokens()UNIMOTokenizer.create_token_type_ids_from_sequences()UNIMOTokenizer.gen_encode()