paddlenlp.transformers#
- albert
- modeling
- tokenizer
AlbertTokenizer
AlbertTokenizer.vocab_size
AlbertTokenizer.tokenize()
AlbertTokenizer.convert_tokens_to_string()
AlbertTokenizer.num_special_tokens_to_add()
AlbertTokenizer.build_inputs_with_special_tokens()
AlbertTokenizer.build_offset_mapping_with_special_tokens()
AlbertTokenizer.get_special_tokens_mask()
AlbertTokenizer.create_token_type_ids_from_sequences()
AlbertTokenizer.save_resources()
- artist
- auto
- modeling
AutoBackbone
AutoModel
AutoModelForPretraining
AutoModelForSequenceClassification
AutoModelForTokenClassification
AutoModelForQuestionAnswering
AutoModelForMultipleChoice
AutoModelForMaskedLM
AutoModelForCausalLM
AutoModelForCausalLMPipe
AutoEncoder
AutoDecoder
AutoGenerator
AutoDiscriminator
AutoModelForConditionalGeneration
- tokenizer
- modeling
- bart
- modeling
- tokenizer
BartTokenizer
BartTokenizer.build_inputs_with_special_tokens()
BartTokenizer.get_special_tokens_mask()
BartTokenizer.create_token_type_ids_from_sequences()
BartTokenizer.get_vocab()
BartTokenizer.vocab_size
BartTokenizer.convert_ids_to_string()
BartTokenizer.save_resources()
BartTokenizer.convert_tokens_to_string()
BartTokenizer.build_offset_mapping_with_special_tokens()
- bert
- modeling
- tokenizer
BasicTokenizer
BertTokenizer
BertTokenizer.vocab_size
BertTokenizer.get_vocab()
BertTokenizer.convert_tokens_to_string()
BertTokenizer.num_special_tokens_to_add()
BertTokenizer.build_inputs_with_special_tokens()
BertTokenizer.build_offset_mapping_with_special_tokens()
BertTokenizer.create_token_type_ids_from_sequences()
BertTokenizer.get_special_tokens_mask()
WordpieceTokenizer
- bert_japanese
- bigbird
- blenderbot
- blenderbot_small
- chinesebert
- codegen
- convbert
- modeling
ConvBertModel
ConvBertForMaskedLM
ConvBertPretrainedModel
ConvBertForTotalPretraining
ConvBertDiscriminator
ConvBertGenerator
ConvBertClassificationHead
ConvBertForSequenceClassification
ConvBertForTokenClassification
ConvBertPretrainingCriterion
ConvBertForQuestionAnswering
ConvBertForMultipleChoice
ConvBertForPretraining
- tokenizer
- modeling
- ctrl
- dallebart
- deberta
- modeling
- tokenizer
DebertaTokenizer
DebertaTokenizer.vocab_size
DebertaTokenizer.convert_ids_to_string()
DebertaTokenizer.create_token_type_ids_from_sequences()
DebertaTokenizer.save_resources()
DebertaTokenizer.convert_tokens_to_string()
DebertaTokenizer.get_vocab()
DebertaTokenizer.prepare_for_tokenization()
DebertaTokenizer.build_inputs_with_special_tokens()
DebertaTokenizer.get_special_tokens_mask()
DebertaTokenizer.build_offset_mapping_with_special_tokens()
- deberta_v2
- modeling
- tokenizer
DebertaV2Tokenizer
DebertaV2Tokenizer.vocab_size
DebertaV2Tokenizer.get_vocab()
DebertaV2Tokenizer.convert_tokens_to_string()
DebertaV2Tokenizer.build_inputs_with_special_tokens()
DebertaV2Tokenizer.get_special_tokens_mask()
DebertaV2Tokenizer.create_token_type_ids_from_sequences()
DebertaV2Tokenizer.prepare_for_tokenization()
DebertaV2Tokenizer.save_vocabulary()
DebertaV2Tokenizer.build_offset_mapping_with_special_tokens()
DebertaV2Tokenizer.save_resources()
- distilbert
- electra
- modeling
ElectraModel
ElectraPretrainedModel
ElectraForTotalPretraining
ElectraDiscriminator
ElectraGenerator
ElectraClassificationHead
ElectraForSequenceClassification
ElectraForTokenClassification
ElectraPretrainingCriterion
ElectraForMultipleChoice
ElectraForQuestionAnswering
ElectraForMaskedLM
ElectraForPretraining
ErnieHealthForTotalPretraining
ErnieHealthPretrainingCriterion
ErnieHealthDiscriminator
- tokenizer
ElectraTokenizer
ElectraTokenizer.vocab_size
ElectraTokenizer.get_vocab()
ElectraTokenizer.convert_tokens_to_string()
ElectraTokenizer.num_special_tokens_to_add()
ElectraTokenizer.build_inputs_with_special_tokens()
ElectraTokenizer.build_offset_mapping_with_special_tokens()
ElectraTokenizer.create_token_type_ids_from_sequences()
ElectraTokenizer.get_special_tokens_mask()
- modeling
- ernie
- modeling
- tokenizer
ErnieTokenizer
ErnieTokenizer.vocab_size
ErnieTokenizer.extend_chinese_char()
ErnieTokenizer.get_vocab()
ErnieTokenizer.convert_tokens_to_string()
ErnieTokenizer.num_special_tokens_to_add()
ErnieTokenizer.build_inputs_with_special_tokens()
ErnieTokenizer.build_offset_mapping_with_special_tokens()
ErnieTokenizer.create_token_type_ids_from_sequences()
ErnieTokenizer.get_special_tokens_mask()
ErnieTinyTokenizer
ErnieTinyTokenizer.vocab_size
ErnieTinyTokenizer.convert_tokens_to_string()
ErnieTinyTokenizer.save_resources()
ErnieTinyTokenizer.num_special_tokens_to_add()
ErnieTinyTokenizer.build_inputs_with_special_tokens()
ErnieTinyTokenizer.build_offset_mapping_with_special_tokens()
ErnieTinyTokenizer.create_token_type_ids_from_sequences()
ErnieTinyTokenizer.get_special_tokens_mask()
ErnieTinyTokenizer.get_vocab()
- ernie_ctm
- ernie_doc
- ernie_gen
- ernie_gram
- ernie_m
- modeling
- tokenizer
ErnieMTokenizer
ErnieMTokenizer.get_offset_mapping()
ErnieMTokenizer.vocab_size
ErnieMTokenizer.get_vocab()
ErnieMTokenizer.clean_text()
ErnieMTokenizer.convert_tokens_to_string()
ErnieMTokenizer.convert_ids_to_string()
ErnieMTokenizer.build_inputs_with_special_tokens()
ErnieMTokenizer.build_offset_mapping_with_special_tokens()
ErnieMTokenizer.get_special_tokens_mask()
ErnieMTokenizer.create_token_type_ids_from_sequences()
ErnieMTokenizer.is_ch_char()
ErnieMTokenizer.is_alpha()
ErnieMTokenizer.is_punct()
ErnieMTokenizer.is_whitespace()
- fnet
- funnel
- gau_alpha
- modeling
- tokenizer
GAUAlphaTokenizer
GAUAlphaTokenizer.vocab_size
GAUAlphaTokenizer.convert_tokens_to_string()
GAUAlphaTokenizer.num_special_tokens_to_add()
GAUAlphaTokenizer.build_inputs_with_special_tokens()
GAUAlphaTokenizer.build_offset_mapping_with_special_tokens()
GAUAlphaTokenizer.create_token_type_ids_from_sequences()
GAUAlphaTokenizer.get_special_tokens_mask()
GAUAlphaTokenizer.get_vocab()
- gpt
- layoutlm
- layoutlmv2
- layoutxlm
- modeling
- tokenizer
LayoutXLMTokenizer
LayoutXLMTokenizer.build_inputs_with_special_tokens()
LayoutXLMTokenizer.get_special_tokens_mask()
LayoutXLMTokenizer.create_token_type_ids_from_sequences()
LayoutXLMTokenizer.vocab_size
LayoutXLMTokenizer.get_vocab()
LayoutXLMTokenizer.convert_tokens_to_string()
LayoutXLMTokenizer.num_special_tokens_to_add()
- visual_backbone
- luke
- modeling
- tokenizer
LukeTokenizer
LukeTokenizer.sep_token_id
LukeTokenizer.cls_token_id
LukeTokenizer.pad_token_id
LukeTokenizer.unk_token_id
LukeTokenizer.get_entity_vocab()
LukeTokenizer.tokenize()
LukeTokenizer.convert_tokens_to_string()
LukeTokenizer.add_special_tokens()
LukeTokenizer.convert_entity_to_id()
LukeTokenizer.entity_encode()
LukeTokenizer.get_offset_mapping()
LukeTokenizer.create_token_type_ids_from_sequences()
LukeTokenizer.num_special_tokens_to_add()
LukeTokenizer.build_inputs_with_special_tokens()
- mbart
- modeling
- tokenizer
MBartTokenizer
MBartTokenizer.save_resources()
MBartTokenizer.vocab_size
MBartTokenizer.get_vocab()
MBartTokenizer.convert_tokens_to_string()
MBartTokenizer.convert_ids_to_string()
MBartTokenizer.get_special_tokens_mask()
MBartTokenizer.build_inputs_with_special_tokens()
MBartTokenizer.build_offset_mapping_with_special_tokens()
MBartTokenizer.set_src_lang_special_tokens()
MBartTokenizer.set_tgt_lang_special_tokens()
MBart50Tokenizer
MBart50Tokenizer.save_resources()
MBart50Tokenizer.get_vocab()
MBart50Tokenizer.vocab_size
MBart50Tokenizer.convert_tokens_to_string()
MBart50Tokenizer.convert_ids_to_string()
MBart50Tokenizer.get_special_tokens_mask()
MBart50Tokenizer.build_inputs_with_special_tokens()
MBart50Tokenizer.build_offset_mapping_with_special_tokens()
MBart50Tokenizer.set_src_lang_special_tokens()
MBart50Tokenizer.set_tgt_lang_special_tokens()
- megatronbert
- modeling
- tokenizer
- mobilebert
- mpnet
- nezha
- modeling
- tokenizer
NeZhaTokenizer
NeZhaTokenizer.vocab_size
NeZhaTokenizer.convert_tokens_to_string()
NeZhaTokenizer.num_special_tokens_to_add()
NeZhaTokenizer.build_inputs_with_special_tokens()
NeZhaTokenizer.build_offset_mapping_with_special_tokens()
NeZhaTokenizer.create_token_type_ids_from_sequences()
NeZhaTokenizer.get_special_tokens_mask()
NeZhaTokenizer.get_vocab()
- opt
- ppminilm
- modeling
- tokenizer
PPMiniLMTokenizer
PPMiniLMTokenizer.vocab_size
PPMiniLMTokenizer.get_vocab()
PPMiniLMTokenizer.convert_tokens_to_string()
PPMiniLMTokenizer.num_special_tokens_to_add()
PPMiniLMTokenizer.build_inputs_with_special_tokens()
PPMiniLMTokenizer.build_offset_mapping_with_special_tokens()
PPMiniLMTokenizer.create_token_type_ids_from_sequences()
PPMiniLMTokenizer.get_special_tokens_mask()
- prophetnet
- modeling
- tokenizer
load_vocab()
ProphetNetTokenizer
ProphetNetTokenizer.vocab_size
ProphetNetTokenizer.get_vocab()
ProphetNetTokenizer.tokenize()
ProphetNetTokenizer.convert_tokens_to_ids()
ProphetNetTokenizer.convert_ids_to_tokens()
ProphetNetTokenizer.convert_tokens_to_string()
ProphetNetTokenizer.get_special_tokens_mask()
ProphetNetTokenizer.create_token_type_ids_from_sequences()
ProphetNetTokenizer.build_inputs_with_special_tokens()
ProphetNetTokenizer.save_vocabulary()
- reformer
- rembert
- roberta
- modeling
- tokenizer
RobertaTokenizer
RobertaChineseTokenizer
RobertaChineseTokenizer.vocab_size
RobertaChineseTokenizer.get_vocab()
RobertaChineseTokenizer.convert_tokens_to_string()
RobertaChineseTokenizer.num_special_tokens_to_add()
RobertaChineseTokenizer.build_inputs_with_special_tokens()
RobertaChineseTokenizer.build_offset_mapping_with_special_tokens()
RobertaChineseTokenizer.create_token_type_ids_from_sequences()
RobertaChineseTokenizer.get_special_tokens_mask()
RobertaBPETokenizer
RobertaBPETokenizer.get_vocab()
RobertaBPETokenizer.build_inputs_with_special_tokens()
RobertaBPETokenizer.get_offset_mapping()
RobertaBPETokenizer.build_offset_mapping_with_special_tokens()
RobertaBPETokenizer.get_special_tokens_mask()
RobertaBPETokenizer.create_token_type_ids_from_sequences()
RobertaBPETokenizer.convert_tokens_to_string()
RobertaBPETokenizer.num_special_tokens_to_add()
RobertaBPETokenizer.prepare_for_tokenization()
- roformer
- modeling
- tokenizer
RoFormerTokenizer
RoFormerTokenizer.vocab_size
RoFormerTokenizer.convert_tokens_to_string()
RoFormerTokenizer.num_special_tokens_to_add()
RoFormerTokenizer.build_inputs_with_special_tokens()
RoFormerTokenizer.build_offset_mapping_with_special_tokens()
RoFormerTokenizer.create_token_type_ids_from_sequences()
RoFormerTokenizer.get_special_tokens_mask()
RoFormerTokenizer.get_vocab()
JiebaBasicTokenizer
- roformerv2
- modeling
- tokenizer
RoFormerv2Tokenizer
RoFormerv2Tokenizer.vocab_size
RoFormerv2Tokenizer.convert_tokens_to_string()
RoFormerv2Tokenizer.num_special_tokens_to_add()
RoFormerv2Tokenizer.build_inputs_with_special_tokens()
RoFormerv2Tokenizer.build_offset_mapping_with_special_tokens()
RoFormerv2Tokenizer.create_token_type_ids_from_sequences()
RoFormerv2Tokenizer.get_special_tokens_mask()
RoFormerv2Tokenizer.get_vocab()
- semantic_search
- skep
- modeling
- tokenizer
SkepTokenizer
SkepTokenizer.vocab_size
SkepTokenizer.num_special_tokens_to_add()
SkepTokenizer.build_offset_mapping_with_special_tokens()
SkepTokenizer.build_inputs_with_special_tokens()
SkepTokenizer.create_token_type_ids_from_sequences()
SkepTokenizer.save_resources()
SkepTokenizer.convert_tokens_to_string()
SkepTokenizer.get_special_tokens_mask()
SkepTokenizer.get_vocab()
- squeezebert
- modeling
- tokenizer
SqueezeBertTokenizer
SqueezeBertTokenizer.vocab_size
SqueezeBertTokenizer.get_vocab()
SqueezeBertTokenizer.convert_tokens_to_string()
SqueezeBertTokenizer.num_special_tokens_to_add()
SqueezeBertTokenizer.build_inputs_with_special_tokens()
SqueezeBertTokenizer.build_offset_mapping_with_special_tokens()
SqueezeBertTokenizer.create_token_type_ids_from_sequences()
SqueezeBertTokenizer.get_special_tokens_mask()
- t5
- modeling
- tokenizer
T5Tokenizer
T5Tokenizer.vocab_size
T5Tokenizer.build_inputs_with_special_tokens()
T5Tokenizer.build_offset_mapping_with_special_tokens()
T5Tokenizer.create_token_type_ids_from_sequences()
T5Tokenizer.get_special_tokens_mask()
T5Tokenizer.convert_tokens_to_string()
T5Tokenizer.batch_decode()
T5Tokenizer.clean_up_tokenization()
- tinybert
- transformer
- unified_transformer
- convert
- modeling
- tokenizer
UnifiedTransformerTokenizer
UnifiedTransformerTokenizer.vocab_size
UnifiedTransformerTokenizer.get_vocab()
UnifiedTransformerTokenizer.convert_tokens_to_string()
UnifiedTransformerTokenizer.convert_ids_to_string()
UnifiedTransformerTokenizer.num_special_tokens_to_add()
UnifiedTransformerTokenizer.build_inputs_with_special_tokens()
UnifiedTransformerTokenizer.build_offset_mapping_with_special_tokens()
UnifiedTransformerTokenizer.create_token_type_ids_from_sequences()
UnifiedTransformerTokenizer.get_special_tokens_mask()
UnifiedTransformerTokenizer.save_resources()
UnifiedTransformerTokenizer.load_vocabulary()
UnifiedTransformerTokenizer.dialogue_encode()
- unimo
- modeling
- tokenizer
UNIMOTokenizer
UNIMOTokenizer.vocab_size
UNIMOTokenizer.load_vocabulary()
UNIMOTokenizer.get_vocab()
UNIMOTokenizer.convert_tokens_to_string()
UNIMOTokenizer.num_special_tokens_to_add()
UNIMOTokenizer.build_inputs_with_special_tokens()
UNIMOTokenizer.merge_subword()
UNIMOTokenizer.build_offset_mapping_with_special_tokens()
UNIMOTokenizer.create_token_type_ids_from_sequences()
UNIMOTokenizer.gen_encode()
- xlm
- modeling
- tokenizer
XLMTokenizer
XLMTokenizer.ja_tokenize()
XLMTokenizer.vocab_size
XLMTokenizer.get_vocab()
XLMTokenizer.tokenize()
XLMTokenizer.convert_tokens_to_string()
XLMTokenizer.build_inputs_with_special_tokens()
XLMTokenizer.create_token_type_ids_from_sequences()
XLMTokenizer.get_special_tokens_mask()
XLMTokenizer.save_resources()
- xlnet
- modeling
- tokenizer
XLNetTokenizer
XLNetTokenizer.sp_model
XLNetTokenizer.vocab_size
XLNetTokenizer.get_vocab()
XLNetTokenizer.convert_tokens_to_string()
XLNetTokenizer.num_special_tokens_to_add()
XLNetTokenizer.build_inputs_with_special_tokens()
XLNetTokenizer.build_offset_mapping_with_special_tokens()
XLNetTokenizer.get_special_tokens_mask()
XLNetTokenizer.create_token_type_ids_from_sequences()
XLNetTokenizer.save_resources()
- attention_utils
- convert_slow_tokenizer
- distill_utils
- export
- generation_utils
- model_outputs
tuple_output()
convert_encoder_output()
ModelOutput
BaseModelOutput
BaseModelOutputWithNoAttention
BaseModelOutputWithPooling
BaseModelOutputWithPast
BaseModelOutputWithPastAndCrossAttentions
BaseModelOutputWithPoolingAndCrossAttentions
SequenceClassifierOutput
TokenClassifierOutput
QuestionAnsweringModelOutput
MultipleChoiceModelOutput
MaskedLMOutput
CausalLMOutputWithPast
CausalLMOutputWithCrossAttentions
Seq2SeqModelOutput
Seq2SeqLMOutput
Seq2SeqQuestionAnsweringModelOutput
Seq2SeqSequenceClassifierOutput
SequenceClassifierOutputWithPast
BackboneOutput
BaseModelOutputWithPoolingAndNoAttention
ImageClassifierOutputWithNoAttention
DepthEstimatorOutput
SemanticSegmenterOutput
Seq2SeqSpectrogramOutput
MoEModelOutputWithPast
MoECausalLMOutputWithPast
- model_utils
PretrainedModel
PretrainedModel.init_weights()
PretrainedModel.from_config()
PretrainedModel.base_model
PretrainedModel.model_name_list
PretrainedModel.can_generate()
PretrainedModel.recompute_enable()
PretrainedModel.recompute_disable()
PretrainedModel.get_memory_footprint()
PretrainedModel.get_input_embeddings()
PretrainedModel.set_input_embeddings()
PretrainedModel.get_output_embeddings()
PretrainedModel.tie_weights()
PretrainedModel.resize_position_embeddings()
PretrainedModel.constructed_from_pretrained_config()
PretrainedModel.save_model_config()
PretrainedModel.save_to_hf_hub()
PretrainedModel.save_to_aistudio()
PretrainedModel.resize_token_embeddings()
PretrainedModel.from_pretrained()
PretrainedModel.save_pretrained()
register_base_model()
- optimization
- sentencepiece_model_pb2
- tokenizer_utils
PretrainedTokenizer
PretrainedTokenizer.vocab_size
PretrainedTokenizer.get_added_vocab()
PretrainedTokenizer.prepare_for_tokenization()
PretrainedTokenizer.tokenize()
PretrainedTokenizer.convert_tokens_to_string()
PretrainedTokenizer.load_vocabulary()
PretrainedTokenizer.save_vocabulary()
PretrainedTokenizer.get_special_tokens_mask()
PretrainedTokenizer.num_special_tokens_to_add()
PretrainedTokenizer.get_offset_mapping()
PretrainedTokenizer.decode_token()
BPETokenizer
tokenize_chinese_chars()
is_chinese_char()
normalize_chars()
tokenize_special_chars()
convert_to_unicode()
- tokenizer_utils_base
AddedToken
FastEncoding
ExplicitEnum
PaddingStrategy
TensorType
to_py_obj()
TruncationStrategy
CharSpan
TokenSpan
BatchEncoding
BatchEncoding.n_sequences
BatchEncoding.is_fast
BatchEncoding.keys()
BatchEncoding.values()
BatchEncoding.items()
BatchEncoding.encodings
BatchEncoding.tokens()
BatchEncoding.sequence_ids()
BatchEncoding.words()
BatchEncoding.word_ids()
BatchEncoding.token_to_sequence()
BatchEncoding.token_to_word()
BatchEncoding.word_to_tokens()
BatchEncoding.token_to_chars()
BatchEncoding.char_to_token()
BatchEncoding.word_to_chars()
BatchEncoding.char_to_word()
BatchEncoding.convert_to_tensors()
SpecialTokensMixin
SpecialTokensMixin.sanitize_special_tokens()
SpecialTokensMixin.add_special_tokens()
SpecialTokensMixin.add_tokens()
SpecialTokensMixin.bos_token
SpecialTokensMixin.eos_token
SpecialTokensMixin.unk_token
SpecialTokensMixin.sep_token
SpecialTokensMixin.pad_token
SpecialTokensMixin.cls_token
SpecialTokensMixin.mask_token
SpecialTokensMixin.additional_special_tokens
SpecialTokensMixin.pad_token_type_id
SpecialTokensMixin.bos_token_id
SpecialTokensMixin.eos_token_id
SpecialTokensMixin.unk_token_id
SpecialTokensMixin.sep_token_id
SpecialTokensMixin.pad_token_id
SpecialTokensMixin.cls_token_id
SpecialTokensMixin.mask_token_id
SpecialTokensMixin.additional_special_tokens_ids
SpecialTokensMixin.special_tokens_map
SpecialTokensMixin.special_tokens_map_extended
SpecialTokensMixin.all_special_tokens
SpecialTokensMixin.all_special_tokens_extended
SpecialTokensMixin.all_special_ids
PretrainedTokenizerBase
PretrainedTokenizerBase.max_len_single_sentence
PretrainedTokenizerBase.max_len_sentences_pair
PretrainedTokenizerBase.get_vocab()
PretrainedTokenizerBase.from_pretrained()
PretrainedTokenizerBase.save_pretrained()
PretrainedTokenizerBase.save_resources()
PretrainedTokenizerBase.save_to_hf_hub()
PretrainedTokenizerBase.save_to_aistudio()
PretrainedTokenizerBase.tokenize()
PretrainedTokenizerBase.__call__()
PretrainedTokenizerBase.encode()
PretrainedTokenizerBase.encode_plus()
PretrainedTokenizerBase.batch_encode()
PretrainedTokenizerBase.pad()
PretrainedTokenizerBase.create_token_type_ids_from_sequences()
PretrainedTokenizerBase.build_inputs_with_special_tokens()
PretrainedTokenizerBase.build_offset_mapping_with_special_tokens()
PretrainedTokenizerBase.prepare_for_model()
PretrainedTokenizerBase.truncate_sequences()
PretrainedTokenizerBase.convert_tokens_to_string()
PretrainedTokenizerBase.batch_decode()
PretrainedTokenizerBase.decode()
PretrainedTokenizerBase.get_special_tokens_mask()
PretrainedTokenizerBase.clean_up_tokenization()
- tokenizer_utils_fast
- utils
convert_ndarray_dtype()
get_scale_by_dtype()
fn_args_to_dict()
adapt_stale_fwd_patch()
InitTrackerMeta
param_in_func()
resolve_cache_dir()
find_transformer_model_type()
find_transformer_model_class_by_name()
convert_file_size_to_int()
cached_file()
get_checkpoint_shard_files()
ContextManagers
dtype_byte_size()
CaptureStd