paddlenlp.metrics.distinct 源代码

# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np
import paddle

__all__ = ['Distinct']


[文档]class Distinct(paddle.metric.Metric): """ `Distinct` is an algorithm for evaluating the textual diversity of the generated text by calculating the number of distinct n-grams. The larger the number of distinct n-grams, the higher the diversity of the text. See details at https://arxiv.org/abs/1510.03055. :class:`Distinct` could be used as a :class:`paddle.metric.Metric` class, or an ordinary class. When :class:`Distinct` is used as a :class:`paddle.metric.Metric` class, a function is needed to transform the network output to a string list. Args: n_size (int, optional): Number of gram for :class:`Distinct` metric. Defaults to 2. trans_func (callable, optional): `trans_func` transforms the network output to a string list. Defaults to None. .. note:: When :class:`Distinct` is used as a :class:`paddle.metric.Metric` class, `trans_func` must be provided. Please note that the input of `trans_func` is numpy array. name (str, optional): Name of :class:`paddle.metric.Metric` instance. Defaults to "distinct". Examples: 1. Using as a general evaluation object. .. code-block:: python from paddlenlp.metrics import Distinct distinct = Distinct() cand = ["The","cat","The","cat","on","the","mat"] #update the states distinct.add_inst(cand) print(distinct.score()) # 0.8333333333333334 2. Using as an instance of `paddle.metric.Metric`. .. code-block:: python import numpy as np from functools import partial import paddle from paddlenlp.transformers import BertTokenizer from paddlenlp.metrics import Distinct def trans_func(logits, tokenizer): '''Transform the network output `logits` to string list.''' # [batch_size, seq_len] token_ids = np.argmax(logits, axis=-1).tolist() cand_list = [] for ids in token_ids: tokens = tokenizer.convert_ids_to_tokens(ids) strings = tokenizer.convert_tokens_to_string(tokens) cand_list.append(strings.split()) return cand_list paddle.seed(2021) tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') distinct = Distinct(trans_func=partial(trans_func, tokenizer=tokenizer)) batch_size, seq_len, vocab_size = 4, 16, tokenizer.vocab_size logits = paddle.rand([batch_size, seq_len, vocab_size]) distinct.update(logits.numpy()) print(distinct.accumulate()) # 1.0 """ def __init__(self, n_size=2, trans_func=None, name="distinct"): super(Distinct, self).__init__() self._name = name self.diff_ngram = set() self.count = 0.0 self.n_size = n_size self.trans_func = trans_func
[文档] def update(self, output, *args): """ Updates the metrics states. This method firstly will use :meth:`trans_func` method to process the `output` to get the tokenized candidate sentence list. Then call :meth:`add_inst` method to process the candidate list one by one. Args: output (numpy.ndarray|Tensor): The outputs of model. args (tuple): The additional inputs. """ if isinstance(output, paddle.Tensor): output = output.numpy() assert self.trans_func is not None, "The `update` method requires user "\ "to provide `trans_func` when initializing `Distinct`." cand_list = self.trans_func(output) for cand in cand_list: self.add_inst(cand)
[文档] def add_inst(self, cand): """ Updates the states based on the candidate. Args: cand (list): Tokenized candidate sentence generated by model. """ for i in range(0, len(cand) - self.n_size + 1): ngram = ' '.join(cand[i:(i + self.n_size)]) self.count += 1 self.diff_ngram.add(ngram)
[文档] def reset(self): """Resets states and result.""" self.diff_ngram = set() self.count = 0.0
[文档] def accumulate(self): """ Calculates the final distinct score. Returns: float: The final distinct score. """ distinct = len(self.diff_ngram) / self.count return distinct
[文档] def score(self): """ The function is the same as :meth:`accumulate` method. Returns: float: The final distinct score. """ return self.accumulate()
[文档] def name(self): """ Returns the metric name. Returns: str: The metric name. """ return self._name