Source code for jina.executors.evaluators

__copyright__ = "Copyright (c) 2020 Jina AI Limited. All rights reserved."
__license__ = "Apache-2.0"

from typing import Any

from .running_stats import RunningStats
from .. import BaseExecutor
from ..compound import CompoundExecutor

[docs]class BaseEvaluator(BaseExecutor): """A :class:`BaseEvaluator` is used to evaluate different messages coming from any kind of executor""" metric = '' #: Get the name of the evaluation metric
[docs] def post_init(self): """Initialize running stats.""" super().post_init() self._running_stats = RunningStats()
[docs] def evaluate(self, actual: Any, desired: Any, *args, **kwargs) -> float: """Evaluates difference between param:`actual` and `param:desired`, needs to be implemented in subclass.""" raise NotImplementedError
@property def mean(self) -> float: """Get the running mean.""" return self._running_stats.mean @property def std(self) -> float: """Get the running standard variance.""" return self._running_stats.std @property def variance(self) -> float: """Get the running variance.""" return self._running_stats.variance
[docs]class FileBasedEvaluator(CompoundExecutor): """A Frequently used pattern for combining A :class:`BinaryPbIndexer` and :class:`BaseEvaluator`. It will be equipped with predefined ``requests.on`` behaviors: - At evaluation time(query or index) - 1. Checks for the incoming document, gets its value from the `BinaryPbIndexer` and fills the `groundtruth of the request - 2. Filter the documents that do not have a corresponding groundtruth - 3. The BaseEvaluator works as if the `groundtruth` had been provided by the client as it comes in the request. .. warning:: The documents that are not found to have an indexed groundtruth are removed from the `request` so that the `Evaluator` only works with documents which have groundtruth. One can use the :class:`FileBasedEvaluator` via .. highlight:: yaml .. code-block:: yaml !FileBasedEvaluator components: - !BinaryPbIndexer with: index_filename: ground_truth.gz metas: name: groundtruth_index # a customized name workspace: ${{TEST_WORKDIR}} - !BaseEvaluator Without defining any ``requests.on`` logic. When load from this YAML, it will be auto equipped with .. highlight:: yaml .. code-block:: yaml on: [SearchRequest, IndexRequest]: - !LoadGroundTruthDriver with: executor: BaseKVIndexer - !BaseEvaluateDriver with: executor: BaseEvaluator ControlRequest: - !ControlReqDriver {} """