Metric

Metric

pydantic settings olive.evaluator.metric.Metric

field name: str [Required]

field type: MetricType [Required]

field backend: str | None = 'torch_metrics'

field sub_types: List[SubMetric] [Required]

field user_config: ConfigBase = None

field data_config: DataConfig | None = None

get_inference_settings(framework)

get_run_kwargs() → Dict[str, Any]

get_sub_type_info(info_name, no_priority_filter=True, callback=<function Metric.<lambda>>)

MetricType

class olive.evaluator.metric.MetricType(value)

An enumeration.

ACCURACY = 'accuracy'

CUSTOM = 'custom'

LATENCY = 'latency'

THROUGHPUT = 'throughput'

AccuracySubType

class olive.evaluator.metric.AccuracySubType(value)

An enumeration.

ACCURACY_SCORE = 'accuracy_score'

AUROC = 'auroc'

F1_SCORE = 'f1_score'

PERPLEXITY = 'perplexity'

PRECISION = 'precision'

RECALL = 'recall'

LatencySubType

class olive.evaluator.metric.LatencySubType(value)

An enumeration.

AVG = 'avg'

MAX = 'max'

MIN = 'min'

P50 = 'p50'

P75 = 'p75'

P90 = 'p90'

P95 = 'p95'

P99 = 'p99'

P999 = 'p999'

ThroughputSubType

class olive.evaluator.metric.ThroughputSubType(value)

An enumeration.

AVG = 'avg'

MAX = 'max'

MIN = 'min'

P50 = 'p50'

P75 = 'p75'

P90 = 'p90'

P95 = 'p95'

P99 = 'p99'

P999 = 'p999'

MetricGoal

pydantic settings olive.evaluator.metric.MetricGoal

field type: str [Required]

field value: float [Required]

has_regression_goal()