diff --git a/prometheus_client/metrics.py b/prometheus_client/metrics.py index b9f25ffc..9e9e3d58 100644 --- a/prometheus_client/metrics.py +++ b/prometheus_client/metrics.py @@ -1,3 +1,4 @@ +import math import os from threading import Lock import time @@ -557,10 +558,16 @@ def create_response(request): The default buckets are intended to cover a typical web/rpc request from milliseconds to seconds. They can be overridden by passing `buckets` keyword argument to `Histogram`. + + In addition, native histograms are experimentally supported, but may change at any time. In order + to use native histograms, one must set `native_histogram_bucket_factor` to a value greater than 1.0. + When native histograms are enabled the classic histogram buckets are only collected if they are + explicitly set. """ _type = 'histogram' _reserved_labelnames = ['le'] DEFAULT_BUCKETS = (.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, INF) + DEFAULT_NATIVE_HISTOGRAM_ZERO_THRESHOLD = 2.938735877055719e-39 def __init__(self, name: str, @@ -571,9 +578,26 @@ def __init__(self, unit: str = '', registry: Optional[CollectorRegistry] = REGISTRY, _labelvalues: Optional[Sequence[str]] = None, - buckets: Sequence[Union[float, str]] = DEFAULT_BUCKETS, + buckets: Optional[Sequence[Union[float, str]]] = None, + native_histogram_initial_schema: Optional[int] = None, + native_histogram_max_buckets: int = 160, + native_histogram_zero_threshold: float = DEFAULT_NATIVE_HISTOGRAM_ZERO_THRESHOLD, + native_histogram_max_exemplars: int = 10, ): + if native_histogram_initial_schema and (native_histogram_initial_schema > 8 or native_histogram_initial_schema < -4): + raise ValueError("native_histogram_initial_schema must be between -4 and 8 inclusive") + + # Use the default buckets iff we are not using a native histogram. + if buckets is None and native_histogram_initial_schema is None: + buckets = self.DEFAULT_BUCKETS + self._prepare_buckets(buckets) + + self._schema = native_histogram_initial_schema + self._max_nh_buckets = native_histogram_max_buckets + self._zero_threshold = native_histogram_zero_threshold + self._max_nh_exemplars = native_histogram_max_exemplars, + super().__init__( name=name, documentation=documentation, @@ -586,7 +610,12 @@ def __init__(self, ) self._kwargs['buckets'] = buckets - def _prepare_buckets(self, source_buckets: Sequence[Union[float, str]]) -> None: + def _prepare_buckets(self, source_buckets: Optional[Sequence[Union[float, str]]]) -> None: + # Only native histograms are supported for this case. + if source_buckets is None: + self._upper_bounds = None + return + buckets = [float(b) for b in source_buckets] if buckets != sorted(buckets): # This is probably an error on the part of the user, @@ -601,17 +630,35 @@ def _prepare_buckets(self, source_buckets: Sequence[Union[float, str]]) -> None: def _metric_init(self) -> None: self._buckets: List[values.ValueClass] = [] self._created = time.time() - bucket_labelnames = self._labelnames + ('le',) - self._sum = values.ValueClass(self._type, self._name, self._name + '_sum', self._labelnames, self._labelvalues, self._documentation) - for b in self._upper_bounds: - self._buckets.append(values.ValueClass( - self._type, - self._name, - self._name + '_bucket', - bucket_labelnames, - self._labelvalues + (floatToGoString(b),), - self._documentation) - ) + + if self._schema is not None: + self._native_histogram = values.NativeHistogramMutexValue( + self._type, + self._name, + self._name, + self._labelnames, + self._labelvalues, + self._documentation, + self._schema, + self._zero_threshold, + self._max_nh_buckets, + self._max_nh_exemplars, + ) + + if self._upper_bounds is not None: + bucket_labelnames = self._labelnames + ('le',) + self._sum = values.ValueClass(self._type, self._name, self._name + '_sum', self._labelnames, self._labelvalues, self._documentation) + for b in self._upper_bounds: + self._buckets.append(values.ValueClass( + self._type, + self._name, + self._name + '_bucket', + bucket_labelnames, + self._labelvalues + (floatToGoString(b),), + self._documentation) + ) + + def observe(self, amount: float, exemplar: Optional[Dict[str, str]] = None) -> None: """Observe the given amount. @@ -624,14 +671,18 @@ def observe(self, amount: float, exemplar: Optional[Dict[str, str]] = None) -> N for details. """ self._raise_if_not_observable() - self._sum.inc(amount) - for i, bound in enumerate(self._upper_bounds): - if amount <= bound: - self._buckets[i].inc(1) - if exemplar: - _validate_exemplar(exemplar) - self._buckets[i].set_exemplar(Exemplar(exemplar, amount, time.time())) - break + if self._upper_bounds is not None: + self._sum.inc(amount) + for i, bound in enumerate(self._upper_bounds): + if amount <= bound: + self._buckets[i].inc(1) + if exemplar: + _validate_exemplar(exemplar) + self._buckets[i].set_exemplar(Exemplar(exemplar, amount, time.time())) + break + + if self._schema and not math.isnan(amount): + self._native_histogram.observe(amount) def time(self) -> Timer: """Time a block of code or function, and observe the duration in seconds. @@ -642,15 +693,19 @@ def time(self) -> Timer: def _child_samples(self) -> Iterable[Sample]: samples = [] - acc = 0.0 - for i, bound in enumerate(self._upper_bounds): - acc += self._buckets[i].get() - samples.append(Sample('_bucket', {'le': floatToGoString(bound)}, acc, None, self._buckets[i].get_exemplar())) - samples.append(Sample('_count', {}, acc, None, None)) - if self._upper_bounds[0] >= 0: - samples.append(Sample('_sum', {}, self._sum.get(), None, None)) - if _use_created: - samples.append(Sample('_created', {}, self._created, None, None)) + if self._upper_bounds is not None: + acc = 0.0 + for i, bound in enumerate(self._upper_bounds): + acc += self._buckets[i].get() + samples.append(Sample('_bucket', {'le': floatToGoString(bound)}, acc, None, self._buckets[i].get_exemplar())) + samples.append(Sample('_count', {}, acc, None, None)) + if self._upper_bounds[0] >= 0: + samples.append(Sample('_sum', {}, self._sum.get(), None, None)) + if _use_created: + samples.append(Sample('_created', {}, self._created, None, None)) + + if self._schema: + samples.append(Sample('', {}, 0.0, None, None, self._native_histogram.get())) return tuple(samples) diff --git a/prometheus_client/registry.py b/prometheus_client/registry.py index 694e4bd8..a02b0405 100644 --- a/prometheus_client/registry.py +++ b/prometheus_client/registry.py @@ -3,6 +3,8 @@ from threading import Lock from typing import Dict, Iterable, List, Optional +from prometheus_client.samples import NativeHistogram + from .metrics_core import Metric @@ -141,6 +143,19 @@ def get_sample_value(self, name: str, labels: Optional[Dict[str, str]] = None) - return s.value return None + def get_native_histogram_value(self, name: str, labels: Optional[Dict[str, str]] = None) -> Optional[NativeHistogram]: + """Returns the sample's native histogram value, or None if not found. + + This is inefficient, and intended only for use in unittests. + """ + if labels is None: + labels = {} + for metric in self.collect(): + for s in metric.samples: + if s.name == name and s.labels == labels: + return s.native_histogram + return None + class RestrictedRegistry: def __init__(self, names: Iterable[str], registry: CollectorRegistry): diff --git a/prometheus_client/values.py b/prometheus_client/values.py index 6ff85e3b..85a67138 100644 --- a/prometheus_client/values.py +++ b/prometheus_client/values.py @@ -1,7 +1,13 @@ +import math import os +from bisect import bisect_left +from collections import Counter from threading import Lock +from typing import List, Optional, Tuple import warnings +from prometheus_client.samples import BucketSpan, Exemplar, NativeHistogram + from .mmap_dict import mmap_key, MmapedDict @@ -35,6 +41,306 @@ def get_exemplar(self): with self._lock: return self._exemplar +_DOUBLE_MAX = 1.7976931348623158E+308 + +# nativeHistogramBounds for the frac of observed values. Only relevant for +# schema > 0. The position in the slice is the schema. (0 is never used, just +# here for convenience of using the schema directly as the index.) See +# https://github.com/prometheus/client_golang/blob/9b83d994624f3cab82ec593133a598b3a27d0841/prometheus/histogram.go#L37 +# for more information and how these were created. +_NATIVE_HISTOGRAM_BOUNDS: List[List[float]] = [ + # Schema "0": + [0.5], + # Schema 1: + [0.5, 0.7071067811865475], + # Schema 2: + [0.5, 0.5946035575013605, 0.7071067811865475, 0.8408964152537144], + # Schema 3: + [ + 0.5, 0.5452538663326288, 0.5946035575013605, 0.6484197773255048, + 0.7071067811865475, 0.7711054127039704, 0.8408964152537144, 0.9170040432046711, + ], + # Schema 4: + [ + 0.5, 0.5221368912137069, 0.5452538663326288, 0.5693943173783458, + 0.5946035575013605, 0.620928906036742, 0.6484197773255048, 0.6771277734684463, + 0.7071067811865475, 0.7384130729697496, 0.7711054127039704, 0.805245165974627, + 0.8408964152537144, 0.8781260801866495, 0.9170040432046711, 0.9576032806985735, + ], + # Schema 5: + [ + 0.5, 0.5109485743270583, 0.5221368912137069, 0.5335702003384117, + 0.5452538663326288, 0.5571933712979462, 0.5693943173783458, 0.5818624293887887, + 0.5946035575013605, 0.6076236799902344, 0.620928906036742, 0.6345254785958666, + 0.6484197773255048, 0.6626183215798706, 0.6771277734684463, 0.6919549409819159, + 0.7071067811865475, 0.7225904034885232, 0.7384130729697496, 0.7545822137967112, + 0.7711054127039704, 0.7879904225539431, 0.805245165974627, 0.8228777390769823, + 0.8408964152537144, 0.8593096490612387, 0.8781260801866495, 0.8973545375015533, + 0.9170040432046711, 0.9370838170551498, 0.9576032806985735, 0.9785720620876999, + ], + # Schema 6: + [ + 0.5, 0.5054446430258502, 0.5109485743270583, 0.5165124395106142, + 0.5221368912137069, 0.5278225891802786, 0.5335702003384117, 0.5393803988785598, + 0.5452538663326288, 0.5511912916539204, 0.5571933712979462, 0.5632608093041209, + 0.5693943173783458, 0.5755946149764913, 0.5818624293887887, 0.5881984958251406, + 0.5946035575013605, 0.6010783657263515, 0.6076236799902344, 0.6142402680534349, + 0.620928906036742, 0.6276903785123455, 0.6345254785958666, 0.6414350080393891, + 0.6484197773255048, 0.6554806057623822, 0.6626183215798706, 0.6698337620266515, + 0.6771277734684463, 0.6845012114872953, 0.6919549409819159, 0.6994898362691555, + 0.7071067811865475, 0.7148066691959849, 0.7225904034885232, 0.7304588970903234, + 0.7384130729697496, 0.7464538641456323, 0.7545822137967112, 0.762799075372269, + 0.7711054127039704, 0.7795022001189185, 0.7879904225539431, 0.7965710756711334, + 0.805245165974627, 0.8140137109286738, 0.8228777390769823, 0.8318382901633681, + 0.8408964152537144, 0.8500531768592616, 0.8593096490612387, 0.8686669176368529, + 0.8781260801866495, 0.8876882462632604, 0.8973545375015533, 0.9071260877501991, + 0.9170040432046711, 0.9269895625416926, 0.9370838170551498, 0.9472879907934827, + 0.9576032806985735, 0.9680308967461471, 0.9785720620876999, 0.9892280131939752, + ], + # Schema 7: + [ + 0.5, 0.5027149505564014, 0.5054446430258502, 0.5081891574554764, + 0.5109485743270583, 0.5137229745593818, 0.5165124395106142, 0.5193170509806894, + 0.5221368912137069, 0.5249720429003435, 0.5278225891802786, 0.5306886136446309, + 0.5335702003384117, 0.5364674337629877, 0.5393803988785598, 0.5423091811066545, + 0.5452538663326288, 0.5482145409081883, 0.5511912916539204, 0.5541842058618393, + 0.5571933712979462, 0.5602188762048033, 0.5632608093041209, 0.5663192597993595, + 0.5693943173783458, 0.572486072215902, 0.5755946149764913, 0.5787200368168754, + 0.5818624293887887, 0.585021884841625, 0.5881984958251406, 0.5913923554921704, + 0.5946035575013605, 0.5978321960199137, 0.6010783657263515, 0.6043421618132907, + 0.6076236799902344, 0.6109230164863786, 0.6142402680534349, 0.6175755319684665, + 0.620928906036742, 0.6243004885946023, 0.6276903785123455, 0.6310986751971253, + 0.6345254785958666, 0.637970889198196, 0.6414350080393891, 0.6449179367033329, + 0.6484197773255048, 0.6519406325959679, 0.6554806057623822, 0.659039800633032, + 0.6626183215798706, 0.6662162735415805, 0.6698337620266515, 0.6734708931164728, + 0.6771277734684463, 0.6808045103191123, 0.6845012114872953, 0.688217985377265, + 0.6919549409819159, 0.6957121878859629, 0.6994898362691555, 0.7032879969095076, + 0.7071067811865475, 0.7109463010845827, 0.7148066691959849, 0.718687998724491, + 0.7225904034885232, 0.7265139979245261, 0.7304588970903234, 0.7344252166684908, + 0.7384130729697496, 0.7424225829363761, 0.7464538641456323, 0.7505070348132126, + 0.7545822137967112, 0.7586795205991071, 0.762799075372269, 0.7669409989204777, + 0.7711054127039704, 0.7752924388424999, 0.7795022001189185, 0.7837348199827764, + 0.7879904225539431, 0.7922691326262467, 0.7965710756711334, 0.8008963778413465, + 0.805245165974627, 0.8096175675974316, 0.8140137109286738, 0.8184337248834821, + 0.8228777390769823, 0.8273458838280969, 0.8318382901633681, 0.8363550898207981, + 0.8408964152537144, 0.8454623996346523, 0.8500531768592616, 0.8546688815502312, + 0.8593096490612387, 0.8639756154809185, 0.8686669176368529, 0.8733836930995842, + 0.8781260801866495, 0.8828942179666361, 0.8876882462632604, 0.8925083056594671, + 0.8973545375015533, 0.9022270839033115, 0.9071260877501991, 0.9120516927035263, + 0.9170040432046711, 0.9219832844793128, 0.9269895625416926, 0.9320230241988943, + 0.9370838170551498, 0.9421720895161669, 0.9472879907934827, 0.9524316709088368, + 0.9576032806985735, 0.9628029718180622, 0.9680308967461471, 0.9732872087896164, + 0.9785720620876999, 0.9838856116165875, 0.9892280131939752, 0.9945994234836328, + ], + # Schema 8: + [ + 0.5, 0.5013556375251013, 0.5027149505564014, 0.5040779490592088, + 0.5054446430258502, 0.5068150424757447, 0.5081891574554764, 0.509566998038869, + 0.5109485743270583, 0.5123338964485679, 0.5137229745593818, 0.5151158188430205, + 0.5165124395106142, 0.5179128468009786, 0.5193170509806894, 0.520725062344158, + 0.5221368912137069, 0.5235525479396449, 0.5249720429003435, 0.526395386502313, + 0.5278225891802786, 0.5292536613972564, 0.5306886136446309, 0.5321274564422321, + 0.5335702003384117, 0.5350168559101208, 0.5364674337629877, 0.5379219445313954, + 0.5393803988785598, 0.5408428074966075, 0.5423091811066545, 0.5437795304588847, + 0.5452538663326288, 0.5467321995364429, 0.5482145409081883, 0.549700901315111, + 0.5511912916539204, 0.5526857228508706, 0.5541842058618393, 0.5556867516724088, + 0.5571933712979462, 0.5587040757836845, 0.5602188762048033, 0.5617377836665098, + 0.5632608093041209, 0.564787964283144, 0.5663192597993595, 0.5678547070789026, + 0.5693943173783458, 0.5709381019847808, 0.572486072215902, 0.5740382394200894, + 0.5755946149764913, 0.5771552102951081, 0.5787200368168754, 0.5802891060137493, + 0.5818624293887887, 0.5834400184762408, 0.585021884841625, 0.5866080400818185, + 0.5881984958251406, 0.5897932637314379, 0.5913923554921704, 0.5929957828304968, + 0.5946035575013605, 0.5962156912915756, 0.5978321960199137, 0.5994530835371903, + 0.6010783657263515, 0.6027080545025619, 0.6043421618132907, 0.6059806996384005, + 0.6076236799902344, 0.6092711149137041, 0.6109230164863786, 0.6125793968185725, + 0.6142402680534349, 0.6159056423670379, 0.6175755319684665, 0.6192499490999082, + 0.620928906036742, 0.622612415087629, 0.6243004885946023, 0.6259931389331581, + 0.6276903785123455, 0.6293922197748583, 0.6310986751971253, 0.6328097572894031, + 0.6345254785958666, 0.6362458516947014, 0.637970889198196, 0.6397006037528346, + 0.6414350080393891, 0.6431741147730128, 0.6449179367033329, 0.6466664866145447, + 0.6484197773255048, 0.6501778216898253, 0.6519406325959679, 0.6537082229673385, + 0.6554806057623822, 0.6572577939746774, 0.659039800633032, 0.6608266388015788, + 0.6626183215798706, 0.6644148621029772, 0.6662162735415805, 0.6680225691020727, + 0.6698337620266515, 0.6716498655934177, 0.6734708931164728, 0.6752968579460171, + 0.6771277734684463, 0.6789636531064505, 0.6808045103191123, 0.6826503586020058, + 0.6845012114872953, 0.6863570825438342, 0.688217985377265, 0.690083933630119, + 0.6919549409819159, 0.6938310211492645, 0.6957121878859629, 0.6975984549830999, + 0.6994898362691555, 0.7013863456101023, 0.7032879969095076, 0.7051948041086352, + 0.7071067811865475, 0.7090239421602076, 0.7109463010845827, 0.7128738720527471, + 0.7148066691959849, 0.7167447066838943, 0.718687998724491, 0.7206365595643126, + 0.7225904034885232, 0.7245495448210174, 0.7265139979245261, 0.7284837772007218, + 0.7304588970903234, 0.7324393720732029, 0.7344252166684908, 0.7364164454346837, + 0.7384130729697496, 0.7404151139112358, 0.7424225829363761, 0.7444354947621984, + 0.7464538641456323, 0.7484777058836176, 0.7505070348132126, 0.7525418658117031, + 0.7545822137967112, 0.7566280937263048, 0.7586795205991071, 0.7607365094544071, + 0.762799075372269, 0.7648672334736434, 0.7669409989204777, 0.7690203869158282, + 0.7711054127039704, 0.7731960915705107, 0.7752924388424999, 0.7773944698885442, + 0.7795022001189185, 0.7816156449856788, 0.7837348199827764, 0.7858597406461707, + 0.7879904225539431, 0.7901268813264122, 0.7922691326262467, 0.7944171921585818, + 0.7965710756711334, 0.7987307989543135, 0.8008963778413465, 0.8030678282083853, + 0.805245165974627, 0.8074284071024302, 0.8096175675974316, 0.8118126635086642, + 0.8140137109286738, 0.8162207259936375, 0.8184337248834821, 0.820652723822003, + 0.8228777390769823, 0.8251087869603088, 0.8273458838280969, 0.8295890460808079, + 0.8318382901633681, 0.8340936325652911, 0.8363550898207981, 0.8386226785089391, + 0.8408964152537144, 0.8431763167241966, 0.8454623996346523, 0.8477546807446661, + 0.8500531768592616, 0.8523579048290255, 0.8546688815502312, 0.8569861239649629, + 0.8593096490612387, 0.8616394738731368, 0.8639756154809185, 0.8663180910111553, + 0.8686669176368529, 0.871022112577578, 0.8733836930995842, 0.8757516765159389, + 0.8781260801866495, 0.8805069215187917, 0.8828942179666361, 0.8852879870317771, + 0.8876882462632604, 0.890095013257712, 0.8925083056594671, 0.8949281411607002, + 0.8973545375015533, 0.8997875124702672, 0.9022270839033115, 0.9046732696855155, + 0.9071260877501991, 0.909585556079304, 0.9120516927035263, 0.9145245157024483, + 0.9170040432046711, 0.9194902933879467, 0.9219832844793128, 0.9244830347552253, + 0.9269895625416926, 0.92950288621441, 0.9320230241988943, 0.9345499949706191, + 0.9370838170551498, 0.93962450902828, 0.9421720895161669, 0.9447265771954693, + 0.9472879907934827, 0.9498563490882775, 0.9524316709088368, 0.9550139751351947, + 0.9576032806985735, 0.9601996065815236, 0.9628029718180622, 0.9654133954938133, + 0.9680308967461471, 0.9706554947643201, 0.9732872087896164, 0.9759260581154889, + 0.9785720620876999, 0.9812252401044634, 0.9838856116165875, 0.9865531961276168, + 0.9892280131939752, 0.9919100824251095, 0.9945994234836328, 0.9972960560854698, + ], +] + +def _spans_and_deltas(buckets: Counter) -> Tuple[Optional[List[BucketSpan]], Optional[List[int]]]: + if len(buckets) == 0: + return None, None + + # Get sorted bucket indices + bucket_indices = sorted(buckets.keys()) + + spans = [] + deltas = [] + prev_count = 0 + next_i = 0 + + def append_delta(count: int) -> int: + spans[-1] = BucketSpan(spans[-1].offset, spans[-1].length + 1) + deltas.append(count - prev_count) + return count + + for n, i in enumerate(bucket_indices): + count = buckets[i] + # Multiple spans with only small gaps in between are probably + # encoded more efficiently as one larger span with a few empty + # buckets. For now, we assume that gaps of one or two buckets + # should not create a new span. + i_delta = i - next_i + if n == 0 or i_delta > 2: + # Create new span, either at start or after gap > 2 buckets + spans.append(BucketSpan(i_delta, 0)) + else: + # Small gap (or no gap), insert empty buckets as needed + for _ in range(i_delta): + prev_count = append_delta(0) + + prev_count = append_delta(count) + next_i = i + 1 + + return spans, deltas + + +class NativeHistogramMutexValue: + """A native histogram protected by a mutex.""" + + _multiprocess = False + def __init__(self, typ, metric_name, name, labelnames, labelvalues, help_text, initial_schema, zero_threshold, max_buckets, max_exemplars, **kwargs): + self._lock = Lock() + self._schema = initial_schema + self._zero_threshold = zero_threshold + self._positive_buckets = Counter() + self._negative_buckets = Counter() + self._zero_bucket: int = 0 + self._count: int = 0 + self._sum = 0.0 + self._max_buckets = max_buckets + + self._max_exemplars = max_exemplars + self._exemplars: List[Exemplar] = [] + + def get(self) -> NativeHistogram: + with self._lock: + pos_spans, pos_deltas = _spans_and_deltas(self._positive_buckets) + neg_spans, neg_deltas = _spans_and_deltas(self._negative_buckets) + return NativeHistogram( + count_value=self._count, + sum_value=self._sum, + schema=self._schema, + zero_threshold=self._zero_threshold, + zero_count=self._zero_bucket, + pos_spans=pos_spans, + neg_spans=neg_spans, + pos_deltas=pos_deltas, + neg_deltas=neg_deltas, + ) + + def observe(self, amount: float): + with self._lock: + self._count += 1 + self._sum += amount + key = self._native_histogram_key(amount) + + if amount > self._zero_threshold: + self._positive_buckets[key] += 1 + elif amount < -self._zero_threshold: + self._negative_buckets[key] += 1 + else: + self._zero_bucket+=1 + + # Reduce resolution until the number of buckets is below the limit. + while len(self._positive_buckets) + len(self._negative_buckets) > self._max_buckets: + if not self._double_bucket_width(): + break + + def _native_histogram_key(self, amount: float) -> float: + if self._schema is None: + raise ValueError("only available for native histograms") + + is_inf = False + if math.isinf(amount): + if amount > 0: + amount = _DOUBLE_MAX + else: + amount = -_DOUBLE_MAX + is_inf = True + + frac, exp = math.frexp(abs(amount)) + if self._schema > 0: + bounds = _NATIVE_HISTOGRAM_BOUNDS[self._schema] + key = bisect_left(bounds, frac) + (exp-1)*len(bounds) + else: + key = exp + if frac == 0.5: + key-=1 + offset = (1 << -self._schema) - 1 + key = (key + offset) >> -self._schema + + if is_inf: + key+=1 + + return key + + def _double_bucket_width(self) -> bool: + if self._schema == -4: + return False + + self._schema-=1 + tmp = Counter() + for k, v in self._positive_buckets.items(): + if k > 0: + k += 1 + k //= 2 + tmp[k] += v + self._positive_buckets = tmp + + tmp = Counter() + for k, v in self._negative_buckets.items(): + if k > 0: + k += 1 + k //= 2 + tmp[k] += v + self._negative_buckets = tmp + return True + def MultiProcessValue(process_identifier=os.getpid): """Returns a MmapedValue class based on a process_identifier function. diff --git a/tests/test_core.py b/tests/test_core.py index 284bce09..53543059 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,4 +1,5 @@ from concurrent.futures import ThreadPoolExecutor +import math import os import time import unittest @@ -14,6 +15,7 @@ ) from prometheus_client.decorator import getargspec from prometheus_client.metrics import _get_use_created +from prometheus_client.samples import BucketSpan, NativeHistogram from prometheus_client.validation import ( disable_legacy_validation, enable_legacy_validation, ) @@ -527,6 +529,268 @@ def test_exemplar_too_long(self): }) +@pytest.mark.parametrize( + "kwargs,observations,expected", + [ + pytest.param({}, [1, 2, 3], None, id="no sparse buckets"), + pytest.param( + {"native_histogram_initial_schema": 3}, + [], + NativeHistogram( + count_value=0, + sum_value=0, + schema=3, + zero_threshold=2.938735877055719e-39, + zero_count=0, + ), + id="no observations", + ), + pytest.param( + {"native_histogram_initial_schema": 3}, + [0, 1, 2, 3], + NativeHistogram( + count_value=4, + sum_value=6.0, + schema=3, + zero_threshold=2.938735877055719e-39, + zero_count=1, + pos_spans=[ + BucketSpan(offset=0, length=1), + BucketSpan(offset=7, length=1), + BucketSpan(offset=4, length=1), + ], + pos_deltas=[1, 0, 0], + ), + id="schema 3", + ), + pytest.param( + {"native_histogram_initial_schema": 2}, + [0, 1, 1.2, 1.4, 1.8, 2], + NativeHistogram( + count_value=6, + sum_value=7.4, + schema=2, + zero_threshold=2.938735877055719e-39, + zero_count=1, + pos_spans=[ + BucketSpan(offset=0, length=5), + ], + pos_deltas=[1, -1, 2, -2, 2], + ), + id="schema 2", + ), + pytest.param( + {"native_histogram_initial_schema": -1}, + [ + 0.0156251, + 0.0625, # Bucket -2: (0.015625, 0.0625) + 0.1, + 0.25, # Bucket -1: (0.0625, 0.25] + 0.5, + 1, # Bucket 0: (0.25, 1] + 1.5, + 2, + 3, + 3.5, # Bucket 1: (1, 4] + 5, + 6, + 7, # Bucket 2: (4, 16] + 33.33, # Bucket 3: (16, 64] + ], + NativeHistogram( + count_value=14, + sum_value=63.2581251, + schema=-1, + zero_threshold=2.938735877055719e-39, + zero_count=0, + pos_spans=[ + BucketSpan(offset=-2, length=6), + ], + pos_deltas=[2, 0, 0, 2, -1, -2], + ), + id="schema -1", + ), + pytest.param( + {"native_histogram_initial_schema": -2}, + [ + 0.0156251, + 0.0625, # Bucket -1: (0.015625, 0.0625] + 0.1, + 0.25, + 0.5, + 1, # Bucket 0: (0.0625, 1] + 1.5, + 2, + 3, + 3.5, + 5, + 6, + 7, # Bucket 1: (1, 16] + 33.33, # Bucket 2: (16, 256] + ], + NativeHistogram( + count_value=14, + sum_value=63.2581251, + schema=-2, + zero_threshold=2.938735877055719e-39, + zero_count=0, + pos_spans=[ + BucketSpan(offset=-1, length=4), + ], + pos_deltas=[2, 2, 3, -6], + ), + id="schema -2", + ), + pytest.param( + {"native_histogram_initial_schema": 2}, + [0, -1, -1.2, -1.4, -1.8, -2], + NativeHistogram( + count_value=6, + sum_value=-7.4, + schema=2, + zero_threshold=2.938735877055719e-39, + zero_count=1, + neg_spans=[ + BucketSpan(offset=0, length=5), + ], + neg_deltas=[1, -1, 2, -2, 2], + ), + id="negative buckets", + ), + pytest.param( + {"native_histogram_initial_schema": 2}, + [0, -1, -1.2, -1.4, -1.8, -2, 1, 1.2, 1.4, 1.8, 2], + NativeHistogram( + count_value=11, + sum_value=0.0, + schema=2, + zero_threshold=2.938735877055719e-39, + zero_count=1, + pos_spans=[ + BucketSpan(offset=0, length=5), + ], + pos_deltas=[1, -1, 2, -2, 2], + neg_spans=[ + BucketSpan(offset=0, length=5), + ], + neg_deltas=[1, -1, 2, -2, 2], + ), + id="negative and positive buckets", + ), + pytest.param( + { + "native_histogram_initial_schema": 2, + "native_histogram_zero_threshold": 1.4, + }, + [0, -1, -1.2, -1.4, -1.8, -2, 1, 1.2, 1.4, 1.8, 2], + NativeHistogram( + count_value=11, + sum_value=0.0, + schema=2, + zero_threshold=1.4, + zero_count=7, + pos_spans=[ + BucketSpan(offset=4, length=1), + ], + pos_deltas=[2], + neg_spans=[ + BucketSpan(offset=4, length=1), + ], + neg_deltas=[2], + ), + id="wide zero bucket", + ), + pytest.param( + { + "native_histogram_initial_schema": 2, + }, + [0, 1, 1.2, 1.4, 1.8, 2, math.inf], + NativeHistogram( + count_value=7, + sum_value=math.inf, + schema=2, + zero_threshold=2.938735877055719e-39, + zero_count=1, + pos_spans=[ + BucketSpan(offset=0, length=5), + BucketSpan(offset=4092, length=1), + ], + pos_deltas=[1, -1, 2, -2, 2, -1], + ), + id="+Inf observation", + ), + pytest.param( + { + "native_histogram_initial_schema": 2, + }, + [0, 1, 1.2, 1.4, 1.8, 2, -math.inf], + NativeHistogram( + count_value=7, + sum_value=-math.inf, + schema=2, + zero_threshold=2.938735877055719e-39, + zero_count=1, + pos_spans=[ + BucketSpan(offset=0, length=5), + ], + pos_deltas=[1, -1, 2, -2, 2], + neg_spans=[ + BucketSpan(offset=4097, length=1), + ], + neg_deltas=[1], + ), + id="-Inf observation", + ), + pytest.param( + { + "native_histogram_initial_schema": 2, + "native_histogram_max_buckets": 4, + }, + [0, 1, 1.1, 1.2, 1.4, 1.8, 2, 3], + NativeHistogram( + count_value=8, + sum_value=11.5, + schema=1, + zero_threshold=2.938735877055719e-39, + zero_count=1, + pos_spans=[ + BucketSpan(offset=0, length=5), + ], + pos_deltas=[1, 2, -1, -2, 1], + ), + id="buckets limited by halving resolution", + ), + pytest.param( + { + "native_histogram_initial_schema": 2, + "native_histogram_max_buckets": 4, + }, + [0, -1, -1.1, -1.2, -1.4, -1.8, -2, -3], + NativeHistogram( + count_value=8, + sum_value=-11.5, + schema=1, + zero_threshold=2.938735877055719e-39, + zero_count=1, + neg_spans=[ + BucketSpan(offset=0, length=5), + ], + neg_deltas=[1, 2, -1, -2, 1], + ), + id="buckets limited by halving resolution, negative observations", + ), + ], +) +def test_native_histograms(kwargs, observations, expected): + registry = CollectorRegistry() + h = Histogram("hist", "help", registry=registry, **kwargs) + for obs in observations: + h.observe(obs) + + result = registry.get_native_histogram_value("hist") + assert expected == result + + class TestInfo(unittest.TestCase): def setUp(self): self.registry = CollectorRegistry()