Skip to content

Commit dc5422a

Browse files
lksfrnzlksfrnz
committed
Added Histogram Metric Type
1 parent d8ae46b commit dc5422a

File tree

13 files changed

+586
-27
lines changed

13 files changed

+586
-27
lines changed

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/AggregationType.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
public enum AggregationType {
2020
LIST(0),
2121
STATISTIC_SET(1),
22+
HISTOGRAM(2),
2223
UNKNOWN_TO_SDK_VERSION(-1);
2324

2425
private final int value;

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/Histogram.java

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
*
3+
* Licensed under the Apache License, Version 2.0 (the "License").
4+
* You may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
package software.amazon.cloudwatchlogs.emf.model;
17+
18+
import com.fasterxml.jackson.annotation.JsonIgnore;
19+
import com.fasterxml.jackson.annotation.JsonProperty;
20+
import java.util.ArrayList;
21+
import java.util.Collections;
22+
import java.util.HashMap;
23+
import java.util.List;
24+
import java.util.Map;
25+
import software.amazon.cloudwatchlogs.emf.Constants;
26+
import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException;
27+
28+
/** Histogram metric type */
29+
class Histogram extends Statistics {
30+
Histogram(List<Double> values, List<Integer> counts) throws IllegalArgumentException {
31+
if (counts.size() != values.size()) {
32+
throw new IllegalArgumentException("Counts and values must have the same size");
33+
}
34+
35+
if (validSize(counts.size())) {
36+
throw new IllegalArgumentException(
37+
String.format(
38+
"Histogram provided with %d bins but CloudWatch will drop Histograms with more than %d bins",
39+
counts.size(), Constants.MAX_DATAPOINTS_PER_METRIC));
40+
}
41+
42+
this.max = Collections.max(values);
43+
this.min = Collections.min(values);
44+
this.count = counts.stream().mapToInt(Integer::intValue).sum();
45+
this.sum = 0d;
46+
for (int i = 0; i < counts.size(); i++) {
47+
this.sum += values.get(i) * counts.get(i);
48+
}
49+
this.counts = counts;
50+
this.values = values;
51+
}
52+
53+
Histogram() {
54+
count = 0;
55+
sum = 0.;
56+
values = new ArrayList<>();
57+
counts = new ArrayList<>();
58+
};
59+
60+
@JsonProperty("Values")
61+
public List<Double> values;
62+
63+
@JsonProperty("Counts")
64+
public List<Integer> counts;
65+
66+
@JsonIgnore private boolean reduced = false;
67+
68+
@JsonIgnore private static final double EPSILON = 0.1;
69+
@JsonIgnore private static final double BIN_SIZE = Math.log(1 + EPSILON);
70+
@JsonIgnore private final Map<Double, Integer> buckets = new HashMap<>();
71+
72+
/**
73+
* @param value the value to add to the histogram
74+
* @throws InvalidMetricException if adding this value would increase the number of bins in the
75+
* histogram to more than {@value Constants#MAX_DATAPOINTS_PER_METRIC}
76+
* @see Constants#MAX_DATAPOINTS_PER_METRIC
77+
*/
78+
@Override
79+
void addValue(double value) throws InvalidMetricException {
80+
reduced = false;
81+
super.addValue(value);
82+
83+
double bucket = getBucket(value);
84+
if (!buckets.containsKey(bucket) && validSize(counts.size() + 1)) {
85+
throw new InvalidMetricException(
86+
String.format(
87+
"Adding this value increases the number of bins in this histogram to %d"
88+
+ ", CloudWatch will drop any Histogram metrics with more than %d bins",
89+
buckets.size() + 1, Constants.MAX_DATAPOINTS_PER_METRIC));
90+
}
91+
// Add the value to the appropriate bucket (or create a new bucket if necessary
92+
buckets.compute(
93+
bucket,
94+
(k, v) -> {
95+
if (v == null) {
96+
return 1;
97+
} else {
98+
return v + 1;
99+
}
100+
});
101+
}
102+
103+
/**
104+
* Updates the Values and Counts lists to represent the buckets of this histogram.
105+
*
106+
* @return the reduced histogram
107+
*/
108+
Histogram reduce() {
109+
if (reduced) {
110+
return this;
111+
}
112+
113+
this.values = new ArrayList<>(buckets.size());
114+
this.counts = new ArrayList<>(buckets.size());
115+
116+
for (Map.Entry<Double, Integer> entry : buckets.entrySet()) {
117+
this.values.add(entry.getKey());
118+
this.counts.add(entry.getValue());
119+
}
120+
121+
reduced = true;
122+
return this;
123+
}
124+
125+
/**
126+
* Gets the value of the bucket for the given value.
127+
*
128+
* @param value the value to find the closest bucket for
129+
* @return the value of the bucket the given value goes in
130+
*/
131+
private static double getBucket(double value) {
132+
short index = (short) Math.floor(Math.log(value) / BIN_SIZE);
133+
return Math.exp((index + 0.5) * BIN_SIZE);
134+
}
135+
136+
private boolean validSize(int size) {
137+
return size > Constants.MAX_DATAPOINTS_PER_METRIC;
138+
}
139+
140+
@Override
141+
public boolean equals(Object o) {
142+
if (this == o) return true;
143+
if (o == null || getClass() != o.getClass()) return false;
144+
Histogram that = (Histogram) o;
145+
return count == that.count
146+
&& that.sum.equals(sum)
147+
&& that.max.equals(max)
148+
&& that.min.equals(min)
149+
&& buckets.equals(that.buckets);
150+
}
151+
152+
@Override
153+
public int hashCode() {
154+
return super.hashCode() + buckets.hashCode();
155+
}
156+
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/HistogramMetric.java

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package software.amazon.cloudwatchlogs.emf.model;
18+
19+
import java.util.LinkedList;
20+
import java.util.List;
21+
import java.util.Queue;
22+
import software.amazon.cloudwatchlogs.emf.Constants;
23+
import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException;
24+
25+
/** Represents the Histogram of the EMF schema. */
26+
public class HistogramMetric extends Metric<Histogram> {
27+
28+
HistogramMetric(
29+
Unit unit,
30+
StorageResolution storageResolution,
31+
List<Double> values,
32+
List<Integer> counts)
33+
throws IllegalArgumentException {
34+
this(unit, storageResolution, new Histogram(values, counts));
35+
}
36+
37+
protected HistogramMetric(
38+
String name, Unit unit, StorageResolution storageResolution, Histogram histogram) {
39+
this.unit = unit;
40+
this.storageResolution = storageResolution;
41+
this.values = histogram;
42+
this.name = name;
43+
}
44+
45+
HistogramMetric(Unit unit, StorageResolution storageResolution, Histogram histogram) {
46+
this.unit = unit;
47+
this.storageResolution = storageResolution;
48+
this.values = histogram;
49+
}
50+
51+
@Override
52+
protected Queue<Metric<Histogram>> serialize() throws InvalidMetricException {
53+
// Histograms will be rejected from CWL if they have more than
54+
// Constants.MAX_DATAPOINTS_PER_METRIC number of bins. Unlike MetricDefinition histograms
55+
// cannot be broken into multiple messages therefore an error is raised to let users know
56+
// their message won't be sent otherwise only this metric will be sent
57+
if (isOversized()) {
58+
throw new InvalidMetricException(
59+
String.format(
60+
"Histogram metric, %s, has %d values which exceeds the maximum amount "
61+
+ "of bins allowed, %d, and Histograms cannot be broken into "
62+
+ "multiple metrics therefore it will not be published",
63+
name, values.values.size(), Constants.MAX_DATAPOINTS_PER_METRIC));
64+
}
65+
Queue<Metric<Histogram>> metrics = new LinkedList<>();
66+
metrics.offer(this);
67+
return metrics;
68+
}
69+
70+
@Override
71+
protected boolean isOversized() {
72+
return values.values.size() > Constants.MAX_DATAPOINTS_PER_METRIC;
73+
}
74+
75+
@Override
76+
public boolean hasValidValues() {
77+
return values != null && values.count > 0 && !isOversized();
78+
}
79+
80+
public static HistogramMetricBuilder builder() {
81+
return new HistogramMetricBuilder();
82+
}
83+
84+
public static class HistogramMetricBuilder
85+
extends Metric.MetricBuilder<Histogram, HistogramMetricBuilder> {
86+
87+
@Override
88+
protected HistogramMetricBuilder getThis() {
89+
return this;
90+
}
91+
92+
public HistogramMetricBuilder() {
93+
this.values = new Histogram();
94+
}
95+
96+
@Override
97+
public Histogram getValues() {
98+
return values.reduce();
99+
}
100+
101+
@Override
102+
public HistogramMetricBuilder addValue(double value) {
103+
this.values.addValue(value);
104+
return this;
105+
}
106+
107+
@Override
108+
public HistogramMetric build() {
109+
values.reduce();
110+
if (name == null) {
111+
return new HistogramMetric(unit, storageResolution, values);
112+
}
113+
return new HistogramMetric(name, unit, storageResolution, values);
114+
}
115+
}
116+
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/Metric.java

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import com.fasterxml.jackson.annotation.JsonProperty;
2222
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
2323
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
24-
import java.util.LinkedList;
24+
import java.util.Queue;
2525
import lombok.AccessLevel;
2626
import lombok.Getter;
2727
import lombok.NonNull;
@@ -52,7 +52,9 @@ public abstract class Metric<V> {
5252
@JsonSerialize(using = StorageResolutionSerializer.class)
5353
protected StorageResolution storageResolution = StorageResolution.STANDARD;
5454

55-
@JsonIgnore @Getter protected V values;
55+
@JsonIgnore
56+
@Getter(AccessLevel.PROTECTED)
57+
protected V values;
5658

5759
/** @return the values of this metric formatted to be flushed */
5860
protected Object getFormattedValues() {
@@ -72,7 +74,7 @@ protected Object getFormattedValues() {
7274
* @return a list of metrics based off of the values of this metric that aren't too large for
7375
* CWL
7476
*/
75-
protected abstract LinkedList<Metric> serialize();
77+
protected abstract Queue<Metric<V>> serialize();
7678

7779
public abstract static class MetricBuilder<V, T extends MetricBuilder<V, T>> extends Metric<V> {
7880

@@ -113,7 +115,7 @@ public boolean hasValidValues() {
113115
}
114116

115117
@Override
116-
protected LinkedList<Metric> serialize() {
118+
protected Queue<Metric<V>> serialize() {
117119
return build().serialize();
118120
}
119121

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import java.util.ArrayList;
2020
import java.util.LinkedList;
2121
import java.util.List;
22+
import java.util.Queue;
2223
import lombok.NonNull;
2324
import software.amazon.cloudwatchlogs.emf.Constants;
2425

@@ -43,8 +44,8 @@ private MetricDefinition(
4344
}
4445

4546
@Override
46-
protected LinkedList<Metric> serialize() {
47-
LinkedList<Metric> metrics = new LinkedList<>();
47+
protected Queue<Metric<List<Double>>> serialize() {
48+
Queue<Metric<List<Double>>> metrics = new LinkedList<>();
4849
MetricDefinition metric = this;
4950
while (metric != null) {
5051
metrics.add(metric.getFirstMetricBatch(Constants.MAX_DATAPOINTS_PER_METRIC));

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,20 @@
1818

1919
import com.fasterxml.jackson.annotation.JsonIgnore;
2020
import com.fasterxml.jackson.annotation.JsonProperty;
21-
import java.util.*;
21+
import java.util.ArrayList;
22+
import java.util.Arrays;
23+
import java.util.Collection;
24+
import java.util.Collections;
25+
import java.util.List;
26+
import java.util.Map;
27+
import java.util.Set;
2228
import java.util.concurrent.ConcurrentHashMap;
2329
import java.util.stream.Collectors;
24-
import lombok.*;
30+
import lombok.AccessLevel;
31+
import lombok.AllArgsConstructor;
32+
import lombok.Getter;
33+
import lombok.Setter;
34+
import lombok.With;
2535
import software.amazon.cloudwatchlogs.emf.exception.DimensionSetExceededException;
2636
import software.amazon.cloudwatchlogs.emf.exception.InvalidMetricException;
2737

@@ -33,7 +43,7 @@ class MetricDirective {
3343
@JsonProperty("Namespace")
3444
private String namespace;
3545

36-
@JsonIgnore @Setter @Getter @With private Map<String, Metric> metrics;
46+
@JsonIgnore @Setter @Getter @With private Map<String, Metric<?>> metrics;
3747

3848
@JsonIgnore
3949
@Getter(AccessLevel.PROTECTED)
@@ -97,6 +107,9 @@ void putMetric(
97107
case STATISTIC_SET:
98108
builder = StatisticSet.builder();
99109
break;
110+
case HISTOGRAM:
111+
builder = HistogramMetric.builder();
112+
break;
100113
case LIST:
101114
default:
102115
builder = MetricDefinition.builder();
@@ -124,13 +137,13 @@ void putMetric(
124137
* @param key the name of the metric
125138
* @param value the value of the metric
126139
*/
127-
void setMetric(String key, Metric value) {
140+
void setMetric(String key, Metric<?> value) {
128141
value.setName(key);
129142
metrics.put(key, value);
130143
}
131144

132145
@JsonProperty("Metrics")
133-
Collection<Metric> getAllMetrics() {
146+
Collection<Metric<?>> getAllMetrics() {
134147
return metrics.values();
135148
}
136149

0 commit comments

Comments
 (0)