andreacavalli/scylla-jmx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge "Adding estimated histogram support for nodetool cfhistogram" from Amnon

Browse Source 
"This series together with the cfhistogram series in scylla adds the
 missing functionality so that nodetoold cfhistogram would work.

 After both series will be apply an execution example is:

 ./bin/nodetool cfhistograms keyspace1 standard1
 keyspace1/standard1 histograms
 Percentile  SSTables     Write Latency      Read Latency    Partition Size        Cell Count
                               (micros)          (micros)           (bytes)
 50%             0.00           6866.00        4866323.00               310                 5
 75%             0.00           8239.00       10090808.00               310                 5
 95%             0.00          20501.00       17436917.00               310                 5
 98%             0.00          35425.00       25109160.00               310                 5
 99%             0.00          51012.00       25109160.00               310                 5
 Min             0.00           2300.00         654950.00               259                 5
 Max             0.00       20924300.00       25109160.00               310                 5"
This commit is contained in:
Pekka Enberg 2015-10-21 14:26:40 +03:00
commit 8e0fb98fc4
8 changed files with 221 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/main/java/com/cloudius/urchin/api/APIClient.java
View File

@ -26,6 +26,7 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.UriBuilder;
import com.cloudius.urchin.utils.EstimatedHistogram;
import com.cloudius.urchin.utils.SnapshotDetailsTabularData;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.WebResource;
@ -50,13 +51,23 @@ public class APIClient {
}
return key;
}
String getFromCache(String key, long duration) {
String getStringFromCache(String key, long duration) {
if (key == null) {
return null;
}
CacheEntry value = cache.get(key);
return (value!= null && value.valid(duration))? value.value : null;
return (value!= null && value.valid(duration))? value.stringValue() : null;
}
EstimatedHistogram getEstimatedHistogramFromCache(String key, long duration) {
if (key == null) {
return null;
}
CacheEntry value = cache.get(key);
return (value!= null && value.valid(duration))? value.getEstimatedHistogram() : null;
}
JsonReaderFactory factory = Json.createReaderFactory(null);
private static final java.util.logging.Logger logger = java.util.logging.Logger
.getLogger(APIClient.class.getName());
@ -131,7 +142,7 @@ public class APIClient {
return "";
}
String key = getCacheKey(string, queryParams, duration);
String res = getFromCache(key, duration);
String res = getStringFromCache(key, duration);
if (res != null) {
return res;
}
@ -611,6 +622,20 @@ public class APIClient {
return getHistogramValue(url, null);
}
public EstimatedHistogram getEstimatedHistogram(String string,
MultivaluedMap<String, String> queryParams, long duration) {
String key = getCacheKey(string, queryParams, duration);
EstimatedHistogram res = getEstimatedHistogramFromCache(key, duration);
if (res != null) {
return res;
}
res = new EstimatedHistogram(getEstimatedHistogramAsLongArrValue(string, queryParams));
if (duration > 0) {
cache.put(key, new CacheEntry(res));
}
return res;
}
public long[] getEstimatedHistogramAsLongArrValue(String string,
MultivaluedMap<String, String> queryParams) {
JsonObject obj = getJsonObj(string, queryParams);

34
src/main/java/com/cloudius/urchin/api/CacheEntry.java
View File

@ -1,20 +1,46 @@
/*
* Copyright 2015 Cloudius Systems
* Copyright (C) 2015 ScyllaDB
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
package com.cloudius.urchin.api;
import com.cloudius.urchin.utils.EstimatedHistogram;
public class CacheEntry {
long time;
public String value;
Object value;
CacheEntry(String value) {
CacheEntry(Object res) {
time = System.currentTimeMillis();
this.value = value;
this.value = res;
}
public boolean valid(long duration) {
return (System.currentTimeMillis() - time) < duration;
}
public String stringValue() {
return (String) value;
}
public EstimatedHistogram getEstimatedHistogram() {
return (EstimatedHistogram)value;
}
}

6
src/main/java/com/cloudius/urchin/utils/EstimatedHistogram.java
View File

@ -67,6 +67,12 @@ public class EstimatedHistogram {
buckets = new AtomicLongArray(bucketData);
}
public EstimatedHistogram(long[] bucketData) {
bucketOffsets = newOffsets(bucketData.length - 1);
buckets = new AtomicLongArray(bucketData);
}
private static long[] newOffsets(int size) {
long[] result = new long[size];
long last = 1;

62
src/main/java/com/cloudius/urchin/utils/RecentEstimatedHistogram.java Normal file
View File

@ -0,0 +1,62 @@
package com.cloudius.urchin.utils;
/*
* Copyright (C) 2015 ScyllaDB
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
/**
*
* RecentEstimatedHistogram In the (deprecated) 'recent' functionality, each
* call to get the values cleans the value.
*
* The RecentEstimatedHistogram support recent call to EstimatedHistogram.
* It holds the latest total values and a call to getBuckets return the delta.
*
*/
public class RecentEstimatedHistogram extends EstimatedHistogram {
public RecentEstimatedHistogram() {
}
public RecentEstimatedHistogram(int bucketCount) {
super(bucketCount);
}
public RecentEstimatedHistogram(long[] offsets, long[] bucketData) {
super(offsets, bucketData);
}
/**
* Set the current buckets to new value and return the delta from the last
* getBuckets call
*
* @param bucketData
* - new bucket value
* @return a long[] containing the current histogram difference buckets
*/
public long[] getBuckets(long[] bucketData) {
final int len = buckets.length();
long[] rv = new long[len];
for (int i = 0; i < len; i++) {
rv[i] = bucketData[i];
rv[i] -= buckets.getAndSet(i, bucketData[i]);
}
return rv;
}
}

32
src/main/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -172,7 +172,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long getMemtableColumnsCount() {
log(" getMemtableColumnsCount()");
return c.getLongValue("");
return metric.memtableColumnsCount.value();
}
/**
@ -196,7 +196,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getRecentSSTablesPerReadHistogram() {
log(" getRecentSSTablesPerReadHistogram()");
return c.getLongArrValue("");
return metric.getRecentSSTablesPerRead();
}
/**
@ -206,7 +206,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getSSTablesPerReadHistogram() {
log(" getSSTablesPerReadHistogram()");
return c.getEstimatedHistogramAsLongArrValue("/column_family/metrics/sstables_per_read_histogram/" +getCFName());
return metric.sstablesPerRead.getBuckets(false);
}
/**
@ -236,7 +236,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getLifetimeReadLatencyHistogramMicros() {
log(" getLifetimeReadLatencyHistogramMicros()");
return c.getLongArrValue("");
return metric.readLatency.totalLatencyHistogram.getBuckets(false);
}
/**
@ -246,7 +246,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getRecentReadLatencyHistogramMicros() {
log(" getRecentReadLatencyHistogramMicros()");
return c.getLongArrValue("");
return metric.readLatency.getRecentLatencyHistogram();
}
/**
@ -256,7 +256,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public double getRecentReadLatencyMicros() {
log(" getRecentReadLatencyMicros()");
return c.getDoubleValue("");
return metric.readLatency.getRecentLatency();
}
/**
@ -286,7 +286,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getLifetimeWriteLatencyHistogramMicros() {
log(" getLifetimeWriteLatencyHistogramMicros()");
return c.getLongArrValue("");
return metric.writeLatency.totalLatencyHistogram.getBuckets(false);
}
/**
@ -296,7 +296,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getRecentWriteLatencyHistogramMicros() {
log(" getRecentWriteLatencyHistogramMicros()");
return c.getLongArrValue("");
return metric.writeLatency.getRecentLatencyHistogram();
}
/**
@ -306,7 +306,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public double getRecentWriteLatencyMicros() {
log(" getRecentWriteLatencyMicros()");
return c.getDoubleValue("");
return metric.writeLatency.getRecentLatency();
}
/**
@ -594,7 +594,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getEstimatedRowSizeHistogram() {
log(" getEstimatedRowSizeHistogram()");
return c.getEstimatedHistogramAsLongArrValue("/column_family/metrics/estimated_row_size_histogram/" + getCFName());
return metric.estimatedRowSizeHistogram.value();
}
/**
@ -603,7 +603,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
@Deprecated
public long[] getEstimatedColumnCountHistogram() {
log(" getEstimatedColumnCountHistogram()");
return c.getLongArrValue("/column_family/metrics/estimated_column_count_histogram/" + getCFName());
return metric.estimatedColumnCountHistogram.value();
}
/**
@ -696,35 +696,35 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean {
public long getRangeCount() {
// TODO Auto-generated method stub
log("getRangeCount()");
return c.getLongValue("");
return metric.rangeLatency.latency.count();
}
@Override
public long getTotalRangeLatencyMicros() {
// TODO Auto-generated method stub
log("getTotalRangeLatencyMicros()");
return c.getLongValue("");
return metric.rangeLatency.totalLatency.count();
}
@Override
public long[] getLifetimeRangeLatencyHistogramMicros() {
// TODO Auto-generated method stub
log("getLifetimeRangeLatencyHistogramMicros()");
return c.getLongArrValue("");
return metric.rangeLatency.totalLatencyHistogram.getBuckets(false);
}
@Override
public long[] getRecentRangeLatencyHistogramMicros() {
// TODO Auto-generated method stub
log("getRecentRangeLatencyHistogramMicros()");
return c.getLongArrValue("");
return metric.rangeLatency.getRecentLatencyHistogram();
}
@Override
public double getRecentRangeLatencyMicros() {
// TODO Auto-generated method stub
log("getRecentRangeLatencyMicros()");
return c.getDoubleValue("");
return metric.rangeLatency.getRecentLatency();
}
@Override

15
src/main/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java
View File

@ -34,6 +34,7 @@ import com.cloudius.urchin.api.APIClient;
import com.cloudius.urchin.metrics.APIMetrics;
import com.cloudius.urchin.metrics.MetricNameFactory;
import com.cloudius.urchin.utils.EstimatedHistogram;
import com.cloudius.urchin.utils.RecentEstimatedHistogram;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.yammer.metrics.Metrics;
@ -166,10 +167,10 @@ public class ColumnFamilyMetrics {
// for backward compatibility
@Deprecated
public final EstimatedHistogram sstablesPerRead = new EstimatedHistogram(35);
public final EstimatedHistogramWrapper sstablesPerRead;
// it should not be called directly
@Deprecated
public final EstimatedHistogram recentSSTablesPerRead = new EstimatedHistogram(
35);
protected final RecentEstimatedHistogram recentSSTablesPerRead = new RecentEstimatedHistogram(35);
private String cfName;
public final static LatencyMetrics globalReadLatency = new LatencyMetrics(
@ -362,6 +363,7 @@ public class ColumnFamilyMetrics {
+ cfName, factory, "CasPropose");
casCommit = new LatencyMetrics("/column_family/metrics/cas_commit/"
+ cfName, factory, "CasCommit");
sstablesPerRead = new EstimatedHistogramWrapper("/column_family/metrics/sstables_per_read_histogram/" + cfName);
}
/**
@ -508,7 +510,7 @@ public class ColumnFamilyMetrics {
/**
* Registers a metric to be removed when unloading CF.
*
*
* @return true if first time metric with that name has been registered
*/
private boolean register(String name, Metric metric) {
@ -519,6 +521,11 @@ public class ColumnFamilyMetrics {
return ret;
}
public long[] getRecentSSTablesPerRead() {
return recentSSTablesPerRead
.getBuckets(sstablesPerRead.getBuckets(false));
}
public class ColumnFamilyHistogram {
public final Histogram[] all;
public final Histogram cf;

55
src/main/java/org/apache/cassandra/metrics/EstimatedHistogramWrapper.java Normal file
View File

@ -0,0 +1,55 @@
package org.apache.cassandra.metrics;
/*
* Copyright (C) 2015 ScyllaDB
*/
/*
* This file is part of Scylla.
*
* Scylla is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Scylla is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Scylla. If not, see <http://www.gnu.org/licenses/>.
*/
import javax.ws.rs.core.MultivaluedMap;
import com.cloudius.urchin.api.APIClient;
import com.cloudius.urchin.utils.EstimatedHistogram;
public class EstimatedHistogramWrapper {
private APIClient c = new APIClient();
private String url;
private MultivaluedMap<String, String> queryParams;
private static final int DURATION = 50;
private int duration;
public EstimatedHistogramWrapper(String url, MultivaluedMap<String, String> queryParams, int duration) {
this.url = url;
this.queryParams = queryParams;
this.duration = duration;
}
public EstimatedHistogramWrapper(String url) {
this(url, null, DURATION);
}
public EstimatedHistogramWrapper(String url, int duration) {
this(url, null, duration);
}
public EstimatedHistogram get() {
return c.getEstimatedHistogram(url, queryParams, duration);
}
public long[] getBuckets(boolean reset) {
return get().getBuckets(reset);
}
}

13
src/main/java/org/apache/cassandra/metrics/LatencyMetrics.java
View File

@ -29,6 +29,8 @@ import java.util.concurrent.TimeUnit;
import com.cloudius.urchin.metrics.APIMetrics;
import com.cloudius.urchin.metrics.DefaultNameFactory;
import com.cloudius.urchin.metrics.MetricNameFactory;
import com.cloudius.urchin.utils.EstimatedHistogram;
import com.cloudius.urchin.utils.RecentEstimatedHistogram;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.yammer.metrics.core.Counter;
@ -49,6 +51,12 @@ public class LatencyMetrics {
protected final MetricNameFactory factory;
protected final String namePrefix;
@Deprecated public EstimatedHistogramWrapper totalLatencyHistogram;
/*
* It should not be called directly, use the getRecentLatencyHistogram
*/
@Deprecated protected final RecentEstimatedHistogram recentLatencyHistogram = new RecentEstimatedHistogram();
protected long lastLatency;
protected long lastOpCount;
@ -106,6 +114,7 @@ public class LatencyMetrics {
TimeUnit.MICROSECONDS, TimeUnit.SECONDS);
totalLatency = APIMetrics.newCounter(url + paramName,
factory.createMetricName(namePrefix + "TotalLatency"));
totalLatencyHistogram = new EstimatedHistogramWrapper(url + "/estimated_histogram" + paramName);
}
/**
@ -155,4 +164,8 @@ public class LatencyMetrics {
lastOpCount = ops;
}
}
public long[] getRecentLatencyHistogram() {
return recentLatencyHistogram.getBuckets(totalLatencyHistogram.getBuckets(false));
}
}