From ea0c593a756420416f9bc5f8e7d63d10c13261da Mon Sep 17 00:00:00 2001 From: Amnon Heiman Date: Thu, 18 Feb 2016 14:50:19 +0200 Subject: [PATCH] MessagingService: Ignore exception on the dropped messages thread The dropped messages thread pull information from the API, in various scenario it can face a connection problem (specifically on startup and shutdown) or other related exception, when scylla shutds down. It shold ignore the connection problem, as it is been taken care of by another thread that check the status and will shutdown when needed. For other exception, it logs them while continue to connect. Fixes scylladb/scylla#902 Signed-off-by: Amnon Heiman Message-Id: <1455799819-17957-1-git-send-email-amnon@scylladb.com> --- .../cassandra/net/MessagingService.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/cassandra/net/MessagingService.java b/src/main/java/org/apache/cassandra/net/MessagingService.java index b2ec29f..9da5281 100644 --- a/src/main/java/org/apache/cassandra/net/MessagingService.java +++ b/src/main/java/org/apache/cassandra/net/MessagingService.java @@ -31,6 +31,7 @@ import javax.json.JsonArray; import javax.json.JsonObject; import javax.management.MBeanServer; import javax.management.ObjectName; +import javax.ws.rs.ProcessingException; import org.apache.cassandra.metrics.DroppedMessageMetrics; @@ -118,6 +119,8 @@ public final class MessagingService implements MessagingServiceMBean { static MessagingService instance = new MessagingService(); private static final class CheckDroppedMessages extends TimerTask { + int connection_failure = 0; + int report_error = 1; @Override public void run() { if (instance.dropped == null) { @@ -126,11 +129,23 @@ public final class MessagingService implements MessagingServiceMBean { instance.dropped.put(v.name(), new DroppedMessageMetrics(v)); } } - Map val = instance.getDroppedMessages(); - for (String k : val.keySet()) { - APISettableMeter meter = instance.dropped.get(k).getMeter(); - meter.set(val.get(k)); - meter.tick(); + try { + Map val = instance.getDroppedMessages(); + for (String k : val.keySet()) { + APISettableMeter meter = instance.dropped.get(k).getMeter(); + meter.set(val.get(k)); + meter.tick(); + } + connection_failure = 0; + report_error = 1; + } catch (ProcessingException e) { + // Connection problem, No need to do anything, just retry. + } catch (Exception e) { + connection_failure++; + if ((connection_failure & report_error) == report_error) { + logger.info("Dropped messages failed with " + e.getMessage() + " total error reported " + connection_failure); + report_error <<= 1; + } } } }