Monitor and Message improvments, include Tablet Recovery information

core/src/main/java/org/apache/accumulo/core/metadata/schema/filters/NoCurrentLocationFilter.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema.filters;
+
+import java.util.Set;
+import java.util.function.Predicate;
+
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+
+public class NoCurrentLocationFilter extends HasCurrentFilter {
+
+  @Override
+  public Set<ColumnType> getColumns() {
+    return super.getColumns();
+  }
+
+  @Override
+  protected Predicate<TabletMetadata> acceptTablet() {
+    return super.acceptTablet().negate();
+  }
+
+}

core/src/main/java/org/apache/accumulo/core/metrics/Metric.java

@@ -392,15 +392,30 @@ public enum Metric {
       MetricDocSection.MANAGER, "Manager Goal State", null, NUMBER),
 
   // Recovery Metrics
-  RECOVERIES_IN_PROGRESS("accumulo.recoveries.in.progress", MetricType.GAUGE,
-      "The number of recoveries in progress.", MetricDocSection.GENERAL_SERVER,
-      "Tablet Recoveries In Progress", null, NUMBER),
-  RECOVERIES_LONGEST_RUNTIME("accumulo.recoveries.runtime.longest", MetricType.GAUGE,
-      "The time (in milliseconds) of the longest running recovery.",
+  RECOVERIES_SORTS_IN_PROGRESS("accumulo.recoveries.sorts.in.progress", MetricType.GAUGE,
+      "The number of log sorts in progress.", MetricDocSection.GENERAL_SERVER,
+      "Log Sorts In Progress", null, NUMBER),
+  RECOVERIES_SORTS_LONGEST_RUNTIME("accumulo.recoveries.sorts.runtime.longest", MetricType.GAUGE,
+      "The time (in milliseconds) of the longest running log sort.",
       MetricDocSection.GENERAL_SERVER, "Tablet Recovery Longest Time", null, DURATION),
-  RECOVERIES_AVG_PROGRESS("accumulo.recoveries.avg.progress", MetricType.GAUGE,
-      "The average percentage (0.0 - 99.9) of the in progress recoveries.",
+  RECOVERIES_SORTS_AVG_PROGRESS("accumulo.recoveries.sorts.avg.progress", MetricType.GAUGE,
+      "The average percentage (0.0 - 99.9) of the in progress log sorts.",
       MetricDocSection.GENERAL_SERVER, "Tablet Recovery Avg Percent Complete", null, PERCENT),
+  RECOVERIES_TABLETS_STARTED("accumulo.recoveries.tablets.started", MetricType.GAUGE,
+      "The number of tablet recoveries started", MetricDocSection.GENERAL_SERVER,
+      "Tablet Recoveries Started", null, NUMBER),
+  RECOVERIES_TABLETS_COMPLETED("accumulo.recoveries.tablets.completed", MetricType.GAUGE,
+      "The number of tablet recoveries completed", MetricDocSection.GENERAL_SERVER,
+      "Tablet Recoveries Completed", null, NUMBER),
+  RECOVERIES_TABLETS_FAILED("accumulo.recoveries.tablets.failed", MetricType.GAUGE,
+      "The number of tablet recoveries failed", MetricDocSection.GENERAL_SERVER,
+      "Tablet Recoveries Failed", null, NUMBER),
+  RECOVERIES_TABLETS_IN_PROGRESS("accumulo.recoveries.tablets.in.progress", MetricType.GAUGE,
+      "The number of tablet recoveries in progress", MetricDocSection.GENERAL_SERVER,
+      "Tablet Recoveries In Progress", null, NUMBER),
+  RECOVERIES_TABLETS_MUTATIONS_REPLAYED("accumulo.recoveries.tablets.mutations.replayed",
+      MetricType.GAUGE, "The number of mutations replayed for tablet recovery",
+      MetricDocSection.GENERAL_SERVER, "Tablet Recoveries Mutations Replayed", null, NUMBER),
 
   // Executor metrics
   EXECUTOR_COMPLETED("executor.completed", MetricType.FUNCTION_COUNTER,
@@ -555,17 +570,20 @@ public static Metric fromName(String name) {
   public static Set<String> getMonitorExclusions(ServerId.Type serverType) {
     switch (serverType) {
       case COMPACTOR:
-        return Set.of(MINC_PAUSED.getName(), RECOVERIES_AVG_PROGRESS.getName(),
-            RECOVERIES_IN_PROGRESS.getName(), RECOVERIES_LONGEST_RUNTIME.getName());
+        return Set.of(MINC_PAUSED.getName(), RECOVERIES_TABLETS_STARTED.getName(),
+            RECOVERIES_TABLETS_COMPLETED.getName(), RECOVERIES_TABLETS_FAILED.getName(),
+            RECOVERIES_TABLETS_IN_PROGRESS.getName(),
+            RECOVERIES_TABLETS_MUTATIONS_REPLAYED.getName());
       case GARBAGE_COLLECTOR:
         return Set.of();
       case MANAGER:
         return Set.of();
       case MONITOR:
         return Set.of();
       case SCAN_SERVER:
-        return Set.of(RECOVERIES_AVG_PROGRESS.getName(), RECOVERIES_IN_PROGRESS.getName(),
-            RECOVERIES_LONGEST_RUNTIME.getName());
+        return Set.of(RECOVERIES_TABLETS_STARTED.getName(), RECOVERIES_TABLETS_COMPLETED.getName(),
+            RECOVERIES_TABLETS_FAILED.getName(), RECOVERIES_TABLETS_IN_PROGRESS.getName(),
+            RECOVERIES_TABLETS_MUTATIONS_REPLAYED.getName());
       case TABLET_SERVER:
         return Set.of(MAJC_PAUSED.getName());
       default:

server/monitor/src/main/java/org/apache/accumulo/monitor/next/Endpoints.java

@@ -59,6 +59,7 @@
 import org.apache.accumulo.monitor.next.SystemInformation.CompactionTableSummary;
 import org.apache.accumulo.monitor.next.SystemInformation.MessageCategory;
 import org.apache.accumulo.monitor.next.SystemInformation.MessagePriority;
+import org.apache.accumulo.monitor.next.SystemInformation.RecoveryInformation;
 import org.apache.accumulo.monitor.next.SystemInformation.TableSummary;
 import org.apache.accumulo.monitor.next.SystemInformation.TimeOrderedRunningCompactionSet;
 import org.apache.accumulo.monitor.next.deployment.DeploymentOverview;
@@ -419,6 +420,14 @@ public List<TabletInformation> getTablets(@PathParam(TABLEID_PARAM_KEY) String t
     return ti;
   }
 
+  @GET
+  @Path("recovery")
+  @Produces(MediaType.APPLICATION_JSON)
+  @Description("Returns information about tservers performing recovery and tablets needing recovery")
+  public RecoveryInformation getTabletRecoveries() {
+    return monitor.getInformationFetcher().getSummaryForEndpoint().getRecoveryInformation();
+  }
+
   @GET
   @Path("deployment")
   @Produces(MediaType.APPLICATION_JSON)