fix: separate rejected and aborted proving jobs (#20777)

This PR tracks rejected and aborted jobs as separate counters enabling us to setup alerts when any proofs fail. This PR has to be backported otherwise we'll get alerts when jobs get cancelled as well.

Author: alexghr

yarn-project/prover-client/src/proving_broker/proving_broker.ts

@@ -314,7 +314,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr
     // notify listeners of the cancellation
     if (!this.resultsCache.has(id)) {
       this.logger.info(`Cancelling job id=${id}`, { provingJobId: id });
-      await this.#reportProvingJobError(id, 'Aborted', false);
+      await this.#reportProvingJobError(id, 'Aborted', false, undefined, true);
     }
   }
 
@@ -395,6 +395,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr
     err: string,
     retry = false,
     filter?: ProvingJobFilter,
+    aborted = false,
   ): Promise<GetProvingJobResponse | undefined> {
     const info = this.inProgress.get(id);
     const item = this.jobsCache.get(id);
@@ -455,7 +456,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr
     this.promises.get(id)!.resolve(result);
     this.completedJobNotifications.push(id);
 
-    this.instrumentation.incRejectedJobs(item.type);
+    if (aborted) {
+      this.instrumentation.incAbortedJobs(item.type);
+    } else {
+      this.instrumentation.incRejectedJobs(item.type);
+    }
     if (info) {
       const duration = this.msTimeSource() - info.startedAt;
       this.instrumentation.recordJobDuration(item.type, duration);

yarn-project/prover-client/src/proving_broker/proving_broker_instrumentation.ts

@@ -18,6 +18,7 @@ export class ProvingBrokerInstrumentation {
   private activeJobs: ObservableGauge;
   private resolvedJobs: UpDownCounter;
   private rejectedJobs: UpDownCounter;
+  private abortedJobs: UpDownCounter;
   private timedOutJobs: UpDownCounter;
   private cachedJobs: UpDownCounter;
   private totalJobs: UpDownCounter;
@@ -39,6 +40,8 @@ export class ProvingBrokerInstrumentation {
 
     this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs);
 
+    this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs);
+
     this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs);
 
     this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs);
@@ -72,6 +75,12 @@ export class ProvingBrokerInstrumentation {
     });
   }
 
+  incAbortedJobs(proofType: ProvingRequestType) {
+    this.abortedJobs.add(1, {
+      [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],
+    });
+  }
+
   incRetriedJobs(proofType: ProvingRequestType) {
     this.retriedJobs.add(1, {
       [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType],

yarn-project/telemetry-client/src/metrics.ts

@@ -1041,6 +1041,11 @@ export const PROVING_QUEUE_REJECTED_JOBS: MetricDefinition = {
   description: 'Number of rejected proving jobs',
   valueType: ValueType.INT,
 };
+export const PROVING_QUEUE_ABORTED_JOBS: MetricDefinition = {
+  name: 'aztec.proving_queue.aborted_jobs_count',
+  description: 'Number of aborted proving jobs',
+  valueType: ValueType.INT,
+};
 export const PROVING_QUEUE_RETRIED_JOBS: MetricDefinition = {
   name: 'aztec.proving_queue.retried_jobs_count',
   description: 'Number of retried proving jobs',