diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker.ts b/yarn-project/prover-client/src/proving_broker/proving_broker.ts index e9d7c191e977..8d9db0e1bf6e 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker.ts @@ -314,7 +314,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr // notify listeners of the cancellation if (!this.resultsCache.has(id)) { this.logger.info(`Cancelling job id=${id}`, { provingJobId: id }); - await this.#reportProvingJobError(id, 'Aborted', false); + await this.#reportProvingJobError(id, 'Aborted', false, undefined, true); } } @@ -395,6 +395,7 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr err: string, retry = false, filter?: ProvingJobFilter, + aborted = false, ): Promise { const info = this.inProgress.get(id); const item = this.jobsCache.get(id); @@ -455,7 +456,11 @@ export class ProvingBroker implements ProvingJobProducer, ProvingJobConsumer, Pr this.promises.get(id)!.resolve(result); this.completedJobNotifications.push(id); - this.instrumentation.incRejectedJobs(item.type); + if (aborted) { + this.instrumentation.incAbortedJobs(item.type); + } else { + this.instrumentation.incRejectedJobs(item.type); + } if (info) { const duration = this.msTimeSource() - info.startedAt; this.instrumentation.recordJobDuration(item.type, duration); diff --git a/yarn-project/prover-client/src/proving_broker/proving_broker_instrumentation.ts b/yarn-project/prover-client/src/proving_broker/proving_broker_instrumentation.ts index 7bcd5d50c780..f559c247cd29 100644 --- a/yarn-project/prover-client/src/proving_broker/proving_broker_instrumentation.ts +++ b/yarn-project/prover-client/src/proving_broker/proving_broker_instrumentation.ts @@ -18,6 +18,7 @@ export class ProvingBrokerInstrumentation { private activeJobs: ObservableGauge; private resolvedJobs: UpDownCounter; private rejectedJobs: UpDownCounter; + private abortedJobs: UpDownCounter; private timedOutJobs: UpDownCounter; private cachedJobs: UpDownCounter; private totalJobs: UpDownCounter; @@ -39,6 +40,8 @@ export class ProvingBrokerInstrumentation { this.rejectedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_REJECTED_JOBS, provingJobAttrs); + this.abortedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_ABORTED_JOBS, provingJobAttrs); + this.retriedJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_RETRIED_JOBS, provingJobAttrs); this.timedOutJobs = createUpDownCounterWithDefault(meter, Metrics.PROVING_QUEUE_TIMED_OUT_JOBS, provingJobAttrs); @@ -72,6 +75,12 @@ export class ProvingBrokerInstrumentation { }); } + incAbortedJobs(proofType: ProvingRequestType) { + this.abortedJobs.add(1, { + [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType], + }); + } + incRetriedJobs(proofType: ProvingRequestType) { this.retriedJobs.add(1, { [Attributes.PROVING_JOB_TYPE]: ProvingRequestType[proofType], diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index c195ce888502..d30a4185e15b 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -1048,6 +1048,11 @@ export const PROVING_QUEUE_REJECTED_JOBS: MetricDefinition = { description: 'Number of rejected proving jobs', valueType: ValueType.INT, }; +export const PROVING_QUEUE_ABORTED_JOBS: MetricDefinition = { + name: 'aztec.proving_queue.aborted_jobs_count', + description: 'Number of aborted proving jobs', + valueType: ValueType.INT, +}; export const PROVING_QUEUE_RETRIED_JOBS: MetricDefinition = { name: 'aztec.proving_queue.retried_jobs_count', description: 'Number of retried proving jobs',