metrics: add full duration metric

skynet/modules/monitoring.py

@@ -28,6 +28,14 @@
     buckets=[5**n for n in range(4)],
 )
 
+SUMMARY_FULL_DURATION_METRIC = Histogram(
+    'summary_full_duration_seconds',
+    documentation='Measures the duration of the summary / action items since they were submitted until they are done in seconds',
+    namespace=PROMETHEUS_NAMESPACE,
+    subsystem=PROMETHEUS_SUMMARIES_SUBSYSTEM,
+    buckets=[5**n for n in range(4)],
+)
+
 SUMMARY_TIME_IN_QUEUE_METRIC = Histogram(
     'summary_queue_time_seconds',
     documentation='Measures the time spent in the queue in seconds',

skynet/modules/ttt/summaries/jobs.py

@@ -11,6 +11,7 @@
     OPENAI_API_RESTART_COUNTER,
     SUMMARY_DURATION_METRIC,
     SUMMARY_ERROR_COUNTER,
+    SUMMARY_FULL_DURATION_METRIC,
     SUMMARY_INPUT_LENGTH_METRIC,
     SUMMARY_QUEUE_SIZE_METRIC,
     SUMMARY_TIME_IN_QUEUE_METRIC,
@@ -163,9 +164,11 @@ async def update_done_job(job: Job, result: str, processor: Processors, has_fail
 
     if updated_job.status != JobStatus.SKIPPED:
         SUMMARY_DURATION_METRIC.observe(updated_job.computed_duration)
+        SUMMARY_FULL_DURATION_METRIC.observe(updated_job.computed_full_duration)
         SUMMARY_INPUT_LENGTH_METRIC.observe(len(updated_job.payload.text))
 
         log.info(f"Job {updated_job.id} duration: {updated_job.computed_duration} seconds")
+        log.info(f"Job {updated_job.id} full duration: {updated_job.computed_full_duration} seconds")
 
 
 async def _run_job(job: Job) -> None:

skynet/modules/ttt/summaries/v1/models.py

@@ -85,6 +85,14 @@ def computed_duration(self) -> float:
 
         return 0.0
 
+    @computed_field
+    @property
+    def computed_full_duration(self) -> float:
+        if self.end:
+            return round(self.end - self.created, 3)
+
+        return 0.0
+
 
 class JobId(BaseModel):
     id: str