Stats endpoints for console dashboard

apis/accounts/v1alpha1/billing.proto

@@ -87,7 +87,9 @@ message GetActivePlanRequest {}
 // GetUsageReportRequest requests current usage and, optionally, historical values.
 message GetUsageReportRequest {
   // The number of history days to return. Zero omits history.
-  uint64 history_days = 1 [(buf.validate.field).uint64.lte = 365];
+  uint64 history_days = 1 [(buf.validate.field).uint64.lte = 370]; // maximum of a little over a year
+  // The keys of the metrics to include in the report. If empty, all metrics are included.
+  repeated string metric_keys = 2 [(buf.validate.field).repeated.unique = true];
 }
 
 // UsageReport contains the usage metrics tracked for an organization.

apis/workflows/v1/job.proto

@@ -5,6 +5,7 @@ edition = "2023";
 package workflows.v1;
 
 import "buf/validate/validate.proto";
+import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
 import "tilebox/v1/id.proto";
 import "tilebox/v1/query.proto";
@@ -172,8 +173,14 @@ message QueryFilters {
     required: false
   };
 
-  // Filter by time (or UUIDs encoding a timestamp)
+  // Filters by job submission time. If absent or neither bound is set, submission time is not restricted. If only
+  // start_time is set, it is used as the lower bound. If only end_time is set, it is used as the upper bound. If both
+  // are set, jobs between the two bounds are returned. start_exclusive and end_inclusive control whether each bound is
+  // included.
   tilebox.v1.TimeInterval time_interval = 1;
+  // Filters by job ID. If absent or neither bound is set, job ID is not restricted. If only start_id is set, it is used
+  // as the lower bound. If only end_id is set, it is used as the upper bound. If both are set, jobs between the two
+  // bounds are returned. start_exclusive and end_inclusive control whether each bound is included.
   tilebox.v1.IDInterval id_interval = 2;
 
   // Filter jobs by automations.
@@ -237,6 +244,58 @@ message CloneJobRequest {
   string job_name = 3 [(buf.validate.field).string.min_len = 1];
 }
 
+// GetJobStateCountsRequest requests counts of jobs by their state, optionally filtered by submission time.
+message GetJobStateCountsRequest {
+  // Filters by job submission time. If absent or neither bound is set, submission time is not restricted. If only
+  // start_time is set, it is used as the lower bound. If only end_time is set, it is used as the upper bound. If both
+  // are set, jobs between the two bounds are counted. start_exclusive and end_inclusive control whether each bound is
+  // included.
+  tilebox.v1.TimeInterval time_interval = 1;
+}
+
+// GetJobStateCountsResponse is the response to a GetJobStateCountsRequest, containing counts of jobs by their state.
+message GetJobStateCountsResponse {
+  repeated JobStateCount state_counts = 1;
+  int64 total_jobs = 2;
+}
+
+// JobStateCount is a message that represents the count of jobs for a specific state.
+message JobStateCount {
+  workflows.v1.JobState state = 1;
+  uint64 count = 2;
+}
+
+// GetTaskQueueStatsRequest requests statistics about jobs in the task queue, filtered by submission time.
+message GetTaskQueueStatsRequest {
+  option (buf.validate.message).oneof = {
+    fields: [
+      "job_time_interval",
+      "job_max_age"
+    ]
+    required: false
+  };
+
+  // Filters by job submission time. If absent or neither bound is set, submission time is not restricted. If only
+  // start_time is set, it is used as the lower bound. If only end_time is set, it is used as the upper bound. If both
+  // are set, jobs between the two bounds are considered. start_exclusive and end_inclusive control whether each bound
+  // is included.
+  tilebox.v1.TimeInterval job_time_interval = 1;
+  // Filters to jobs submitted within this duration before the current time.
+  google.protobuf.Duration job_max_age = 2;
+}
+
+// GetTaskQueueStatsResponse represents statistics about jobs and tasks in the task queue.
+message GetTaskQueueStatsResponse {
+  // Jobs in SUBMITTED or STARTED.
+  uint64 waiting_jobs = 1;
+  // Jobs in RUNNING.
+  uint64 running_jobs = 2;
+  // QUEUED tasks across all jobs in SUBMITTED, STARTED, or RUNNING state.
+  uint64 queued_tasks = 3;
+  // The oldest job in the requested interval in SUBMITTED or STARTED state.
+  // Will have the id, name, and submitted_at fields set. If there are no waiting jobs, this field will be absent.
+  Job oldest_waiting_job = 4;
+}
 // A service for interacting with jobs.
 service JobService {
   rpc SubmitJob(SubmitJobRequest) returns (Job);
@@ -248,4 +307,6 @@ service JobService {
   rpc QueryJobs(QueryJobsRequest) returns (QueryJobsResponse);
   rpc GetJobPrototype(GetJobPrototypeRequest) returns (GetJobPrototypeResponse);
   rpc CloneJob(CloneJobRequest) returns (Job);
+  rpc GetJobStateCounts(GetJobStateCountsRequest) returns (GetJobStateCountsResponse);
+  rpc GetTaskQueueStats(GetTaskQueueStatsRequest) returns (GetTaskQueueStatsResponse);
 }

apis/workflows/v1/telemetry.proto

@@ -8,6 +8,25 @@ import "opentelemetry/proto/trace/v1/trace.proto";
 import "tilebox/v1/id.proto";
 import "tilebox/v1/query.proto";
 
+// LogSeverityGroup is an enum that groups log severity levels into broader categories for easier filtering and analysis.
+enum LogSeverityGroup {
+  LOG_SEVERITY_GROUP_UNSPECIFIED = 0;
+  LOG_SEVERITY_GROUP_TRACE = 1;
+  LOG_SEVERITY_GROUP_DEBUG = 2;
+  LOG_SEVERITY_GROUP_INFO = 3;
+  LOG_SEVERITY_GROUP_WARNING = 4;
+
+  // Includes OpenTelemetry ERROR and FATAL severities.
+  LOG_SEVERITY_GROUP_ERROR = 5;
+}
+
+// LogQueryFilters define additional query filters for log records, supported by both the QueryJobLogs and QueryLogsInInterval endpoints.
+// These filters can be used to narrow down the log entries returned by the query.
+message LogQueryFilters {
+  // Filter log entries by severity levels. If not specified, all severity levels will be returned.
+  repeated LogSeverityGroup severity_levels = 1 [(buf.validate.field).repeated.unique = true];
+}
+
 // QueryJobLogsRequest is the request message for querying logs of a specific job, in ascending or descending order,
 // with pagination support.
 message QueryJobLogsRequest {
@@ -19,17 +38,21 @@ message QueryJobLogsRequest {
   tilebox.v1.SortDirection sort_direction = 3;
   // The ID of the task to query logs for. If not specified, logs for the full job are returned.
   tilebox.v1.ID task_id = 4 [features.field_presence = EXPLICIT];
+  // Additional filters to apply to the log query.
+  LogQueryFilters filters = 5;
 }
 
 // QueryLogsInIntervalRequest is the request message for querying logs in a specific time interval.
 // It can be used to query all log messages across multiple jobs, and can be filtered by other parameters in the future.
 message QueryLogsInIntervalRequest {
-  // The start time of the interval to query logs for.
+  // The time interval to query logs for.
   tilebox.v1.TimeInterval time_interval = 1;
   // The pagination parameters for this request.
   tilebox.v1.Pagination page = 2 [features.field_presence = EXPLICIT];
   // The direction in which to sort log entries. If not specified, defaults to descending order (newest entries first).
   tilebox.v1.SortDirection sort_direction = 3;
+  // Additional filters to apply to the log query.
+  LogQueryFilters filters = 4;
 }
 
 // PaginatedLogsData is the response message for paginated log queries. It's a message compatible with LogsData,
@@ -41,6 +64,24 @@ message PaginatedLogsData {
   tilebox.v1.Pagination next_page = 2 [features.field_presence = EXPLICIT];
 }
 
+// GetLogMessageCountsRequest is the request message for retrieving counts of log messages per severity level within a specified time interval.
+message GetLogMessageCountsRequest {
+  // The time interval to fetch log message counts for.
+  tilebox.v1.TimeInterval time_interval = 1;
+}
+
+// GetLogMessageCountsResponse is the response message for retrieving counts of log messages per severity level within a specified time interval.
+message GetLogMessageCountsResponse {
+  repeated LogSeverityCount counts = 1;
+  int64 total_messages = 2;
+}
+
+// LogSeverityCount is a message that represents the count of log messages for a specific severity level.
+message LogSeverityCount {
+  LogSeverityGroup severity = 1;
+  int64 count = 2;
+}
+
 // QueryJobLogsRequest is the request message for querying logs of a specific job, in ascending or descending order,
 // with pagination support.
 message QueryJobSpansRequest {
@@ -68,6 +109,7 @@ message PaginatedSpansData {
 service TelemetryQueryService {
   rpc QueryJobLogs(QueryJobLogsRequest) returns (PaginatedLogsData);
   rpc QueryLogsInInterval(QueryLogsInIntervalRequest) returns (PaginatedLogsData);
+  rpc GetLogMessageCounts(GetLogMessageCountsRequest) returns (GetLogMessageCountsResponse);
 
   rpc QueryJobSpans(QueryJobSpansRequest) returns (PaginatedSpansData);
 }