@fairmint/canton-node-sdk
Version:
Canton Node SDK
553 lines (552 loc) • 19.9 kB
YAML
apiVersion: 1
groups:
- orgId: 1
name: automation
folder: canton-network
interval: 5m
rules:
- uid: fe73c0e7-dcb3-4975-a7d1-04ed8da087be
title: Automation Failures
condition: threshold
data:
- refId: total
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: sum by(namespace, node_type, trigger_name, migration) (delta(splice_trigger_completed_total{trigger_name=~".+", outcome=~".+"}[10m]))
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: total
- refId: failures
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
datasource:
type: prometheus
uid: prometheus
editorMode: code
expr: sum by(namespace, node_type, trigger_name, migration) (delta(splice_trigger_completed_total{trigger_name=~".+", outcome=~"failure"}[10m])) or on() vector(0)
hide: false
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: failures
- refId: failure_pct
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: ${failures} / ${total} * 100
hide: false
intervalMs: 1000
maxDataPoints: 43200
refId: failure_pct
type: math
- refId: threshold
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: failure_pct
hide: false
intervalMs: 1000
maxDataPoints: 43200
refId: threshold
type: threshold
dashboardUid: a3e1385f-6f03-46d9-908c-34aca0f507a6
panelId: 14
noDataState: $NODATA
execErrState: Alerting
for: 5m
annotations:
__dashboardUid__: a3e1385f-6f03-46d9-908c-34aca0f507a6
__panelId__: "14"
description: The {{ index $labels "trigger_name" }} for the {{ index $labels "node_type" }} app in the {{ index $labels "namespace" }} namespace on migration id {{ index $labels "migration" }} experienced {{ index $values "failure_pct" }}% failures in the last 10 minutes.
severity: |-
{{- if (gt $values.failure_pct.Value 50.0) -}}
critical
{{- else -}}
warning
{{- end -}}
summary: '{{ index $values "failure_pct" }}% fatal errors occurred in {{ index $labels "namespace" }} - {{ index $labels "trigger_name" }} automation trigger'
labels:
gcloud_filter: 'resource.labels.namespace_name=%22{{ index $labels "namespace" }}%22%0A%22{{ index $labels "trigger_name" }}%22'
isPaused: false
- uid: adwt1yr5xuscge
title: ACS snapshot taking too long
condition: too_long
data:
- refId: latency
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: histogram_quantile(0.99, rate(splice_trigger_latency_duration_seconds{trigger_name="AcsSnapshotTrigger", namespace="sv-1"}[10m]))
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: latency
- refId: too_long
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 600
- 0
type: gt
operator:
type: and
query:
params: [ ]
reducer:
params: [ ]
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: latency
intervalMs: 1000
maxDataPoints: 43200
refId: too_long
type: threshold
noDataState: OK
execErrState: Alerting
for: 5m
annotations:
description: ""
runbook_url: ""
severity: warning
summary: ACS snapshot took longer than 10m in {{ index $labels "namespace" }}'s Scan
labels:
"": ""
gcloud_filter: resource.labels.namespace_name=%22{{ index "namespace" }}%22%0A%22{{ index "trigger_name" }}%22
isPaused: false
- uid: ady2ks9ehbw1sb
title: Busy task-based automation
condition: threshold
data:
- refId: runs
relativeTimeRange:
from: 300
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: sum by(namespace, node_type, node_name, job, trigger_name, migration, party) (rate(splice_trigger_completed_total{trigger_name!~"ScanVerdictStoreIngestion|ScanHistoryBackfillingTrigger|AcsSnapshotTrigger|ScanBackfillAggregatesTrigger|TxLogBackfillingTrigger|ReportValidatorLicenseMetricsExportTrigger"}[5m]))
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: runs
- refId: threshold
relativeTimeRange:
from: 300
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 1
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: runs
intervalMs: 1000
maxDataPoints: 43200
refId: threshold
type: threshold
dashboardUid: a3e1385f-6f03-46d9-908c-34aca0f507a6
panelId: 14
noDataState: OK
execErrState: OK
for: 5m
annotations:
__dashboardUid__: a3e1385f-6f03-46d9-908c-34aca0f507a6
__panelId__: "14"
description: The {{ index $labels "trigger_name" }} for the {{ index $labels "node_type" }} app in the {{ index $labels "namespace" }} namespace on migration id {{ index $labels "migration" }} experienced {{ index $values "runs" }} runs per second in the last 5 minutes.
runbook_url: ""
severity: |-
{{- if and $values.runs (gt $values.runs.Value 2.0) -}}
critical
{{- else -}}
warning
{{- end -}}
summary: '{{ index $values "runs" }} trigger runs per second occurred in {{ index $labels "namespace" }} - {{ index $labels "trigger_name" }} automation trigger'
labels:
"": ""
gcloud_filter: resource.labels.namespace_name=%22{{ index "namespace" }}%22%0A%22{{ index "trigger_name" }}%22
isPaused: false
- uid: edz6eq1kc543ke
title: Busy polling-based automation
condition: threshold
data:
- refId: runs
relativeTimeRange:
from: 300
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: sum by(namespace, node_type, node_name, job, trigger_name, migration, party) (rate(splice_trigger_iterations_total{trigger_name!~"ScanVerdictStoreIngestion|ScanHistoryBackfillingTrigger|AcsSnapshotTrigger|ScanBackfillAggregatesTrigger|TxLogBackfillingTrigger"}[5m]))
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: runs
- refId: threshold
relativeTimeRange:
from: 300
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 1
- 0
type: gt
operator:
type: and
query:
params: []
reducer:
params: []
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: runs
intervalMs: 1000
maxDataPoints: 43200
refId: threshold
type: threshold
dashboardUid: a3e1385f-6f03-46d9-908c-34aca0f507a6
panelId: 14
noDataState: OK
execErrState: OK
for: 5m
annotations:
__dashboardUid__: a3e1385f-6f03-46d9-908c-34aca0f507a6
__panelId__: "14"
description: The {{ index $labels "trigger_name" }} for the {{ index $labels "node_type" }} app in the {{ index $labels "namespace" }} namespace on migration id {{ index $labels "migration" }} experienced {{ index $values "runs" }} runs per second in the last 5 minutes.
runbook_url: ""
severity: |-
{{- if and $values.runs (gt $values.runs.Value 2.0) -}}
critical
{{- else -}}
warning
{{- end -}}
summary: '{{ index $values "runs" }} trigger runs per second occurred in {{ index $labels "namespace" }} - {{ index $labels "trigger_name" }} automation trigger'
labels:
"": ""
gcloud_filter: resource.labels.namespace_name=%22{{ index "namespace" }}%22%0A%22{{ index "trigger_name" }}%22
isPaused: false
- uid: fe12i7xur3eo0d
title: Backfilling not progressing
condition: C
data:
- refId: Max (rate vs completed)
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: max by(namespace)(rate(splice_history_backfilling_transaction_count[5m]) > 0 or splice_history_backfilling_completed)
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: Max (rate vs completed)
- refId: C
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 1e-11
type: lt
operator:
type: and
query:
params:
- C
reducer:
params: [ ]
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: Max (rate vs completed)
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
noDataState: $NODATA
execErrState: Alerting
for: 5m
annotations:
description: ""
runbook_url: ""
summary: History backfilling is not making any progress in {{ index $labels "namespace" }}
labels:
"": ""
isPaused: false
- uid: bel66uf182ha8e
title: TxLog backfilling not progressing
condition: C
data:
- refId: Max (rate vs completed)
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: |-
max by(namespace)(
rate(splice_history_txlog_backfilling_transaction_count[5m]) > 0 or
splice_history_txlog_backfilling_completed or
# TxLog backfilling can't start until history backfilling is completed
absent(splice_history_backfilling_completed) or
# Note: "== bool" is a boolean operator, "==" is a filter
(splice_history_backfilling_completed == bool 0)
)
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: Max (rate vs completed)
- refId: C
relativeTimeRange:
from: 600
to: 0
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 1e-11
type: lt
operator:
type: and
query:
params:
- C
reducer:
params: [ ]
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: Max (rate vs completed)
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
noDataState: $NODATA
execErrState: Alerting
for: 5m
annotations:
description: ""
runbook_url: ""
summary: TxLog backfilling is not making any progress in {{ index $labels "namespace" }}
labels:
"": ""
isPaused: false
- uid: fentlrcbcrsaoa
title: Delegateless trigger contention
condition: threshold
data:
- refId: total_attempts
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: sum(rate(splice_trigger_attempted_total{isDsoDelegateTrigger="true", node_type="sv"}[30m])) by (namespace)
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: total_attempts
- refId: total_contention
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
datasource:
type: prometheus
uid: prometheus
editorMode: code
expr: sum(rate(splice_trigger_attempted_total{statusCode!~"OK", isDsoDelegateTrigger="true", node_type="sv", contentionFailure="true"}[30m])) by (namespace)
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: total_contention
- refId: threshold
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- 0
- 0
type: gt
operator:
type: and
query:
params: [ ]
reducer:
params: [ ]
type: avg
type: query
datasource:
name: Expression
type: __expr__
uid: __expr__
expression: (${total_contention} > $CONTENTION_THRESHOLD_PERCENTAGE_PER_NAMESPACE * ${total_attempts}) && (${total_contention} > 0)
intervalMs: 1000
maxDataPoints: 43200
refId: threshold
type: math
# Alert also reports noData when there is no contention.
noDataState: OK
execErrState: Alerting
for: 5m
annotations:
description: '{{ index $labels "namespace" }} had more than 5% contention in the last 30 minutes.'
isPaused: false
- uid: df3t10f23cm68c
title: 'Ingestion saturation'
condition: C
data:
- refId: A
relativeTimeRange:
from: 600
to: 0
datasourceUid: prometheus
model:
editorMode: code
expr: histogram_avg(sum by (le, namespace,job,store_name,store_party,synchronizer_id) (rate(splice_store_ingestion_batch_size{namespace=~"sv"}[20m])))
instant: true
intervalMs: 1000
legendFormat: __auto
maxDataPoints: 43200
range: false
refId: A
- refId: C
datasourceUid: __expr__
model:
conditions:
- evaluator:
params:
- $INGESTION_ENTRIES_PER_BATCH_THRESHOLD
type: gt
operator:
type: and
query:
params:
- C
reducer:
params: [ ]
type: last
type: query
datasource:
type: __expr__
uid: __expr__
expression: A
intervalMs: 1000
maxDataPoints: 43200
refId: C
type: threshold
noDataState: OK
execErrState: OK
for: 30m
annotations:
description: |
The ingestion batch size has exceeded $INGESTION_ENTRIES_PER_BATCH_THRESHOLD entries/batch for a prolonged time in {{ index $labels "namespace" }}'s {{ index $labels "node_type" }}.
This might mean that ingestion is close to not being able to keep up with ledger activity.
This can also happen when an SV was just onboarded and is catching up with the ACS.
summary: The ingestion batch size has exceeded the expected size for a prolonged time.
isPaused: false