Add subsystem metrics

- Adds a Metrics() class that can track data such as number of
events the callback receiver inserted into database
- Exposes this metric data at the api/v2/metrics/ endpoint.
This data is prometheus-friendly
- Metric data is stored in memory, then periodically saved to Redis.
- Metric data is periodically broadcast to other nodes in the cluster,
so that each node has a copy of the most recent metric data collected.
This commit is contained in:
Seth Foster
2021-03-23 16:05:10 -04:00
parent f8a698d127
commit 0c569c67fd
12 changed files with 410 additions and 20 deletions

View File

@@ -56,24 +56,28 @@ def test_metrics_counts(organization_factory, job_template_factory, workflow_job
assert EXPECTED_VALUES[name] == value
def get_metrics_view_db_only():
return reverse('api:metrics_view') + '?dbonly=1'
@pytest.mark.django_db
def test_metrics_permissions(get, admin, org_admin, alice, bob, organization):
assert get(reverse('api:metrics_view'), user=admin).status_code == 200
assert get(reverse('api:metrics_view'), user=org_admin).status_code == 403
assert get(reverse('api:metrics_view'), user=alice).status_code == 403
assert get(reverse('api:metrics_view'), user=bob).status_code == 403
assert get(get_metrics_view_db_only(), user=admin).status_code == 200
assert get(get_metrics_view_db_only(), user=org_admin).status_code == 403
assert get(get_metrics_view_db_only(), user=alice).status_code == 403
assert get(get_metrics_view_db_only(), user=bob).status_code == 403
organization.auditor_role.members.add(bob)
assert get(reverse('api:metrics_view'), user=bob).status_code == 403
assert get(get_metrics_view_db_only(), user=bob).status_code == 403
Role.singleton('system_auditor').members.add(bob)
bob.is_system_auditor = True
assert get(reverse('api:metrics_view'), user=bob).status_code == 200
assert get(get_metrics_view_db_only(), user=bob).status_code == 200
@pytest.mark.django_db
def test_metrics_http_methods(get, post, patch, put, options, admin):
assert get(reverse('api:metrics_view'), user=admin).status_code == 200
assert put(reverse('api:metrics_view'), user=admin).status_code == 405
assert patch(reverse('api:metrics_view'), user=admin).status_code == 405
assert post(reverse('api:metrics_view'), user=admin).status_code == 405
assert options(reverse('api:metrics_view'), user=admin).status_code == 200
assert get(get_metrics_view_db_only(), user=admin).status_code == 200
assert put(get_metrics_view_db_only(), user=admin).status_code == 405
assert patch(get_metrics_view_db_only(), user=admin).status_code == 405
assert post(get_metrics_view_db_only(), user=admin).status_code == 405
assert options(get_metrics_view_db_only(), user=admin).status_code == 200