import { Controller, Get, Inject, ServiceUnavailableException, } from '@nestjs/common'; import { InjectConnection } from '@nestjs/mongoose'; import { ApiOkResponse, ApiOperation, ApiServiceUnavailableResponse, ApiTags, } from '@nestjs/swagger'; import { Public } from '@task-stream/api/identity/presenters'; import { EventPipelineReading, EventPipelineStatus, REDIS_CLIENT, } from '@task-stream/api/shared/infrastructure'; import Redis from 'ioredis'; import { Connection } from 'mongoose'; import { DataSource } from 'typeorm'; @ApiTags('Health') @Controller('health') export class HealthController { constructor( private readonly postgres: DataSource, @InjectConnection() private readonly mongo: Connection, @Inject(REDIS_CLIENT) private readonly redis: Redis, private readonly events: EventPipelineStatus, ) {} @Public() @Get() @ApiOperation({ summary: 'API, databases and the way from write to read', description: '503 when a database is down. With the databases up, `status` is `degraded` (still 200, so a lagging read side does not get the container restarted) when events wait in the outbox over a minute, 500 or more wait to be projected, or some were set aside after failing again and again (`task-stream:events:dead`); `events.problems` says which.', }) @ApiOkResponse({ schema: { example: { status: 'ok', postgres: 'up', mongo: 'up', redis: 'up', events: { unpublished: 0, oldestUnpublishedSeconds: null, pending: 0, dead: 0, problems: [], }, }, }, }) @ApiServiceUnavailableResponse({ description: 'A database is down' }) async check() { const checks = await Promise.allSettled([ this.postgres.query('SELECT 1'), this.mongo.db?.admin().ping(), this.redis.ping(), ]); const [postgres, mongo, redis] = checks.map((c) => c.status === 'fulfilled' ? 'up' : 'down', ); const events = await this.events.read().then( (reading) => ({ ...reading, problems: problemsOf(reading) }), () => null, ); const result = { status: events?.problems.length ? 'degraded' : 'ok', postgres, mongo, redis, events, }; if (checks.some((c) => c.status === 'rejected')) { throw new ServiceUnavailableException({ ...result, status: 'degraded' }); } return result; } } /** What is wrong on the way from write to read, if anything. */ function problemsOf(reading: EventPipelineReading): string[] { const problems: string[] = []; if ((reading.oldestUnpublishedSeconds ?? 0) > 60) problems.push( `${reading.unpublished} events wait in the outbox (the oldest, ${reading.oldestUnpublishedSeconds} s): is Redis reachable?`, ); if (reading.pending >= 500) problems.push( `${reading.pending} events wait to be projected: is Mongo reachable?`, ); if (reading.dead > 0) problems.push( `${reading.dead} events were set aside after failing again and again (task-stream:events:dead)`, ); return problems; }