pigui/task-stream
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(api): a read-side failure waits for its event instead of losing it

Browse Source 
The Redis consumer used to leave a failed event pending and go on to the
next ones, retrying it only at the next start. With Mongo down for a
while, a later change of the same ticket was then projected first, and
the version check dropped the earlier one for good (a status change
could vanish from the read model), besides the read side lagging until
a restart.

Now a failed event is retried in place, waiting longer each time (up to
30 s), and the next one waits for it. Failures of the infrastructure
(Mongo, Redis, the network: isTransientError) wait as long as it takes,
so the read side catches up by itself; an event that keeps failing on
its own goes, after 5 attempts, to <stream>:dead with its error, so it
does not hold up the rest.

GET /api/health reports the way from write to read (events waiting in
the outbox, pending to be projected, set aside) and says `degraded`,
still with 200, when something is stuck.

Tried for real: with Mongo stopped, two status changes went in; the
first failed to project with ECONNREFUSED, and once Mongo was back the
ticket's history had both steps, in order, without restarting the API.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Josep Piguillem
2026-10-16 22:24:19 +02:00
parent d66b1f3ece
commit a32c937765
16 changed files with 492 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
apps/api/openapi.json
View File

@@ -3,6 +3,7 @@
"paths": {
"/api/health": {
"get": {
"description": "503 when a database is down. With the databases up, `status` is `degraded` (still 200, so a lagging read side does not get the container restarted) when events wait in the outbox over a minute, 500 or more wait to be projected, or some were set aside after failing again and again (`task-stream:events:dead`); `events.problems` says which.",
"operationId": "HealthController_check",
"parameters": [],
"responses": {
@@ -15,7 +16,14 @@
"status": "ok",
"postgres": "up",
"mongo": "up",
"redis": "up"
"redis": "up",
"events": {
"unpublished": 0,
"oldestUnpublishedSeconds": null,
"pending": 0,
"dead": 0,
"problems": []
}
}
}
}
@@ -25,7 +33,7 @@
"description": "A database is down"
}
},
"summary": "API and databases status",
"summary": "API, databases and the way from write to read",
"tags": [
"Health"
],

57
apps/api/src/app/health.controller.ts
View File

@@ -12,7 +12,11 @@ import {
ApiTags,
} from '@nestjs/swagger';
import { Public } from '@task-stream/api/identity/presenters';
import { REDIS_CLIENT } from '@task-stream/api/shared/infrastructure';
import {
EventPipelineReading,
EventPipelineStatus,
REDIS_CLIENT,
} from '@task-stream/api/shared/infrastructure';
import Redis from 'ioredis';
import { Connection } from 'mongoose';
import { DataSource } from 'typeorm';
@@ -24,14 +28,31 @@ export class HealthController {
private readonly postgres: DataSource,
@InjectConnection() private readonly mongo: Connection,
@Inject(REDIS_CLIENT) private readonly redis: Redis,
private readonly events: EventPipelineStatus,
) {}
@Public()
@Get()
@ApiOperation({ summary: 'API and databases status' })
@ApiOperation({
summary: 'API, databases and the way from write to read',
description:
'503 when a database is down. With the databases up, `status` is `degraded` (still 200, so a lagging read side does not get the container restarted) when events wait in the outbox over a minute, 500 or more wait to be projected, or some were set aside after failing again and again (`task-stream:events:dead`); `events.problems` says which.',
})
@ApiOkResponse({
schema: {
example: { status: 'ok', postgres: 'up', mongo: 'up', redis: 'up' },
example: {
status: 'ok',
postgres: 'up',
mongo: 'up',
redis: 'up',
events: {
unpublished: 0,
oldestUnpublishedSeconds: null,
pending: 0,
dead: 0,
problems: [],
},
},
},
})
@ApiServiceUnavailableResponse({ description: 'A database is down' })
@@ -44,10 +65,38 @@ export class HealthController {
const [postgres, mongo, redis] = checks.map((c) =>
c.status === 'fulfilled' ? 'up' : 'down',
);
const result = { status: 'ok', postgres, mongo, redis };
const events = await this.events.read().then(
(reading) => ({ ...reading, problems: problemsOf(reading) }),
() => null,
);
const result = {
status: events?.problems.length ? 'degraded' : 'ok',
postgres,
mongo,
redis,
events,
};
if (checks.some((c) => c.status === 'rejected')) {
throw new ServiceUnavailableException({ ...result, status: 'degraded' });
}
return result;
}
}
/** What is wrong on the way from write to read, if anything. */
function problemsOf(reading: EventPipelineReading): string[] {
const problems: string[] = [];
if ((reading.oldestUnpublishedSeconds ?? 0) > 60)
problems.push(
`${reading.unpublished} events wait in the outbox (the oldest, ${reading.oldestUnpublishedSeconds} s): is Redis reachable?`,
);
if (reading.pending >= 500)
problems.push(
`${reading.pending} events wait to be projected: is Mongo reachable?`,
);
if (reading.dead > 0)
problems.push(
`${reading.dead} events were set aside after failing again and again (task-stream:events:dead)`,
);
return problems;
}

5
apps/mcp/src/app/api/taskstream.schema.ts
View File

@@ -11,7 +11,10 @@ export interface paths {
path?: never;
cookie?: never;
};
/** API and databases status */
/**
* API, databases and the way from write to read
* @description 503 when a database is down. With the databases up, `status` is `degraded` (still 200, so a lagging read side does not get the container restarted) when events wait in the outbox over a minute, 500 or more wait to be projected, or some were set aside after failing again and again (`task-stream:events:dead`); `events.problems` says which.
*/
get: operations["HealthController_check"];
put?: never;
post?: never;