Node.js Graceful Shutdown: Handle SIGTERM in Express and Fastify

1
type Cleanup = {name: string; close: () => Promise<void>};
2

3
const cleanups: Cleanup[] = [];
4

5
/** Register anything holding a socket or a file handle. Order matters:
6
 *  cleanups run in reverse registration order, so register the database
7
 *  before the things that depend on it. */
8
export function onShutdown(name: string, close: () => Promise<void>): void {
9
  cleanups.push({name, close});
10
}
11

12
async function runCleanups(): Promise<void> {
13
  // Reverse order, and each one isolated: one failing pool should not
14
  // prevent the others from closing.
15
  for (const {name, close} of [...cleanups].reverse()) {
16
    try {
17
      await close();
18
      console.log(`[shutdown] closed ${name}`);
19
    } catch (err) {
20
      console.error(`[shutdown] failed to close ${name}`, err);
21
    }
22
  }
23
}

1
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
2

3
function closeServer(server: Server, timeoutMs: number): Promise<void> {
4
  return new Promise((resolve) => {
5
    let settled = false;
6
    const done = () => {
7
      if (!settled) {
8
        settled = true;
9
        resolve();
10
      }
11
    };
12

13
    // Stops accepting new connections. Existing ones keep being served,
14
    // and the callback fires once they are all done.
15
    server.close(done);
16

17
    // The part people miss. An idle keep-alive socket is an open
18
    // connection, so server.close() waits for it even though no request
19
    // is in flight. Node 18.2+ lets us drop those immediately.
20
    server.closeIdleConnections?.();
21

22
    // Last resort. If a request is genuinely stuck, we would rather
23
    // cut it than get SIGKILLed and cut everything.
24
    setTimeout(() => {
25
      console.warn('[shutdown] drain timed out, forcing connections closed');
26
      server.closeAllConnections?.();
27
      done();
28
    }, timeoutMs).unref();
29
  });
30
}

1
export function installShutdownHandlers(server: Server): void {
2
  const shutdown = async (signal: string) => {
3
    if (isShuttingDown) {
4
      console.log(`[shutdown] already shutting down, ignoring ${signal}`);
5
      return;
6
    }
7
    isShuttingDown = true;
8
    console.log(`[shutdown] ${signal} received`);
9

10
    // 1. Fail readiness immediately. This starts the clock on
11
    //    Kubernetes taking us out of the endpoint list.
12
    // 2. Keep serving while that propagates.
13
    await sleep(options.drainDelayMs);
14

15
    // 3. Stop accepting, finish what is open.
16
    await closeServer(server, options.shutdownTimeoutMs);
17

18
    // 4. Now nothing is using the database, so close it.
19
    await runCleanups();
20

21
    console.log('[shutdown] done');
22
    process.exit(0);
23
  };
24

25
  process.on('SIGTERM', () => void shutdown('SIGTERM'));
26
  process.on('SIGINT', () => void shutdown('SIGINT'));
27

28
  // An unhandled rejection leaves the process in an unknown state.
29
  // Shut down rather than continuing to serve from it.
30
  process.on('unhandledRejection', (reason) => {
31
    console.error('[shutdown] unhandled rejection', reason);
32
    void shutdown('unhandledRejection');
33
  });
34
}

1
import {
2
  installShutdownHandlers,
3
  isReady,
4
  onShutdown,
5
} from './graceful_shutdown';
6
import express from 'express';
7
import {Pool} from 'pg';
8

9
const app = express();
10
const pool = new Pool({connectionString: process.env.DATABASE_URL});
11

12
// Readiness reflects shutdown state. Liveness must NOT: a shutting-down
13
// process is unhealthy for traffic but perfectly alive, and failing
14
// liveness here would make the kubelet kill it mid-drain.
15
app.get('/readyz', (_req, res) =>
16
  isReady() ? res.sendStatus(200) : res.sendStatus(503),
17
);
18
app.get('/livez', (_req, res) => res.sendStatus(200));
19

20
app.get('/', async (_req, res) => {
21
  const {rows} = await pool.query('select now()');
22
  res.json(rows[0]);
23
});
24

25
const server = app.listen(3000, () => console.log('listening on 3000'));
26

27
// Registered first, so it closes last, after the server has drained.
28
onShutdown('postgres pool', () => pool.end());
29

30
installShutdownHandlers(server);

1
import {isReady, onShutdown} from './graceful_shutdown';
2
import Fastify from 'fastify';
3
import {Pool} from 'pg';
4

5
const app = Fastify({logger: true});
6
const pool = new Pool({connectionString: process.env.DATABASE_URL});
7

8
app.get('/readyz', async (_req, reply) =>
9
  isReady() ? reply.code(200).send() : reply.code(503).send(),
10
);
11

12
app.addHook('onClose', async () => {
13
  await pool.end();
14
});
15

16
await app.listen({port: 3000, host: '0.0.0.0'});
17

18
// Fastify's own close() already drains and runs onClose hooks, so the
19
// only thing missing is the delay before it starts.
20
const shutdown = async (signal: string) => {
21
  app.log.info(`${signal} received`);
22
  await new Promise((r) => setTimeout(r, 5_000)); // endpoint propagation
23
  await app.close();
24
  process.exit(0);
25
};
26

27
process.on('SIGTERM', () => void shutdown('SIGTERM'));
28
process.on('SIGINT', () => void shutdown('SIGINT'));

1
spec:
2
  # 5s drain delay + 15s shutdown timeout + pool close, plus headroom.
3
  # If this is smaller than the sum, you get SIGKILLed mid-drain and
4
  # lose the requests you were protecting.
5
  terminationGracePeriodSeconds: 45
6
  containers:
7
    - name: api
8
      readinessProbe:
9
        httpGet: {path: /readyz, port: 3000}
10
        periodSeconds: 5
11
      livenessProbe:
12
        httpGet: {path: /livez, port: 3000}
13
        periodSeconds: 10