Fix bug with fetch & add retry functionality

This commit is contained in:
Lumeille 2024-06-02 15:30:43 +10:00
parent 2d4a6073a6
commit d4ade0de35
3 changed files with 15 additions and 5 deletions

View File

@ -28,7 +28,8 @@ const configSchema = z.object({
name: z.string(), name: z.string(),
interval: z.number().default(60), interval: z.number().default(60),
notifyInterval: z.number().default(300), notifyInterval: z.number().default(300),
alertEndpoints: z.array(z.string()) alertEndpoints: z.array(z.string()),
retries: z.number().int().min(0).default(1)
}).and(z.union([ }).and(z.union([
z.object({ z.object({
type: z.literal("fetch"), type: z.literal("fetch"),

View File

@ -8,20 +8,28 @@ type EndpointState = State extends Map<any, infer L> ? L : never
export const handleDown = (endpointState: EndpointState, curTime: number, endpoint: Config["pollEndpoints"][number]) => { export const handleDown = (endpointState: EndpointState, curTime: number, endpoint: Config["pollEndpoints"][number]) => {
const prevDown = endpointState.isDown const prevDown = endpointState.isDown
endpointState.isDown = true endpointState.isDown = true
endpointState.attemptsFailed++
console.log(`Endpoint ${endpoint.name} has failed ${endpointState.attemptsFailed} times`)
if (!prevDown) endpointState.downStart = curTime if (!prevDown) endpointState.downStart = curTime
if (curTime - endpointState.lastDownAlert < (endpoint.notifyInterval) * 1000) return if (curTime - endpointState.lastDownAlert < (endpoint.notifyInterval) * 1000) return
const message = prevDown ?
if (endpointState.attemptsFailed > endpoint.retries) {
const message = prevDown ?
`[${formatTS(curTime)}] ${endpoint.name} is still down and initially went down at ${formatTS(endpointState.downStart)}` : `[${formatTS(curTime)}] ${endpoint.name} is still down and initially went down at ${formatTS(endpointState.downStart)}` :
`[${formatTS(curTime)}] ${endpoint.name} went down` `[${formatTS(curTime)}] ${endpoint.name} went down`
notify(message, endpoint) notify(message, endpoint)
endpointState.lastDownAlert = curTime endpointState.lastDownAlert = curTime
}
} }
export const handleUp = (endpointState: EndpointState, curTime: number, endpoint: Config["pollEndpoints"][number]) => { export const handleUp = (endpointState: EndpointState, curTime: number, endpoint: Config["pollEndpoints"][number]) => {
if (endpointState.isDown) { if (endpointState.isDown) {
endpointState.isDown = false endpointState.isDown = false
endpointState.attemptsFailed = 0
const message = `[${formatTS(curTime)}] ${endpoint.name} is back up` const message = `[${formatTS(curTime)}] ${endpoint.name} is back up`
notify(message, endpoint) notify(message, endpoint)
} }

View File

@ -9,13 +9,14 @@ let state = new Map(config.pollEndpoints.map(x => [x.name, {
lastExec: 0, lastExec: 0,
downStart: 0, downStart: 0,
lastDownAlert: 0, lastDownAlert: 0,
attemptsFailed: 0,
isDown: false isDown: false
}])) }]))
export type State = typeof state export type State = typeof state
const isEndpointUp = async (endpoint: Config["pollEndpoints"][number]) => { const isEndpointUp = async (endpoint: Config["pollEndpoints"][number]) => {
if (endpoint.type === "fetch" ) { if (endpoint.type === "fetch" ) {
const r = await fetch(endpoint.url, {method: "GET"}) const r = await fetch(endpoint.url, {method: "GET"}).catch(e => ({ok: false}))
return r.ok return r.ok
} }
else if (endpoint.type === "tcp-ping" ) { else if (endpoint.type === "tcp-ping" ) {