Guide

Go integration

Wire Drumbeats into a Go service using only net/http, with panics and goroutines handled properly.

package main

import (
    "net/http"
    "time"
)

const monitor = "https://api.drumbeats.io/v1/ping/<monitor-id>"

var client = &http.Client{Timeout: 3 * time.Second}

func main() {
    ping("/start")
    if err := runJob(); err != nil {
        ping("/failure")
        return
    }
    ping("/success")
}

func ping(event string) {
    resp, err := client.Get(monitor + event)
    if err != nil {
        return // side channel, never fail the job
    }
    resp.Body.Close()
}

package main

import (
    "net/http"
    "time"
)

const monitor = "https://api.drumbeats.io/v1/ping/<monitor-id>"

var client = &http.Client{Timeout: 3 * time.Second}

func main() {
    ping("/start")
    if err := runJob(); err != nil {
        ping("/failure")
        return
    }
    ping("/success")
}

func ping(event string) {
    resp, err := client.Get(monitor + event)
    if err != nil {
        return // side channel, never fail the job
    }
    resp.Body.Close()
}

Always close the response body, even on a ping you do not read. Leaking bodies in a long-lived worker exhausts the connection pool, and the symptom shows up somewhere unrelated hours later.

No third-party dependencies. net/http covers everything here.

Wrap it once#

package monitoring

import (
    "bytes"
    "encoding/json"
    "fmt"
    "net/http"
    "net/url"
    "os"
    "time"
)

const (
    pingTimeout = 3 * time.Second
    maxPayload  = 20_000
)

var baseURL = func() string {
    if v := os.Getenv("DRUMBEATS_BASE_URL"); v != "" {
        return v
    }
    return "https://api.drumbeats.io/v1"
}()

var Monitors = map[string]string{
    "daily_backup":    "11111111-2222-3333-4444-555555555555",
    "hourly_sync":     "66666666-7777-8888-9999-aaaaaaaaaaaa",
    "newsletter_send": "bbbbbbbb-cccc-dddd-eeee-ffffffffffff",
}

var client = &http.Client{Timeout: pingTimeout}

// ping is best effort. It never returns an error and never blocks
// longer than pingTimeout.
func ping(monitorID, event, runID, payload string) {
    u, err := url.Parse(fmt.Sprintf("%s/ping/%s/%s", baseURL, monitorID, event))
    if err != nil {
        return
    }
    if runID != "" {
        q := u.Query()
        q.Set("run_id", runID)
        u.RawQuery = q.Encode()
    }

    var resp *http.Response
    if payload == "" {
        resp, err = client.Get(u.String())
    } else {
        if len(payload) > maxPayload {
            payload = payload[len(payload)-maxPayload:]
        }
        body, _ := json.Marshal(map[string]string{"payload": payload})
        resp, err = client.Post(u.String(), "application/json", bytes.NewReader(body))
    }
    if err != nil {
        return
    }
    resp.Body.Close()
}

// WithMonitor runs fn under the given monitor key. The job's error is
// propagated; ping failures are swallowed. A panic in fn is reported as a
// failure and then re-panics.
func WithMonitor(key, runID string, fn func() error) (err error) {
    id, ok := Monitors[key]
    if !ok {
        return fmt.Errorf("unknown monitor key %q", key)
    }
    if runID == "" {
        runID = fmt.Sprintf("%s-%d", key, time.Now().UnixNano())
    }

    ping(id, "start", runID, "")

    defer func() {
        if r := recover(); r != nil {
            ping(id, "failure", runID, fmt.Sprintf("panic: %v", r))
            panic(r)
        }
    }()

    if err = fn(); err != nil {
        ping(id, "failure", runID, err.Error())
        return err
    }
    ping(id, "success", runID, "")
    return nil
}

package monitoring

import (
    "bytes"
    "encoding/json"
    "fmt"
    "net/http"
    "net/url"
    "os"
    "time"
)

const (
    pingTimeout = 3 * time.Second
    maxPayload  = 20_000
)

var baseURL = func() string {
    if v := os.Getenv("DRUMBEATS_BASE_URL"); v != "" {
        return v
    }
    return "https://api.drumbeats.io/v1"
}()

var Monitors = map[string]string{
    "daily_backup":    "11111111-2222-3333-4444-555555555555",
    "hourly_sync":     "66666666-7777-8888-9999-aaaaaaaaaaaa",
    "newsletter_send": "bbbbbbbb-cccc-dddd-eeee-ffffffffffff",
}

var client = &http.Client{Timeout: pingTimeout}

// ping is best effort. It never returns an error and never blocks
// longer than pingTimeout.
func ping(monitorID, event, runID, payload string) {
    u, err := url.Parse(fmt.Sprintf("%s/ping/%s/%s", baseURL, monitorID, event))
    if err != nil {
        return
    }
    if runID != "" {
        q := u.Query()
        q.Set("run_id", runID)
        u.RawQuery = q.Encode()
    }

    var resp *http.Response
    if payload == "" {
        resp, err = client.Get(u.String())
    } else {
        if len(payload) > maxPayload {
            payload = payload[len(payload)-maxPayload:]
        }
        body, _ := json.Marshal(map[string]string{"payload": payload})
        resp, err = client.Post(u.String(), "application/json", bytes.NewReader(body))
    }
    if err != nil {
        return
    }
    resp.Body.Close()
}

// WithMonitor runs fn under the given monitor key. The job's error is
// propagated; ping failures are swallowed. A panic in fn is reported as a
// failure and then re-panics.
func WithMonitor(key, runID string, fn func() error) (err error) {
    id, ok := Monitors[key]
    if !ok {
        return fmt.Errorf("unknown monitor key %q", key)
    }
    if runID == "" {
        runID = fmt.Sprintf("%s-%d", key, time.Now().UnixNano())
    }

    ping(id, "start", runID, "")

    defer func() {
        if r := recover(); r != nil {
            ping(id, "failure", runID, fmt.Sprintf("panic: %v", r))
            panic(r)
        }
    }()

    if err = fn(); err != nil {
        ping(id, "failure", runID, err.Error())
        return err
    }
    ping(id, "success", runID, "")
    return nil
}

err := monitoring.WithMonitor("daily_backup", "", func() error {
    if err := dumpDatabase(); err != nil {
        return fmt.Errorf("dump: %w", err)
    }
    return uploadToS3()
})

err := monitoring.WithMonitor("daily_backup", "", func() error {
    if err := dumpDatabase(); err != nil {
        return fmt.Errorf("dump: %w", err)
    }
    return uploadToS3()
})

The defer with recover reports a panic as a failure and then re-panics, so Drumbeats hears about it without changing your program's crash behaviour.

What happens when it breaks#

Situation	What Drumbeats sees	What to do
`fn` returns an error	`failure` with the error text	Nothing. The wrapper handles it
`fn` panics	`failure` with the panic value, then the panic continues	Nothing. The `recover` block handles it
A goroutine started inside `fn` panics	Nothing. A panic in another goroutine kills the process outright	Recover inside each goroutine, or do not spawn detached work inside a wrapped job
`fn` returns before its goroutines finish	`success` fires while work is still running	Use `errgroup.Wait()` or a `sync.WaitGroup` before returning
Context cancelled and the error swallowed	`success` on a run that did not complete	Return `ctx.Err()` rather than treating cancellation as normal
The process is `SIGKILL`ed	A `start` with no finish	Set `max_duration_seconds` so the hang is caught server-side

The goroutine rows matter most in Go. A WithMonitor that returns while its workers are still going will report success for work that has not happened yet.

How you get alerted#

A failure opens a FAILED incident once failure_tolerance is reached and pages every notification group on the monitor. The error string or panic value rides along as the payload.

Worker pools#

Call WithMonitor per message, using the message ID as the run ID:

for msg := range messages {
    go func(m Message) {
        _ = monitoring.WithMonitor("order_processor", m.ID, func() error {
            return process(m)
        })
    }(msg)
}

for msg := range messages {
    go func(m Message) {
        _ = monitoring.WithMonitor("order_processor", m.ID, func() error {
            return process(m)
        })
    }(msg)
}

A hundred concurrent messages produce a hundred distinct runs on one monitor, kept apart by run_id. Do not create a monitor per worker.

Next#

Production hardening for the language-agnostic patterns. Event-driven pings for correlation details. Monitor types for picking the right type. Alternatives if you are still choosing a vendor.