diff --git a/heicode/controller/heicode_config.go b/heicode/controller/heicode_config.go index 30329bcc..367851f6 100644 --- a/heicode/controller/heicode_config.go +++ b/heicode/controller/heicode_config.go @@ -24,6 +24,8 @@ func HeicodeConfig(c *gin.Context) { "endpoint": "/api/heicode/telemetry/events", "max_batch": common.GetEnvOrDefault("HEICODE_TELEMETRY_MAX_BATCH", telemetryMaxBatch), "flush_interval_sec": common.GetEnvOrDefault("HEICODE_TELEMETRY_FLUSH_INTERVAL_SEC", 30), + // Server retention window (#32): events older than this are purged. + "retention_days": telemetryRetentionDays(), }, }) } diff --git a/heicode/controller/heicode_telemetry.go b/heicode/controller/heicode_telemetry.go index 00956665..a64b0736 100644 --- a/heicode/controller/heicode_telemetry.go +++ b/heicode/controller/heicode_telemetry.go @@ -25,8 +25,65 @@ const ( telemetryMaxBatch = 20 telemetryMaxBodySize = 256 * 1024 headerDeviceID = "X-Heicode-Device-Id" + // Per-field hard cap after redaction (#32): a backstop so a single event + // can't park an unbounded blob in the telemetry store even within batch + // limits. stack_top / context are truncated past this many bytes. + telemetryMaxFieldBytes = 8 * 1024 ) +// telemetryContextAllowedKeys whitelists the non-content diagnostic keys the +// client may attach to an event's `context` (#32). Anything else is dropped +// before persistence, so a client regression can't land arbitrary — possibly +// identifying — JSON (prompts, code, tokens, emails, full file paths, raw IPs) +// in the telemetry store. Keep in sync with the client telemetry contract +// (winos#23) and docs/integration/heicode-desktop-client-api.md. Additions must +// be reviewed against the "no identifying content" rule in issue #32. +var telemetryContextAllowedKeys = map[string]bool{ + "route": true, // logical UI route, e.g. "chat" (no params) + "retryable": true, // bool + "phase": true, // lifecycle phase enum + "exit_code": true, // process exit code (int) + "duration_ms": true, // numeric timing + "attempt": true, // retry attempt count +} + +// filterTelemetryContext keeps only whitelisted keys from the client-supplied +// context object, then redacts and size-caps the result (#32). Returns "" when +// the context is empty, unparseable, or has no allowed keys — telemetry is +// best-effort diagnostics, so dropping an unrecognized payload is preferable to +// storing arbitrary JSON. +func filterTelemetryContext(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var obj map[string]json.RawMessage + if err := common.Unmarshal(raw, &obj); err != nil { + return "" // not an object (or malformed) -> drop + } + filtered := make(map[string]json.RawMessage, len(obj)) + for k, v := range obj { + if telemetryContextAllowedKeys[k] { + filtered[k] = v + } + } + if len(filtered) == 0 { + return "" + } + b, err := common.Marshal(filtered) + if err != nil { + return "" + } + return capTelemetryField(model.RedactText(string(b))) +} + +// capTelemetryField truncates an already-redacted field to telemetryMaxFieldBytes. +func capTelemetryField(s string) string { + if len(s) <= telemetryMaxFieldBytes { + return s + } + return s[:telemetryMaxFieldBytes] +} + type telemetryEventIn struct { ClientId string `json:"client_id"` SchemaVersion int `json:"schema_version"` @@ -73,13 +130,12 @@ func (e telemetryEventIn) toModel(userID int, deviceID string, now int64) model. stackTopJSON := "" if len(e.StackTop) > 0 { if b, err := common.Marshal(e.StackTop); err == nil { - stackTopJSON = model.RedactText(string(b)) + stackTopJSON = capTelemetryField(model.RedactText(string(b))) } } - ctxJSON := "" - if len(e.Context) > 0 { - ctxJSON = model.RedactText(string(e.Context)) - } + // #32: context is restricted to a key whitelist (drop arbitrary/identifying + // JSON), then redacted and size-capped. + ctxJSON := filterTelemetryContext(e.Context) return model.TelemetryEvent{ ReceivedAt: now, UserId: userID, diff --git a/heicode/controller/heicode_telemetry_test.go b/heicode/controller/heicode_telemetry_test.go index b9dec944..aa9f0535 100644 --- a/heicode/controller/heicode_telemetry_test.go +++ b/heicode/controller/heicode_telemetry_test.go @@ -65,12 +65,45 @@ func TestTelemetryToModel_RedactsSecrets(t *testing.T) { ev := telemetryEventIn{ ClientId: "dev-1", StackTop: []string{"at boom (auth.ts) key=sk-abcDEF1234567890"}, - Context: json.RawMessage(`{"hdr":"Authorization: Bearer aZ09tokenVALUE","ok":true}`), + // "route" is whitelisted (#32) so it survives the field filter; redaction + // must still strip the Bearer token carried inside an allowed key. + Context: json.RawMessage(`{"route":"Authorization: Bearer aZ09tokenVALUE","retryable":true}`), } m := ev.toModel(7, "dev-1", 1700000000) require.NotContains(t, m.StackTopJSON, "sk-abcDEF1234567890", "sk- secret must be redacted in stack_top") require.Contains(t, m.StackTopJSON, "REDACTED") require.NotContains(t, m.ContextJSON, "aZ09tokenVALUE", "Bearer token must be redacted in context") - require.Contains(t, m.ContextJSON, "ok") // non-secret content preserved + require.Contains(t, m.ContextJSON, "retryable") // non-secret whitelisted content preserved +} + +// #32: context must be restricted to a key whitelist so a client regression +// cannot land arbitrary/identifying JSON in the telemetry store. +func TestFilterTelemetryContext_Whitelist(t *testing.T) { + // allowed keys kept, unknown keys (incl. potentially identifying) dropped + out := filterTelemetryContext(json.RawMessage( + `{"route":"chat","retryable":true,"email":"a@b.com","file":"C:/Users/x/secret.go","prompt":"hi"}`)) + require.Contains(t, out, "route") + require.Contains(t, out, "retryable") + require.NotContains(t, out, "email") + require.NotContains(t, out, "a@b.com") + require.NotContains(t, out, "secret.go") + require.NotContains(t, out, "prompt") + + // no allowed keys -> dropped entirely + require.Equal(t, "", filterTelemetryContext(json.RawMessage(`{"email":"a@b.com"}`))) + // non-object / malformed -> dropped + require.Equal(t, "", filterTelemetryContext(json.RawMessage(`"a string"`))) + require.Equal(t, "", filterTelemetryContext(json.RawMessage(`not json`))) + require.Equal(t, "", filterTelemetryContext(nil)) +} + +// #32: per-field size cap is a backstop against unbounded blobs. +func TestCapTelemetryField(t *testing.T) { + require.Equal(t, "short", capTelemetryField("short")) + big := make([]byte, telemetryMaxFieldBytes+100) + for i := range big { + big[i] = 'a' + } + require.Len(t, capTelemetryField(string(big)), telemetryMaxFieldBytes) } diff --git a/heicode/controller/telemetry_retention.go b/heicode/controller/telemetry_retention.go new file mode 100644 index 00000000..5601b1ab --- /dev/null +++ b/heicode/controller/telemetry_retention.go @@ -0,0 +1,77 @@ +package controller + +import ( + "fmt" + "sync" + "time" + + "github.com/heicode/manager/common" + "github.com/heicode/manager/model" +) + +// Telemetry retention (#32). Account-linkable client error-telemetry must not be +// kept indefinitely: a daily task deletes rows older than the retention window. +// Disabled when telemetry ingest is off (default) or retention <= 0. +// +// - HEICODE_TELEMETRY_RETENTION_DAYS (default 30): rows received earlier than +// now-RETENTION are deleted. <=0 disables the purge (keep-forever — only for +// explicit operator opt-out; not recommended for production). +// - HEICODE_TELEMETRY_RETENTION_INTERVAL_HOURS (default 24): sweep cadence. +// +// Master-only (wired from main.go under IsMasterNode) so multiple nodes don't +// all sweep the shared LOG_DB. + +var telemetryRetentionTaskOnce sync.Once + +func telemetryRetentionDays() int { + return common.GetEnvOrDefault("HEICODE_TELEMETRY_RETENTION_DAYS", 30) +} + +// StartTelemetryRetentionTask launches the daily telemetry retention sweep. +func StartTelemetryRetentionTask() { + telemetryRetentionTaskOnce.Do(func() { + // Only meaningful once ingest is enabled; if the endpoint is off there is + // nothing being written, but we still allow the sweep to drain any rows + // captured during a prior enabled window. Gate on retention days instead. + days := telemetryRetentionDays() + if days <= 0 { + common.SysLog("telemetry retention task disabled (HEICODE_TELEMETRY_RETENTION_DAYS<=0; rows kept indefinitely)") + return + } + intervalHours := common.GetEnvOrDefault("HEICODE_TELEMETRY_RETENTION_INTERVAL_HOURS", 24) + if intervalHours < 1 { + intervalHours = 24 + } + go func() { + time.Sleep(5 * time.Minute) // avoid startup churn + runTelemetryRetentionOnce() + ticker := time.NewTicker(time.Duration(intervalHours) * time.Hour) + defer ticker.Stop() + for range ticker.C { + runTelemetryRetentionOnce() + } + }() + common.SysLog(fmt.Sprintf("telemetry retention task started: retention=%dd interval=%dh", days, intervalHours)) + }) +} + +func runTelemetryRetentionOnce() { + defer func() { + if r := recover(); r != nil { + common.SysLog(fmt.Sprintf("telemetry retention task panic recovered: %v", r)) + } + }() + days := telemetryRetentionDays() + if days <= 0 { + return + } + cutoff := time.Now().Unix() - int64(days)*86400 + deleted, err := model.DeleteTelemetryEventsBefore(cutoff) + if err != nil { + common.SysLog("telemetry retention task: " + err.Error()) + return + } + if deleted > 0 { + common.SysLog(fmt.Sprintf("telemetry retention task: deleted %d events older than %d days", deleted, days)) + } +} diff --git a/heicode/main.go b/heicode/main.go index 77812f70..3b04573f 100644 --- a/heicode/main.go +++ b/heicode/main.go @@ -134,6 +134,9 @@ func main() { // retention window (issue #4). Master-only so multiple nodes don't all purge. if common.IsMasterNode { controller.StartSecretPurgeTask() + // Telemetry retention: daily purge of client error-telemetry older than + // HEICODE_TELEMETRY_RETENTION_DAYS (#32). Master-only. + controller.StartTelemetryRetentionTask() } if common.IsMasterNode && constant.UpdateTask { diff --git a/heicode/model/telemetry_event.go b/heicode/model/telemetry_event.go index c7e23c13..65fac5ce 100644 --- a/heicode/model/telemetry_event.go +++ b/heicode/model/telemetry_event.go @@ -39,3 +39,15 @@ func InsertTelemetryEvents(events []TelemetryEvent) error { } return LOG_DB.Create(&events).Error } + +// DeleteTelemetryEventsBefore removes telemetry rows received before cutoffUnix +// (server unix seconds), enforcing the retention window (#32). Returns the +// number of rows deleted. Account-linkable device telemetry must not be kept +// indefinitely; callers run this from a periodic retention task. +func DeleteTelemetryEventsBefore(cutoffUnix int64) (int64, error) { + if LOG_DB == nil { + return 0, nil + } + res := LOG_DB.Where("received_at < ?", cutoffUnix).Delete(&TelemetryEvent{}) + return res.RowsAffected, res.Error +}