Add Apprise notification integration (#570)

This commit is contained in:
Pulse Automation Bot
2025-10-18 16:39:39 +00:00
parent 0b4e4f9c59
commit 80b9d0602a
18 changed files with 1284 additions and 106 deletions

View File

@@ -70,13 +70,3 @@ Packages the Helm chart and pushes it to the GitHub Container Registry (OCI) whe
- Chart and app versions mirror the Pulse release tag (e.g., `v4.24.0``4.24.0`)
- Publishes to `oci://ghcr.io/<owner>/pulse-chart`
- Requires no additional secrets—uses the built-in `GITHUB_TOKEN` with `packages: write` permission
## Helm Integration (Kind)
**File**: `helm-integration.yml`
Creates a disposable Kind cluster, installs the chart, waits for the hub deployment to report ready, and performs a `/health` smoke check from inside the cluster.
- Triggered alongside the lint workflow for PRs/pushes touching Helm content
- Disables persistence to keep the Kind cluster lightweight
- Provides early detection of runtime issues (missing secrets, invalid probes, etc.)

View File

@@ -1,58 +0,0 @@
name: Helm Integration
on:
push:
branches: [main]
paths:
- "deploy/helm/**"
- ".github/workflows/helm-*.yml"
- "docs/KUBERNETES.md"
- "README.md"
pull_request:
paths:
- "deploy/helm/**"
- ".github/workflows/helm-*.yml"
- "docs/KUBERNETES.md"
- "README.md"
workflow_dispatch: {}
jobs:
kind-smoke-test:
name: Deploy to Kind and Smoke Test
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Helm
uses: azure/setup-helm@v4
with:
version: v3.15.2
- name: Create Kind cluster
uses: helm/kind-action@v1.8.0
with:
wait: 120s
- name: Install Pulse chart
run: |
helm upgrade --install pulse ./deploy/helm/pulse \
--namespace pulse \
--create-namespace \
--set persistence.enabled=false \
--set server.secretEnv.create=true \
--set server.secretEnv.data.API_TOKENS=dummy-token \
--wait \
--timeout 5m
- name: Verify deployment is available
run: kubectl -n pulse wait --for=condition=available deployment/pulse --timeout=120s
- name: Hit health endpoint from inside the cluster
run: |
kubectl -n pulse run smoke-test \
--rm \
--image=curlimages/curl:8.3.0 \
--restart=Never \
-- curl -fsS http://pulse:7655/health

View File

@@ -320,6 +320,12 @@ Quick start - most settings are in the web UI:
- **Settings → Security**: Authentication and API tokens
- **Alerts**: Thresholds and notifications
### Apprise CLI Notifications
Pulse can broadcast grouped alerts through the [Apprise](https://github.com/caronc/apprise) CLI. Install Apprise on the Pulse host (for example with `pip install apprise`) and configure the targets under **Alerts → Notifications**. Each target URL should be a valid Apprise destination (Discord, Slack, email, SMS, etc.).
You can also override the CLI path and execution timeout if Apprise is installed in a non-standard location. Pulse automatically skips Apprise delivery when no targets are configured.
### Configuration Files
Pulse uses three separate configuration files with clear separation of concerns:

View File

@@ -57,6 +57,13 @@ export interface Webhook {
customFields?: Record<string, string>;
}
export interface AppriseConfig {
enabled: boolean;
targets: string[];
cliPath?: string;
timeoutSeconds?: number;
}
export interface NotificationTestRequest {
type: 'email' | 'webhook';
config?: Record<string, unknown>; // Backend expects different format than frontend types
@@ -66,6 +73,17 @@ export interface NotificationTestRequest {
export class NotificationsAPI {
private static baseUrl = '/api/notifications';
static async getAppriseConfig(): Promise<AppriseConfig> {
return apiFetchJSON(`${this.baseUrl}/apprise`);
}
static async updateAppriseConfig(config: AppriseConfig): Promise<AppriseConfig> {
return apiFetchJSON(`${this.baseUrl}/apprise`, {
method: 'PUT',
body: JSON.stringify(config),
});
}
// Email configuration
static async getEmailConfig(): Promise<EmailConfig> {
const backendConfig = await apiFetchJSON<Record<string, unknown>>(`${this.baseUrl}/email`);

View File

@@ -133,7 +133,15 @@ export function ResourceTable(props: ResourceTableProps) {
return props.resources ?? [];
};
const hasRows = () => flattenResources().length > 0;
const hasRows = () => {
if (flattenResources().length > 0) {
return true;
}
if (props.groupedResources && Object.keys(props.groupedResources).length > 0) {
return true;
}
return Boolean(props.globalDefaults);
};
const [activeMetricInput, setActiveMetricInput] = createSignal<{ resourceId: string; metric: string } | null>(null);
const [showDelayRow, setShowDelayRow] = createSignal(false);

View File

@@ -15,7 +15,7 @@ import type {
DockerHost,
DockerContainer,
} from '@/types/api';
import type { RawOverrideConfig, PMGThresholdDefaults } from '@/types/alerts';
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
type OverrideType =
| 'guest'
@@ -169,6 +169,14 @@ interface ThresholdsTableProps {
| Record<string, Record<string, number>>
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
) => void;
snapshotDefaults: () => SnapshotAlertConfig;
setSnapshotDefaults: (
value:
| SnapshotAlertConfig
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
) => void;
snapshotFactoryDefaults?: SnapshotAlertConfig;
resetSnapshotDefaults?: () => void;
setHasUnsavedChanges: (value: boolean) => void;
activeAlerts?: Record<string, Alert>;
removeAlerts?: (predicate: (alert: Alert) => boolean) => void;
@@ -200,6 +208,9 @@ interface ThresholdsTableProps {
setDisableAllDockerHostsOffline: (value: boolean) => void;
}
const DEFAULT_SNAPSHOT_WARNING = 30;
const DEFAULT_SNAPSHOT_CRITICAL = 45;
export function ThresholdsTable(props: ThresholdsTableProps) {
const navigate = useNavigate();
const location = useLocation();
@@ -744,6 +755,74 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
/* no-op placeholder for future scroll restoration */
};
const snapshotFactoryConfig = () =>
props.snapshotFactoryDefaults ?? {
enabled: false,
warningDays: DEFAULT_SNAPSHOT_WARNING,
criticalDays: DEFAULT_SNAPSHOT_CRITICAL,
};
const sanitizeSnapshotConfig = (config: SnapshotAlertConfig): SnapshotAlertConfig => {
let warning = Math.max(0, Math.round(config.warningDays ?? 0));
let critical = Math.max(0, Math.round(config.criticalDays ?? 0));
if (critical > 0 && warning > critical) {
warning = critical;
}
if (critical === 0 && warning > 0) {
critical = warning;
}
return {
enabled: !!config.enabled,
warningDays: warning,
criticalDays: critical,
};
};
const updateSnapshotDefaults = (
updater:
| SnapshotAlertConfig
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
) => {
props.setSnapshotDefaults((prev) => {
const next =
typeof updater === 'function'
? (updater as (prev: SnapshotAlertConfig) => SnapshotAlertConfig)(prev)
: { ...prev, ...updater };
return sanitizeSnapshotConfig(next);
});
props.setHasUnsavedChanges(true);
};
const snapshotDefaultsRecord = createMemo(() => {
const current = props.snapshotDefaults();
return {
'warning days': current.warningDays ?? 0,
'critical days': current.criticalDays ?? 0,
};
});
const snapshotFactoryDefaultsRecord = createMemo(() => {
const factory = snapshotFactoryConfig();
return {
'warning days': factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING,
'critical days': factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL,
};
});
const snapshotOverridesCount = createMemo(() => {
const current = props.snapshotDefaults();
const factory = snapshotFactoryConfig();
return current.enabled !== factory.enabled ||
(current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !==
(factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) ||
(current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !==
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL)
? 1
: 0;
});
// Process guests with their overrides and group by node
const guestsGroupedByNode = createMemo<Record<string, Resource[]>>((prev = {}) => {
// If we're currently editing, return the previous value to avoid re-renders
@@ -1082,16 +1161,23 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
},
{
key: 'storage' as const,
label: 'Storage',
total: props.storage?.length ?? 0,
overrides: countOverrides(storageWithOverrides()),
tab: 'proxmox' as const,
},
{
key: 'pbs' as const,
label: 'PBS Servers',
total: props.pbsInstances?.length ?? 0,
overrides: countOverrides(pbsServersWithOverrides()),
label: 'Storage',
total: props.storage?.length ?? 0,
overrides: countOverrides(storageWithOverrides()),
tab: 'proxmox' as const,
},
{
key: 'snapshots' as const,
label: 'Snapshots',
total: 1,
overrides: snapshotOverridesCount(),
tab: 'proxmox' as const,
},
{
key: 'pbs' as const,
label: 'PBS Servers',
total: props.pbsInstances?.length ?? 0,
overrides: countOverrides(pbsServersWithOverrides()),
tab: 'proxmox' as const,
},
{
@@ -1883,6 +1969,72 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
</div>
</Show>
<Show when={hasSection('snapshots')}>
<div ref={registerSection('snapshots')} class="scroll-mt-24">
<ResourceTable
title="Snapshot Age"
resources={[]}
columns={['Warning Days', 'Critical Days']}
activeAlerts={props.activeAlerts}
emptyMessage=""
onEdit={startEditing}
onSaveEdit={saveEdit}
onCancelEdit={cancelEdit}
onRemoveOverride={removeOverride}
showOfflineAlertsColumn={false}
editingId={editingId}
editingThresholds={editingThresholds}
setEditingThresholds={setEditingThresholds}
formatMetricValue={formatMetricValue}
hasActiveAlert={hasActiveAlert}
globalDefaults={snapshotDefaultsRecord()}
setGlobalDefaults={(value) => {
updateSnapshotDefaults((prev) => {
const currentRecord = {
'warning days': prev.warningDays ?? 0,
'critical days': prev.criticalDays ?? 0,
};
const nextRecord =
typeof value === 'function'
? value(currentRecord)
: { ...currentRecord, ...value };
return {
...prev,
warningDays:
typeof nextRecord['warning days'] === 'number'
? nextRecord['warning days']
: prev.warningDays,
criticalDays:
typeof nextRecord['critical days'] === 'number'
? nextRecord['critical days']
: prev.criticalDays,
};
});
}}
setHasUnsavedChanges={props.setHasUnsavedChanges}
globalDisableFlag={() => !props.snapshotDefaults().enabled}
onToggleGlobalDisable={() =>
updateSnapshotDefaults((prev) => ({
...prev,
enabled: !prev.enabled,
}))
}
factoryDefaults={snapshotFactoryDefaultsRecord()}
onResetDefaults={
props.resetSnapshotDefaults
? () => {
props.resetSnapshotDefaults?.();
props.setHasUnsavedChanges(true);
}
: () =>
updateSnapshotDefaults({
...snapshotFactoryConfig(),
})
}
/>
</div>
</Show>
<Show when={hasSection('storage')}>
<div ref={registerSection('storage')} class="scroll-mt-24">
<ResourceTable

View File

@@ -6,7 +6,7 @@ import {
ThresholdsTable,
normalizeDockerIgnoredInput,
} from '../ThresholdsTable';
import type { PMGThresholdDefaults } from '@/types/alerts';
import type { PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
vi.mock('@solidjs/router', () => ({
useNavigate: () => vi.fn(),
@@ -87,6 +87,10 @@ const baseProps = () => ({
factoryNodeDefaults: {},
factoryDockerDefaults: {},
factoryStorageDefault: 85,
snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }),
setSnapshotDefaults: vi.fn(),
snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig,
resetSnapshotDefaults: vi.fn(),
timeThresholds: () => ({ guest: 5, node: 5, storage: 5, pbs: 5 }),
metricTimeThresholds: () => ({}),
setMetricTimeThresholds: vi.fn(),

View File

@@ -3,18 +3,18 @@ import type { JSX } from 'solid-js';
import { EmailProviderSelect } from '@/components/Alerts/EmailProviderSelect';
import { WebhookConfig } from '@/components/Alerts/WebhookConfig';
import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable';
import type { RawOverrideConfig, PMGThresholdDefaults } from '@/types/alerts';
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
import { Card } from '@/components/shared/Card';
import { SectionHeader } from '@/components/shared/SectionHeader';
import { SettingsPanel } from '@/components/shared/SettingsPanel';
import { Toggle } from '@/components/shared/Toggle';
import { formField, labelClass, controlClass, formHelpText } from '@/components/shared/Form';
import { formField, formControl, formHelpText, labelClass, controlClass } from '@/components/shared/Form';
import { useWebSocket } from '@/App';
import { showSuccess, showError } from '@/utils/toast';
import { showTooltip, hideTooltip } from '@/components/shared/Tooltip';
import { AlertsAPI } from '@/api/alerts';
import { NotificationsAPI, Webhook } from '@/api/notifications';
import type { EmailConfig } from '@/api/notifications';
import type { EmailConfig, AppriseConfig } from '@/api/notifications';
import type { HysteresisThreshold } from '@/types/alerts';
import type { Alert, State, VM, Container, DockerHost, DockerContainer } from '@/types/api';
import { useNavigate, useLocation } from '@solidjs/router';
@@ -98,6 +98,7 @@ export const tabFromPath = (
// Store reference interfaces
interface DestinationsRef {
emailConfig?: () => EmailConfig;
appriseConfig?: () => AppriseConfig;
}
// Override interface for both guests and nodes
@@ -151,6 +152,13 @@ interface UIEmailConfig {
rateLimit: number;
}
interface UIAppriseConfig {
enabled: boolean;
targetsText: string;
cliPath: string;
timeoutSeconds: number;
}
interface QuietHoursConfig {
enabled: boolean;
start: string;
@@ -226,6 +234,22 @@ export const createDefaultGrouping = (): GroupingConfig => ({
byGuest: false,
});
const createDefaultAppriseConfig = (): UIAppriseConfig => ({
enabled: false,
targetsText: '',
cliPath: 'apprise',
timeoutSeconds: 15,
});
const parseAppriseTargets = (value: string): string[] =>
value
.split(/\r?\n|,/)
.map((entry) => entry.trim())
.filter((entry, index, arr) => entry.length > 0 && arr.indexOf(entry) === index);
const formatAppriseTargets = (targets: string[] | undefined | null): string =>
targets && targets.length > 0 ? targets.join('\n') : '';
export const normalizeMetricDelayMap = (
input: Record<string, Record<string, number>> | undefined | null,
): Record<string, Record<string, number>> => {
@@ -348,11 +372,11 @@ export function Alerts() {
>({}); // Store raw config
// Email configuration state moved to parent to persist across tab changes
const [emailConfig, setEmailConfig] = createSignal<UIEmailConfig>({
enabled: false,
provider: '',
server: '', // Fixed: use 'server' not 'smtpHost'
port: 587, // Fixed: use 'port' not 'smtpPort'
const [emailConfig, setEmailConfig] = createSignal<UIEmailConfig>({
enabled: false,
provider: '',
server: '', // Fixed: use 'server' not 'smtpHost'
port: 587, // Fixed: use 'port' not 'smtpPort'
username: '',
password: '',
from: '',
@@ -362,8 +386,12 @@ export function Alerts() {
replyTo: '',
maxRetries: 3,
retryDelay: 5,
rateLimit: 60,
});
rateLimit: 60,
});
const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
createDefaultAppriseConfig(),
);
// Schedule configuration state moved to parent to persist across tab changes
const [scheduleQuietHours, setScheduleQuietHours] =
@@ -395,6 +423,16 @@ export function Alerts() {
} as EmailConfig;
};
destinationsRef.appriseConfig = () => {
const config = appriseConfig();
return {
enabled: config.enabled,
targets: parseAppriseTargets(config.targetsText),
cliPath: config.cliPath,
timeoutSeconds: config.timeoutSeconds,
} as AppriseConfig;
};
// Process raw overrides config when state changes
createEffect(() => {
// Skip this effect if there are unsaved changes to prevent losing focus
@@ -658,6 +696,8 @@ export function Alerts() {
rateLimit: 60,
});
setAppriseConfig(createDefaultAppriseConfig());
try {
const config = await AlertsAPI.getConfig();
@@ -729,6 +769,29 @@ export function Alerts() {
setMetricTimeThresholds({});
}
// Load snapshot thresholds
if (config.snapshotDefaults) {
const enabled = Boolean(config.snapshotDefaults.enabled);
const rawWarning = config.snapshotDefaults.warningDays ?? 30;
const rawCritical = config.snapshotDefaults.criticalDays ?? 45;
const safeCritical = Math.max(0, rawCritical);
const normalizedWarning = Math.max(0, rawWarning);
const warningDays =
safeCritical > 0 && normalizedWarning > safeCritical ? safeCritical : normalizedWarning;
const criticalDays = Math.max(safeCritical, warningDays);
setSnapshotDefaults({
enabled,
warningDays,
criticalDays,
});
} else {
setSnapshotDefaults({
enabled: false,
warningDays: 30,
criticalDays: 45,
});
}
// Load PMG thresholds
if (config.pmgDefaults) {
setPMGThresholds({
@@ -867,6 +930,21 @@ export function Alerts() {
console.error('Failed to load email configuration:', emailErr);
}
try {
const appriseData = await NotificationsAPI.getAppriseConfig();
setAppriseConfig({
enabled: appriseData.enabled ?? false,
targetsText: formatAppriseTargets(appriseData.targets),
cliPath: appriseData.cliPath || 'apprise',
timeoutSeconds:
typeof appriseData.timeoutSeconds === 'number' && appriseData.timeoutSeconds > 0
? appriseData.timeoutSeconds
: 15,
});
} catch (appriseErr) {
console.error('Failed to load Apprise configuration:', appriseErr);
}
if (options.notify) {
showSuccess('Changes discarded');
}
@@ -911,6 +989,22 @@ export function Alerts() {
.catch((err) => {
console.error('Failed to reload email configuration:', err);
});
NotificationsAPI.getAppriseConfig()
.then((appriseData) => {
setAppriseConfig({
enabled: appriseData.enabled ?? false,
targetsText: formatAppriseTargets(appriseData.targets),
cliPath: appriseData.cliPath || 'apprise',
timeoutSeconds:
typeof appriseData.timeoutSeconds === 'number' && appriseData.timeoutSeconds > 0
? appriseData.timeoutSeconds
: 15,
});
})
.catch((err) => {
console.error('Failed to reload Apprise configuration:', err);
});
}
});
@@ -959,6 +1053,11 @@ export function Alerts() {
};
const FACTORY_STORAGE_DEFAULT = 85;
const FACTORY_SNAPSHOT_DEFAULTS: SnapshotAlertConfig = {
enabled: false,
warningDays: 30,
criticalDays: 45,
};
// Threshold states - using trigger values for display
const [guestDefaults, setGuestDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_GUEST_DEFAULTS });
@@ -997,15 +1096,22 @@ export function Alerts() {
setStorageDefault(FACTORY_STORAGE_DEFAULT);
setHasUnsavedChanges(true);
};
const [timeThreshold, setTimeThreshold] = createSignal(DEFAULT_DELAY_SECONDS); // Legacy
const [timeThresholds, setTimeThresholds] = createSignal({
guest: DEFAULT_DELAY_SECONDS,
node: DEFAULT_DELAY_SECONDS,
storage: DEFAULT_DELAY_SECONDS,
pbs: DEFAULT_DELAY_SECONDS,
});
const resetSnapshotDefaults = () => {
setSnapshotDefaults({ ...FACTORY_SNAPSHOT_DEFAULTS });
setHasUnsavedChanges(true);
};
const [timeThreshold, setTimeThreshold] = createSignal(DEFAULT_DELAY_SECONDS); // Legacy
const [timeThresholds, setTimeThresholds] = createSignal({
guest: DEFAULT_DELAY_SECONDS,
node: DEFAULT_DELAY_SECONDS,
storage: DEFAULT_DELAY_SECONDS,
pbs: DEFAULT_DELAY_SECONDS,
});
const [metricTimeThresholds, setMetricTimeThresholds] =
createSignal<Record<string, Record<string, number>>>({});
const [snapshotDefaults, setSnapshotDefaults] = createSignal<SnapshotAlertConfig>({
...FACTORY_SNAPSHOT_DEFAULTS,
});
const [pmgThresholds, setPMGThresholds] = createSignal({
queueTotalWarning: 500,
@@ -1117,6 +1223,14 @@ const [timeThresholds, setTimeThresholds] = createSignal({
};
};
const snapshotConfig = snapshotDefaults();
const normalizedWarningDays = Math.max(0, snapshotConfig.warningDays ?? 0);
const normalizedCriticalDays = Math.max(0, snapshotConfig.criticalDays ?? 0);
const finalCriticalDays =
normalizedCriticalDays > 0
? Math.max(normalizedCriticalDays, normalizedWarningDays)
: normalizedWarningDays;
const alertConfig = {
enabled: true,
// Global disable flags per resource type
@@ -1168,6 +1282,11 @@ const [timeThresholds, setTimeThresholds] = createSignal({
timeThreshold: timeThreshold() || 0, // Legacy
timeThresholds: timeThresholds(),
metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()),
snapshotDefaults: {
enabled: snapshotConfig.enabled,
warningDays: normalizedWarningDays,
criticalDays: finalCriticalDays,
},
pmgDefaults: pmgThresholds(),
// Use rawOverridesConfig which is already properly formatted with disabled flags
overrides: rawOverridesConfig(),
@@ -1217,6 +1336,20 @@ const [timeThresholds, setTimeThresholds] = createSignal({
await NotificationsAPI.updateEmailConfig(emailData);
}
if (destinationsRef.appriseConfig) {
const appriseData = destinationsRef.appriseConfig();
const updatedApprise = await NotificationsAPI.updateAppriseConfig(appriseData);
setAppriseConfig({
enabled: updatedApprise.enabled ?? false,
targetsText: formatAppriseTargets(updatedApprise.targets),
cliPath: updatedApprise.cliPath || 'apprise',
timeoutSeconds:
typeof updatedApprise.timeoutSeconds === 'number' && updatedApprise.timeoutSeconds > 0
? updatedApprise.timeoutSeconds
: 15,
});
}
setHasUnsavedChanges(false);
showSuccess('Configuration saved successfully!');
} catch (err) {
@@ -1357,13 +1490,17 @@ const [timeThresholds, setTimeThresholds] = createSignal({
resetDockerDefaults={resetDockerDefaults}
resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes}
resetStorageDefault={resetStorageDefault}
resetSnapshotDefaults={resetSnapshotDefaults}
factoryGuestDefaults={FACTORY_GUEST_DEFAULTS}
factoryNodeDefaults={FACTORY_NODE_DEFAULTS}
factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS}
factoryStorageDefault={FACTORY_STORAGE_DEFAULT}
snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS}
timeThresholds={timeThresholds}
metricTimeThresholds={metricTimeThresholds}
setMetricTimeThresholds={setMetricTimeThresholds}
snapshotDefaults={snapshotDefaults}
setSnapshotDefaults={setSnapshotDefaults}
pmgThresholds={pmgThresholds}
setPMGThresholds={setPMGThresholds}
activeAlerts={activeAlerts}
@@ -1404,6 +1541,8 @@ const [timeThresholds, setTimeThresholds] = createSignal({
setHasUnsavedChanges={setHasUnsavedChanges}
emailConfig={emailConfig}
setEmailConfig={setEmailConfig}
appriseConfig={appriseConfig}
setAppriseConfig={setAppriseConfig}
/>
</Show>
@@ -1881,6 +2020,14 @@ interface ThresholdsTabProps {
| Record<string, Record<string, number>>
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
) => void;
snapshotDefaults: () => SnapshotAlertConfig;
setSnapshotDefaults: (
value:
| SnapshotAlertConfig
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
) => void;
snapshotFactoryDefaults: SnapshotAlertConfig;
resetSnapshotDefaults: () => void;
setOverrides: (value: Override[]) => void;
setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void;
activeAlerts: Record<string, Alert>;
@@ -1926,7 +2073,6 @@ interface ThresholdsTabProps {
}
function ThresholdsTab(props: ThresholdsTabProps) {
// Use the new table component for a cleaner, more information-dense layout
return (
<ThresholdsTable
overrides={props.overrides}
@@ -1958,6 +2104,10 @@ function ThresholdsTab(props: ThresholdsTabProps) {
timeThresholds={props.timeThresholds}
metricTimeThresholds={props.metricTimeThresholds}
setMetricTimeThresholds={props.setMetricTimeThresholds}
snapshotDefaults={props.snapshotDefaults}
setSnapshotDefaults={props.setSnapshotDefaults}
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
resetSnapshotDefaults={props.resetSnapshotDefaults}
setHasUnsavedChanges={props.setHasUnsavedChanges}
activeAlerts={props.activeAlerts}
removeAlerts={props.removeAlerts}
@@ -2005,13 +2155,18 @@ interface DestinationsTabProps {
setHasUnsavedChanges: (value: boolean) => void;
emailConfig: () => UIEmailConfig;
setEmailConfig: (config: UIEmailConfig) => void;
appriseConfig: () => UIAppriseConfig;
setAppriseConfig: (config: UIAppriseConfig) => void;
}
function DestinationsTab(props: DestinationsTabProps) {
const [webhooks, setWebhooks] = createSignal<Webhook[]>([]);
const [testingEmail, setTestingEmail] = createSignal(false);
const [testingWebhook, setTestingWebhook] = createSignal<string | null>(null);
const appriseState = () => props.appriseConfig();
const updateApprise = (partial: Partial<UIAppriseConfig>) => {
props.setAppriseConfig({ ...props.appriseConfig(), ...partial });
};
// Load webhooks on mount (email config is now loaded in parent)
onMount(async () => {
try {
@@ -2105,6 +2260,78 @@ function DestinationsTab(props: DestinationsTabProps) {
</div>
</SettingsPanel>
<SettingsPanel
title="Apprise notifications"
description="Relay grouped alerts through the Apprise CLI."
action={
<Toggle
checked={appriseState().enabled}
onChange={(e) => {
updateApprise({ enabled: e.currentTarget.checked });
props.setHasUnsavedChanges(true);
}}
containerClass="sm:self-start"
label={
<span class="text-xs font-medium text-gray-600 dark:text-gray-400">
{appriseState().enabled ? 'Enabled' : 'Disabled'}
</span>
}
/>
}
class="min-w-0"
bodyClass="space-y-4"
>
<div class="space-y-4">
<div class={formField}>
<label class={labelClass('text-xs uppercase tracking-[0.08em]')}>Delivery targets</label>
<textarea
rows={4}
class={`${formControl} font-mono min-h-[120px]`}
value={appriseState().targetsText}
placeholder={`discord://token\nmailto://alerts@example.com`}
onInput={(e) => {
updateApprise({ targetsText: e.currentTarget.value });
props.setHasUnsavedChanges(true);
}}
/>
<p class={formHelpText}>Enter one Apprise URL per line. Commas are also supported.</p>
</div>
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2">
<div class={formField}>
<label class={labelClass('text-xs uppercase tracking-[0.08em]')}>CLI path</label>
<input
type="text"
value={appriseState().cliPath}
class={formControl}
placeholder="apprise"
onInput={(e) => {
updateApprise({ cliPath: e.currentTarget.value });
props.setHasUnsavedChanges(true);
}}
/>
<p class={formHelpText}>Leave blank to use the default `apprise` executable.</p>
</div>
<div class={formField}>
<label class={labelClass('text-xs uppercase tracking-[0.08em]')}>Timeout (seconds)</label>
<input
type="number"
min="5"
max="120"
value={appriseState().timeoutSeconds}
class={formControl}
onInput={(e) => {
const raw = e.currentTarget.valueAsNumber;
const safe = Number.isNaN(raw) ? 15 : Math.min(120, Math.max(5, Math.trunc(raw)));
updateApprise({ timeoutSeconds: safe });
props.setHasUnsavedChanges(true);
}}
/>
<p class={formHelpText}>Maximum time to wait for the Apprise CLI to complete.</p>
</div>
</div>
</div>
</SettingsPanel>
<SettingsPanel
title="Webhooks"
description="Push alerts to chat apps or automation systems."

View File

@@ -85,6 +85,12 @@ export interface PMGThresholdDefaults {
quarantineGrowthCritMin?: number;
}
export interface SnapshotAlertConfig {
enabled: boolean;
warningDays: number;
criticalDays: number;
}
export interface AlertConfig {
enabled: boolean;
guestDefaults: AlertThresholds;
@@ -93,6 +99,7 @@ export interface AlertConfig {
dockerDefaults?: DockerThresholdConfig;
dockerIgnoredContainerPrefixes?: string[];
pmgDefaults?: PMGThresholdDefaults;
snapshotDefaults?: SnapshotAlertConfig;
customRules?: CustomAlertRule[];
overrides: Record<string, RawOverrideConfig>; // key: resource ID
minimumDelta?: number;

View File

@@ -283,6 +283,13 @@ type PMGThresholdConfig struct {
QuarantineGrowthCritMin int `json:"quarantineGrowthCritMin"` // Minimum message growth for critical (default: 500)
}
// SnapshotAlertConfig represents snapshot age alert configuration
type SnapshotAlertConfig struct {
Enabled bool `json:"enabled"`
WarningDays int `json:"warningDays"`
CriticalDays int `json:"criticalDays"`
}
// AlertConfig represents the complete alert configuration
type AlertConfig struct {
Enabled bool `json:"enabled"`
@@ -292,6 +299,7 @@ type AlertConfig struct {
DockerDefaults DockerThresholdConfig `json:"dockerDefaults"`
DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"`
PMGDefaults PMGThresholdConfig `json:"pmgDefaults"`
SnapshotDefaults SnapshotAlertConfig `json:"snapshotDefaults"`
Overrides map[string]ThresholdConfig `json:"overrides"` // keyed by resource ID
CustomRules []CustomAlertRule `json:"customRules,omitempty"`
Schedule ScheduleConfig `json:"schedule"`
@@ -473,6 +481,11 @@ func NewManager() *Manager {
QuarantineGrowthCritPct: 50, // Critical if growth ≥50%
QuarantineGrowthCritMin: 500, // AND ≥500 messages
},
SnapshotDefaults: SnapshotAlertConfig{
Enabled: false,
WarningDays: 30,
CriticalDays: 45,
},
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
MinimumDelta: 2.0, // 2% minimum change
SuppressionWindow: 5, // 5 minutes
@@ -680,6 +693,16 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
config.PMGDefaults.QuarantineGrowthCritMin = 500
}
if config.SnapshotDefaults.WarningDays < 0 {
config.SnapshotDefaults.WarningDays = 0
}
if config.SnapshotDefaults.CriticalDays < 0 {
config.SnapshotDefaults.CriticalDays = 0
}
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
}
// Ensure minimums for other important fields
if config.MinimumDelta <= 0 {
config.MinimumDelta = 2.0
@@ -738,6 +761,10 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
m.config.Overrides[id] = override
}
if !m.config.SnapshotDefaults.Enabled {
m.clearSnapshotAlertsForInstanceLocked("")
}
m.applyGlobalOfflineSettingsLocked()
log.Info().
@@ -2729,6 +2756,216 @@ func (m *Manager) CheckStorage(storage models.Storage) {
}
}
func buildSnapshotGuestKey(instance, node string, vmid int) string {
instance = strings.TrimSpace(instance)
node = strings.TrimSpace(node)
if instance == "" {
instance = node
}
if instance == node {
return fmt.Sprintf("%s-%d", node, vmid)
}
return fmt.Sprintf("%s-%s-%d", instance, node, vmid)
}
// CheckSnapshotsForInstance evaluates guest snapshots for age-based alerts.
func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []models.GuestSnapshot, guestNames map[string]string) {
m.mu.RLock()
enabled := m.config.Enabled
snapshotCfg := m.config.SnapshotDefaults
m.mu.RUnlock()
if !enabled {
return
}
if !snapshotCfg.Enabled {
m.clearSnapshotAlertsForInstance(instanceName)
return
}
now := time.Now()
validAlerts := make(map[string]struct{})
for _, snapshot := range snapshots {
if instanceName != "" && snapshot.Instance != "" && snapshot.Instance != instanceName {
continue
}
if snapshot.Time.IsZero() {
continue
}
ageHours := now.Sub(snapshot.Time).Hours()
if ageHours < 0 {
continue
}
ageDays := ageHours / 24
var level AlertLevel
var threshold int
if snapshotCfg.CriticalDays > 0 && ageDays >= float64(snapshotCfg.CriticalDays) {
level = AlertLevelCritical
threshold = snapshotCfg.CriticalDays
} else if snapshotCfg.WarningDays > 0 && ageDays >= float64(snapshotCfg.WarningDays) {
level = AlertLevelWarning
threshold = snapshotCfg.WarningDays
} else {
continue
}
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
validAlerts[alertID] = struct{}{}
guestKey := buildSnapshotGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
guestName := strings.TrimSpace(guestNames[guestKey])
guestType := "VM"
if strings.EqualFold(snapshot.Type, "lxc") {
guestType = "Container"
}
if guestName == "" {
switch guestType {
case "Container":
guestName = fmt.Sprintf("CT %d", snapshot.VMID)
default:
guestName = fmt.Sprintf("VM %d", snapshot.VMID)
}
}
snapshotName := strings.TrimSpace(snapshot.Name)
if snapshotName == "" {
snapshotName = "(unnamed)"
}
ageDaysRounded := math.Round(ageDays*10) / 10
message := fmt.Sprintf(
"%s snapshot '%s' for %s is %.1f days old on %s (threshold: %d days)",
guestType,
snapshotName,
guestName,
ageDaysRounded,
snapshot.Node,
threshold,
)
thresholdTime := snapshot.Time.Add(time.Duration(threshold) * 24 * time.Hour)
if thresholdTime.After(now) {
thresholdTime = now
}
metadata := map[string]interface{}{
"snapshotName": snapshot.Name,
"snapshotCreatedAt": snapshot.Time,
"snapshotAgeDays": ageDays,
"snapshotAgeHours": ageHours,
"guestName": guestName,
"guestType": guestType,
"guestInstance": snapshot.Instance,
"guestNode": snapshot.Node,
"guestVmid": snapshot.VMID,
"thresholdDays": threshold,
}
resourceName := fmt.Sprintf("%s snapshot '%s'", guestName, snapshotName)
m.mu.Lock()
if existing, exists := m.activeAlerts[alertID]; exists {
existing.LastSeen = now
existing.Level = level
existing.Value = ageDays
existing.Threshold = float64(threshold)
existing.Message = message
existing.ResourceName = resourceName
if existing.Metadata == nil {
existing.Metadata = make(map[string]interface{})
}
for k, v := range metadata {
existing.Metadata[k] = v
}
m.mu.Unlock()
continue
}
alert := &Alert{
ID: alertID,
Type: "snapshot-age",
Level: level,
ResourceID: snapshot.ID,
ResourceName: resourceName,
Node: snapshot.Node,
Instance: snapshot.Instance,
Message: message,
Value: ageDays,
Threshold: float64(threshold),
StartTime: thresholdTime,
LastSeen: now,
Metadata: metadata,
}
m.preserveAlertState(alertID, alert)
m.activeAlerts[alertID] = alert
m.recentAlerts[alertID] = alert
m.historyManager.AddAlert(*alert)
go func() {
defer func() {
if r := recover(); r != nil {
log.Error().Interface("panic", r).Msg("Panic in SaveActiveAlerts goroutine (snapshot)")
}
}()
if err := m.SaveActiveAlerts(); err != nil {
log.Error().Err(err).Msg("Failed to save active alerts after snapshot alert creation")
}
}()
if !m.checkRateLimit(alertID) {
m.mu.Unlock()
log.Debug().
Str("alertID", alertID).
Str("guest", guestName).
Msg("Snapshot alert suppressed due to rate limit")
continue
}
if m.onAlert != nil {
nowCopy := now
alert.LastNotified = &nowCopy
if m.dispatchAlert(alert, true) {
log.Info().
Str("alertID", alertID).
Str("guest", guestName).
Msg("Snapshot age alert dispatched")
} else {
alert.LastNotified = nil
}
} else {
log.Warn().
Str("alertID", alertID).
Msg("Snapshot age alert created but no onAlert callback set")
}
m.mu.Unlock()
}
m.mu.Lock()
for alertID, alert := range m.activeAlerts {
if alert == nil || alert.Type != "snapshot-age" {
continue
}
if instanceName != "" && alert.Instance != instanceName {
continue
}
if _, ok := validAlerts[alertID]; ok {
continue
}
m.clearAlertNoLock(alertID)
}
m.mu.Unlock()
}
// checkZFSPoolHealth checks ZFS pool for errors and degraded state
func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
pool := storage.ZFSPool
@@ -6208,3 +6445,21 @@ func (m *Manager) clearAlertNoLock(alertID string) {
Str("alertID", alertID).
Msg("Alert cleared")
}
func (m *Manager) clearSnapshotAlertsForInstance(instance string) {
m.mu.Lock()
m.clearSnapshotAlertsForInstanceLocked(instance)
m.mu.Unlock()
}
func (m *Manager) clearSnapshotAlertsForInstanceLocked(instance string) {
for alertID, alert := range m.activeAlerts {
if alert == nil || alert.Type != "snapshot-age" {
continue
}
if instance != "" && alert.Instance != instance {
continue
}
m.clearAlertNoLock(alertID)
}
}

View File

@@ -57,7 +57,7 @@ func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
GuestDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
},
TimeThreshold: 0,
TimeThreshold: 0,
TimeThresholds: map[string]int{},
NodeDefaults: ThresholdConfig{
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
@@ -212,6 +212,67 @@ func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
}
}
func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
m := NewManager()
m.ClearActiveAlerts()
cfg := AlertConfig{
Enabled: true,
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
SnapshotDefaults: SnapshotAlertConfig{
Enabled: true,
WarningDays: 7,
CriticalDays: 14,
},
Overrides: make(map[string]ThresholdConfig),
}
m.UpdateConfig(cfg)
m.mu.Lock()
m.config.TimeThreshold = 0
m.config.TimeThresholds = map[string]int{}
m.mu.Unlock()
now := time.Now()
snapshots := []models.GuestSnapshot{
{
ID: "inst-node-100-weekly",
Name: "weekly",
Node: "node",
Instance: "inst",
Type: "qemu",
VMID: 100,
Time: now.Add(-15 * 24 * time.Hour),
},
}
guestNames := map[string]string{
"inst-node-100": "app-server",
}
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
m.mu.RLock()
alert, exists := m.activeAlerts["snapshot-age-inst-node-100-weekly"]
m.mu.RUnlock()
if !exists {
t.Fatalf("expected snapshot age alert to be created")
}
if alert.Level != AlertLevelCritical {
t.Fatalf("expected critical level for old snapshot, got %s", alert.Level)
}
if alert.ResourceName != "app-server snapshot 'weekly'" {
t.Fatalf("unexpected resource name: %s", alert.ResourceName)
}
m.CheckSnapshotsForInstance("inst", nil, guestNames)
m.mu.RLock()
_, exists = m.activeAlerts["snapshot-age-inst-node-100-weekly"]
m.mu.RUnlock()
if exists {
t.Fatalf("expected snapshot alert to be cleared when snapshot missing")
}
}
func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
m := NewManager()

View File

@@ -87,6 +87,51 @@ func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.
json.NewEncoder(w).Encode(map[string]string{"status": "success"})
}
// GetAppriseConfig returns the current Apprise configuration.
func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) {
config := h.monitor.GetNotificationManager().GetAppriseConfig()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(config); err != nil {
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
}
}
// UpdateAppriseConfig updates the Apprise configuration.
func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var config notifications.AppriseConfig
if err := json.Unmarshal(body, &config); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
log.Info().
Bool("enabled", config.Enabled).
Int("targetCount", len(config.Targets)).
Str("cliPath", config.CLIPath).
Int("timeoutSeconds", config.TimeoutSeconds).
Msg("Parsed Apprise configuration update")
h.monitor.GetNotificationManager().SetAppriseConfig(config)
if err := h.monitor.GetConfigPersistence().SaveAppriseConfig(config); err != nil {
log.Error().Err(err).Msg("Failed to save Apprise configuration")
}
normalized := h.monitor.GetNotificationManager().GetAppriseConfig()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(normalized); err != nil {
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
}
}
// GetWebhooks returns all webhook configurations
func (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) {
webhooks := h.monitor.GetNotificationManager().GetWebhooks()
@@ -479,6 +524,10 @@ func (h *NotificationHandlers) HandleNotifications(w http.ResponseWriter, r *htt
h.GetEmailConfig(w, r)
case path == "/email" && r.Method == http.MethodPut:
h.UpdateEmailConfig(w, r)
case path == "/apprise" && r.Method == http.MethodGet:
h.GetAppriseConfig(w, r)
case path == "/apprise" && r.Method == http.MethodPut:
h.UpdateAppriseConfig(w, r)
case path == "/webhooks" && r.Method == http.MethodGet:
h.GetWebhooks(w, r)
case path == "/webhooks" && r.Method == http.MethodPost:

View File

@@ -25,6 +25,7 @@ type ExportData struct {
Alerts alerts.AlertConfig `json:"alerts"`
Email notifications.EmailConfig `json:"email"`
Webhooks []notifications.WebhookConfig `json:"webhooks"`
Apprise notifications.AppriseConfig `json:"apprise"`
System SystemSettings `json:"system"`
GuestMetadata map[string]*GuestMetadata `json:"guestMetadata,omitempty"`
OIDC *OIDCConfig `json:"oidc,omitempty"`
@@ -55,6 +56,11 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
return "", fmt.Errorf("failed to load email config: %w", err)
}
appriseConfig, err := c.LoadAppriseConfig()
if err != nil {
return "", fmt.Errorf("failed to load Apprise config: %w", err)
}
webhooks, err := c.LoadWebhooks()
if err != nil {
return "", fmt.Errorf("failed to load webhooks: %w", err)
@@ -90,6 +96,7 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
Alerts: *alertConfig,
Email: *emailConfig,
Webhooks: webhooks,
Apprise: *appriseConfig,
System: *systemSettings,
GuestMetadata: guestMetadata,
OIDC: oidcConfig,
@@ -153,6 +160,10 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
return fmt.Errorf("failed to import email config: %w", err)
}
if err := c.SaveAppriseConfig(exportData.Apprise); err != nil {
return fmt.Errorf("failed to import Apprise config: %w", err)
}
if err := c.SaveWebhooks(exportData.Webhooks); err != nil {
return fmt.Errorf("failed to import webhooks: %w", err)
}

View File

@@ -24,6 +24,7 @@ type ConfigPersistence struct {
alertFile string
emailFile string
webhookFile string
appriseFile string
nodesFile string
systemFile string
oidcFile string
@@ -49,6 +50,7 @@ func NewConfigPersistence(configDir string) *ConfigPersistence {
alertFile: filepath.Join(configDir, "alerts.json"),
emailFile: filepath.Join(configDir, "email.enc"),
webhookFile: filepath.Join(configDir, "webhooks.enc"),
appriseFile: filepath.Join(configDir, "apprise.enc"),
nodesFile: filepath.Join(configDir, "nodes.enc"),
systemFile: filepath.Join(configDir, "system.json"),
oidcFile: filepath.Join(configDir, "oidc.enc"),
@@ -183,6 +185,15 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
config.TimeThresholds["all"] = config.TimeThreshold
}
if config.SnapshotDefaults.WarningDays < 0 {
config.SnapshotDefaults.WarningDays = 0
}
if config.SnapshotDefaults.CriticalDays < 0 {
config.SnapshotDefaults.CriticalDays = 0
}
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
}
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
data, err := json.MarshalIndent(config, "", " ")
@@ -235,7 +246,12 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
MinimumDelta: 2.0,
SuppressionWindow: 5,
HysteresisMargin: 5.0,
Overrides: make(map[string]alerts.ThresholdConfig),
SnapshotDefaults: alerts.SnapshotAlertConfig{
Enabled: false,
WarningDays: 30,
CriticalDays: 45,
},
Overrides: make(map[string]alerts.ThresholdConfig),
}, nil
}
return nil, err
@@ -285,6 +301,15 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
config.TimeThresholds["all"] = config.TimeThreshold
}
if config.SnapshotDefaults.WarningDays < 0 {
config.SnapshotDefaults.WarningDays = 0
}
if config.SnapshotDefaults.CriticalDays < 0 {
config.SnapshotDefaults.CriticalDays = 0
}
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
}
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
@@ -386,6 +411,82 @@ func (c *ConfigPersistence) LoadEmailConfig() (*notifications.EmailConfig, error
return &config, nil
}
// SaveAppriseConfig saves Apprise configuration to file (encrypted if available)
func (c *ConfigPersistence) SaveAppriseConfig(config notifications.AppriseConfig) error {
c.mu.Lock()
defer c.mu.Unlock()
config = notifications.NormalizeAppriseConfig(config)
data, err := json.MarshalIndent(config, "", " ")
if err != nil {
return err
}
if err := c.EnsureConfigDir(); err != nil {
return err
}
if c.crypto != nil {
encrypted, err := c.crypto.Encrypt(data)
if err != nil {
return err
}
data = encrypted
}
if err := os.WriteFile(c.appriseFile, data, 0600); err != nil {
return err
}
log.Info().
Str("file", c.appriseFile).
Bool("encrypted", c.crypto != nil).
Msg("Apprise configuration saved")
return nil
}
// LoadAppriseConfig loads Apprise configuration from file (decrypts if encrypted)
func (c *ConfigPersistence) LoadAppriseConfig() (*notifications.AppriseConfig, error) {
c.mu.RLock()
defer c.mu.RUnlock()
data, err := os.ReadFile(c.appriseFile)
if err != nil {
if os.IsNotExist(err) {
defaultCfg := notifications.AppriseConfig{
Enabled: false,
Targets: []string{},
CLIPath: "apprise",
TimeoutSeconds: 15,
}
return &defaultCfg, nil
}
return nil, err
}
if c.crypto != nil {
decrypted, err := c.crypto.Decrypt(data)
if err != nil {
return nil, err
}
data = decrypted
}
var config notifications.AppriseConfig
if err := json.Unmarshal(data, &config); err != nil {
return nil, err
}
normalized := notifications.NormalizeAppriseConfig(config)
log.Info().
Str("file", c.appriseFile).
Bool("encrypted", c.crypto != nil).
Msg("Apprise configuration loaded")
return &normalized, nil
}
// SaveWebhooks saves webhook configurations to file
func (c *ConfigPersistence) SaveWebhooks(webhooks []notifications.WebhookConfig) error {
c.mu.Lock()

View File

@@ -9,6 +9,7 @@ import (
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
)
func TestSaveAlertConfig_PreservesStorageOverrideHysteresis(t *testing.T) {
@@ -134,6 +135,11 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
TimeThreshold: 0,
TimeThresholds: map[string]int{"guest": 0, "node": 0},
DockerIgnoredContainerPrefixes: []string{" Runner "},
SnapshotDefaults: alerts.SnapshotAlertConfig{
Enabled: true,
WarningDays: 20,
CriticalDays: 10,
},
NodeDefaults: alerts.ThresholdConfig{
Temperature: &alerts.HysteresisThreshold{Trigger: 0, Clear: 0},
},
@@ -172,4 +178,67 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
if !reflect.DeepEqual(loaded.DockerIgnoredContainerPrefixes, expectedPrefixes) {
t.Fatalf("expected normalized prefixes %v, got %v", expectedPrefixes, loaded.DockerIgnoredContainerPrefixes)
}
if loaded.SnapshotDefaults.Enabled != true {
t.Fatalf("expected snapshot defaults to preserve enabled state")
}
if loaded.SnapshotDefaults.WarningDays != 10 {
t.Fatalf("expected snapshot warning days normalized to critical, got %d", loaded.SnapshotDefaults.WarningDays)
}
if loaded.SnapshotDefaults.CriticalDays != 10 {
t.Fatalf("expected snapshot critical days preserved at 10, got %d", loaded.SnapshotDefaults.CriticalDays)
}
}
func TestAppriseConfigPersistence(t *testing.T) {
tempDir := t.TempDir()
cp := config.NewConfigPersistence(tempDir)
if err := cp.EnsureConfigDir(); err != nil {
t.Fatalf("EnsureConfigDir: %v", err)
}
cfg := notifications.AppriseConfig{
Enabled: true,
Targets: []string{" discord://token ", "", "mailto://alerts@example.com"},
CLIPath: " /usr/local/bin/apprise ",
TimeoutSeconds: 3,
}
if err := cp.SaveAppriseConfig(cfg); err != nil {
t.Fatalf("SaveAppriseConfig: %v", err)
}
loaded, err := cp.LoadAppriseConfig()
if err != nil {
t.Fatalf("LoadAppriseConfig: %v", err)
}
if !loaded.Enabled {
t.Fatalf("expected config to remain enabled")
}
expectedTargets := []string{"discord://token", "mailto://alerts@example.com"}
if !reflect.DeepEqual(loaded.Targets, expectedTargets) {
t.Fatalf("unexpected targets: got %v want %v", loaded.Targets, expectedTargets)
}
if loaded.CLIPath != "/usr/local/bin/apprise" {
t.Fatalf("expected CLI path to be trimmed, got %q", loaded.CLIPath)
}
if loaded.TimeoutSeconds != 5 {
t.Fatalf("expected timeout normalized to minimum 5 seconds, got %d", loaded.TimeoutSeconds)
}
// Clearing targets should disable the config on next load
if err := cp.SaveAppriseConfig(notifications.AppriseConfig{Enabled: true}); err != nil {
t.Fatalf("SaveAppriseConfig empty: %v", err)
}
empty, err := cp.LoadAppriseConfig()
if err != nil {
t.Fatalf("LoadAppriseConfig empty: %v", err)
}
if empty.Enabled {
t.Fatalf("expected disabled configuration when no targets stored")
}
}

View File

@@ -1363,6 +1363,12 @@ func New(cfg *config.Config) (*Monitor, error) {
log.Warn().Err(err).Msg("Failed to load email configuration")
}
if appriseConfig, err := m.configPersist.LoadAppriseConfig(); err == nil {
m.notificationMgr.SetAppriseConfig(*appriseConfig)
} else {
log.Warn().Err(err).Msg("Failed to load Apprise configuration")
}
// Migrate webhooks if needed (from unencrypted to encrypted)
if err := m.configPersist.MigrateWebhooksIfNeeded(); err != nil {
log.Warn().Err(err).Msg("Failed to migrate webhooks")
@@ -5686,6 +5692,21 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
}
m.mu.RUnlock()
guestKey := func(instance, node string, vmid int) string {
if instance == node {
return fmt.Sprintf("%s-%d", node, vmid)
}
return fmt.Sprintf("%s-%s-%d", instance, node, vmid)
}
guestNames := make(map[string]string, len(vms)+len(containers))
for _, vm := range vms {
guestNames[guestKey(instanceName, vm.Node, vm.VMID)] = vm.Name
}
for _, ct := range containers {
guestNames[guestKey(instanceName, ct.Node, ct.VMID)] = ct.Name
}
activeGuests := 0
for _, vm := range vms {
if !vm.Template {
@@ -5857,6 +5878,10 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
// Update state with guest snapshots for this instance
m.state.UpdateGuestSnapshotsForInstance(instanceName, allSnapshots)
if m.alertManager != nil {
m.alertManager.CheckSnapshotsForInstance(instanceName, allSnapshots, guestNames)
}
log.Debug().
Str("instance", instanceName).
Int("count", len(allSnapshots)).

View File

@@ -2,11 +2,13 @@ package notifications
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os/exec"
"strings"
"sync"
"text/template"
@@ -103,6 +105,7 @@ type NotificationManager struct {
mu sync.RWMutex
emailConfig EmailConfig
webhooks []WebhookConfig
appriseConfig AppriseConfig
enabled bool
cooldown time.Duration
lastNotified map[string]notificationRecord
@@ -114,8 +117,11 @@ type NotificationManager struct {
groupByGuest bool
webhookHistory []WebhookDelivery // Keep last 100 webhook deliveries for debugging
webhookRateLimits map[string]*webhookRateLimit // Track rate limits per webhook URL
appriseExec appriseExecFunc
}
type appriseExecFunc func(ctx context.Context, path string, args []string) ([]byte, error)
// copyEmailConfig returns a defensive copy of EmailConfig including its slices to avoid data races.
func copyEmailConfig(cfg EmailConfig) EmailConfig {
copy := cfg
@@ -154,6 +160,58 @@ func copyWebhookConfigs(webhooks []WebhookConfig) []WebhookConfig {
return copies
}
func copyAppriseConfig(cfg AppriseConfig) AppriseConfig {
copy := cfg
if len(cfg.Targets) > 0 {
copy.Targets = append([]string(nil), cfg.Targets...)
}
return copy
}
// NormalizeAppriseConfig cleans and normalizes Apprise configuration values.
func NormalizeAppriseConfig(cfg AppriseConfig) AppriseConfig {
normalized := cfg
normalized.CLIPath = strings.TrimSpace(normalized.CLIPath)
if normalized.CLIPath == "" {
normalized.CLIPath = "apprise"
}
if normalized.TimeoutSeconds <= 0 {
normalized.TimeoutSeconds = 15
} else if normalized.TimeoutSeconds > 120 {
normalized.TimeoutSeconds = 120
} else if normalized.TimeoutSeconds < 5 {
normalized.TimeoutSeconds = 5
}
cleanTargets := make([]string, 0, len(normalized.Targets))
seen := make(map[string]struct{}, len(normalized.Targets))
for _, target := range normalized.Targets {
trimmed := strings.TrimSpace(target)
if trimmed == "" {
continue
}
lower := strings.ToLower(trimmed)
if _, exists := seen[lower]; exists {
continue
}
seen[lower] = struct{}{}
cleanTargets = append(cleanTargets, trimmed)
}
normalized.Targets = cleanTargets
if len(cleanTargets) == 0 {
normalized.Enabled = false
}
return normalized
}
func defaultAppriseExec(ctx context.Context, path string, args []string) ([]byte, error) {
cmd := exec.CommandContext(ctx, path, args...)
return cmd.CombinedOutput()
}
type notificationRecord struct {
lastSent time.Time
alertStart time.Time
@@ -200,6 +258,14 @@ type WebhookConfig struct {
CustomFields map[string]string `json:"customFields,omitempty"`
}
// AppriseConfig holds Apprise CLI notification settings.
type AppriseConfig struct {
Enabled bool `json:"enabled"`
Targets []string `json:"targets"`
CLIPath string `json:"cliPath,omitempty"`
TimeoutSeconds int `json:"timeoutSeconds,omitempty"`
}
// NewNotificationManager creates a new notification manager
func NewNotificationManager(publicURL string) *NotificationManager {
if publicURL != "" {
@@ -208,10 +274,16 @@ func NewNotificationManager(publicURL string) *NotificationManager {
log.Info().Msg("NotificationManager initialized without public URL - webhook links may not work")
}
return &NotificationManager{
enabled: true,
cooldown: 5 * time.Minute,
lastNotified: make(map[string]notificationRecord),
webhooks: []WebhookConfig{},
enabled: true,
cooldown: 5 * time.Minute,
lastNotified: make(map[string]notificationRecord),
webhooks: []WebhookConfig{},
appriseConfig: AppriseConfig{
Enabled: false,
Targets: []string{},
CLIPath: "apprise",
TimeoutSeconds: 15,
},
groupWindow: 30 * time.Second,
pendingAlerts: make([]*alerts.Alert, 0),
groupByNode: true,
@@ -219,6 +291,7 @@ func NewNotificationManager(publicURL string) *NotificationManager {
webhookHistory: make([]WebhookDelivery, 0, WebhookHistoryMaxSize),
webhookRateLimits: make(map[string]*webhookRateLimit),
publicURL: publicURL,
appriseExec: defaultAppriseExec,
}
}
@@ -229,6 +302,20 @@ func (n *NotificationManager) SetEmailConfig(config EmailConfig) {
n.emailConfig = config
}
// SetAppriseConfig updates Apprise configuration.
func (n *NotificationManager) SetAppriseConfig(config AppriseConfig) {
n.mu.Lock()
defer n.mu.Unlock()
n.appriseConfig = NormalizeAppriseConfig(config)
}
// GetAppriseConfig returns a copy of the Apprise configuration.
func (n *NotificationManager) GetAppriseConfig() AppriseConfig {
n.mu.RLock()
defer n.mu.RUnlock()
return copyAppriseConfig(n.appriseConfig)
}
// SetCooldown updates the cooldown duration
func (n *NotificationManager) SetCooldown(minutes int) {
n.mu.Lock()
@@ -434,6 +521,7 @@ func (n *NotificationManager) sendGroupedAlerts() {
// Snapshot configuration while holding the lock to avoid races with concurrent updates
emailConfig := copyEmailConfig(n.emailConfig)
webhooks := copyWebhookConfigs(n.webhooks)
appriseConfig := copyAppriseConfig(n.appriseConfig)
// Send notifications using the captured snapshots outside the lock to avoid blocking writers
if emailConfig.Enabled {
@@ -457,6 +545,10 @@ func (n *NotificationManager) sendGroupedAlerts() {
}
}
if appriseConfig.Enabled && len(appriseConfig.Targets) > 0 {
go n.sendGroupedApprise(appriseConfig, alertsToSend)
}
// Update last notified time for all alerts
now := time.Now()
for _, alert := range alertsToSend {
@@ -480,6 +572,76 @@ func (n *NotificationManager) sendGroupedEmail(config EmailConfig, alertList []*
n.sendHTMLEmail(subject, htmlBody, textBody, config)
}
func (n *NotificationManager) sendGroupedApprise(config AppriseConfig, alertList []*alerts.Alert) {
if len(alertList) == 0 {
return
}
cfg := NormalizeAppriseConfig(config)
if !cfg.Enabled || len(cfg.Targets) == 0 {
return
}
primary := alertList[0]
alertCount := len(alertList)
title := fmt.Sprintf("Pulse alert: %s", primary.ResourceName)
if alertCount > 1 {
title = fmt.Sprintf("Pulse alerts (%d)", alertCount)
}
var bodyBuilder strings.Builder
bodyBuilder.WriteString(primary.Message)
bodyBuilder.WriteString("\n\n")
for _, alert := range alertList {
bodyBuilder.WriteString(fmt.Sprintf("[%s] %s", strings.ToUpper(string(alert.Level)), alert.ResourceName))
bodyBuilder.WriteString(fmt.Sprintf(" — value %.2f (threshold %.2f)\n", alert.Value, alert.Threshold))
if alert.Node != "" {
bodyBuilder.WriteString(fmt.Sprintf("Node: %s\n", alert.Node))
}
if alert.Instance != "" && alert.Instance != alert.Node {
bodyBuilder.WriteString(fmt.Sprintf("Instance: %s\n", alert.Instance))
}
bodyBuilder.WriteString("\n")
}
if n.publicURL != "" {
bodyBuilder.WriteString("Dashboard: " + n.publicURL + "\n")
}
body := bodyBuilder.String()
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.TimeoutSeconds)*time.Second)
defer cancel()
args := []string{"-t", title, "-b", body}
args = append(args, cfg.Targets...)
execFn := n.appriseExec
if execFn == nil {
execFn = defaultAppriseExec
}
output, err := execFn(ctx, cfg.CLIPath, args)
if err != nil {
log.Warn().
Err(err).
Str("cliPath", cfg.CLIPath).
Strs("targets", cfg.Targets).
Msg("Failed to send Apprise notification")
return
}
if len(output) > 0 {
log.Debug().
Str("cliPath", cfg.CLIPath).
Strs("targets", cfg.Targets).
Str("output", string(output)).
Msg("Apprise CLI output")
}
}
// sendEmail sends an email notification
func (n *NotificationManager) sendEmail(alert *alerts.Alert) {
n.mu.RLock()

View File

@@ -1,6 +1,7 @@
package notifications
import (
"context"
"testing"
"time"
@@ -17,6 +18,96 @@ func flushPending(n *NotificationManager) {
n.sendGroupedAlerts()
}
func TestNormalizeAppriseConfig(t *testing.T) {
original := AppriseConfig{
Enabled: true,
Targets: []string{" discord://token ", "", "DISCORD://TOKEN"},
CLIPath: " ",
TimeoutSeconds: -5,
}
normalized := NormalizeAppriseConfig(original)
if normalized.CLIPath != "apprise" {
t.Fatalf("expected default CLI path 'apprise', got %q", normalized.CLIPath)
}
if normalized.TimeoutSeconds != 15 {
t.Fatalf("expected timeout of 15 seconds, got %d", normalized.TimeoutSeconds)
}
if !normalized.Enabled {
t.Fatalf("expected config to remain enabled when targets exist")
}
if len(normalized.Targets) != 1 || normalized.Targets[0] != "discord://token" {
t.Fatalf("unexpected targets normalization result: %#v", normalized.Targets)
}
// When all targets removed, enabled should reset to false
empty := NormalizeAppriseConfig(AppriseConfig{Enabled: true})
if empty.Enabled {
t.Fatalf("expected enabled to be false when no targets configured")
}
}
func TestSendGroupedAppriseInvokesExecutor(t *testing.T) {
nm := NewNotificationManager("")
nm.SetGroupingWindow(0)
nm.SetEmailConfig(EmailConfig{Enabled: false})
done := make(chan struct{})
var capturedArgs []string
nm.appriseExec = func(ctx context.Context, path string, args []string) ([]byte, error) {
if path != "apprise" {
t.Fatalf("expected CLI path 'apprise', got %q", path)
}
capturedArgs = append([]string(nil), args...)
close(done)
return []byte("success"), nil
}
nm.SetAppriseConfig(AppriseConfig{
Enabled: true,
Targets: []string{"discord://token"},
TimeoutSeconds: 10,
})
alert := &alerts.Alert{
ID: "test",
Type: "cpu",
Level: alerts.AlertLevelCritical,
ResourceID: "vm-100",
ResourceName: "vm-100",
Message: "CPU usage high",
Value: 95,
Threshold: 90,
StartTime: time.Now().Add(-time.Minute),
LastSeen: time.Now(),
}
nm.mu.Lock()
nm.pendingAlerts = append(nm.pendingAlerts, alert)
nm.mu.Unlock()
nm.sendGroupedAlerts()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatalf("timed out waiting for Apprise executor to run")
}
if len(capturedArgs) == 0 {
t.Fatalf("expected Apprise executor to receive arguments")
}
if capturedArgs[len(capturedArgs)-1] != "discord://token" {
t.Fatalf("expected target URL as last argument, got %v", capturedArgs)
}
}
func TestNotificationCooldownAllowsNewAlertInstance(t *testing.T) {
nm := NewNotificationManager("")
nm.SetCooldown(1) // 1 minute cooldown