mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-02-18 00:17:39 +01:00
Add Apprise notification integration (#570)
This commit is contained in:
10
.github/workflows/README.md
vendored
10
.github/workflows/README.md
vendored
@@ -70,13 +70,3 @@ Packages the Helm chart and pushes it to the GitHub Container Registry (OCI) whe
|
||||
- Chart and app versions mirror the Pulse release tag (e.g., `v4.24.0` → `4.24.0`)
|
||||
- Publishes to `oci://ghcr.io/<owner>/pulse-chart`
|
||||
- Requires no additional secrets—uses the built-in `GITHUB_TOKEN` with `packages: write` permission
|
||||
|
||||
## Helm Integration (Kind)
|
||||
|
||||
**File**: `helm-integration.yml`
|
||||
|
||||
Creates a disposable Kind cluster, installs the chart, waits for the hub deployment to report ready, and performs a `/health` smoke check from inside the cluster.
|
||||
|
||||
- Triggered alongside the lint workflow for PRs/pushes touching Helm content
|
||||
- Disables persistence to keep the Kind cluster lightweight
|
||||
- Provides early detection of runtime issues (missing secrets, invalid probes, etc.)
|
||||
|
||||
58
.github/workflows/helm-integration.yml
vendored
58
.github/workflows/helm-integration.yml
vendored
@@ -1,58 +0,0 @@
|
||||
name: Helm Integration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "deploy/helm/**"
|
||||
- ".github/workflows/helm-*.yml"
|
||||
- "docs/KUBERNETES.md"
|
||||
- "README.md"
|
||||
pull_request:
|
||||
paths:
|
||||
- "deploy/helm/**"
|
||||
- ".github/workflows/helm-*.yml"
|
||||
- "docs/KUBERNETES.md"
|
||||
- "README.md"
|
||||
workflow_dispatch: {}
|
||||
|
||||
jobs:
|
||||
kind-smoke-test:
|
||||
name: Deploy to Kind and Smoke Test
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.15.2
|
||||
|
||||
- name: Create Kind cluster
|
||||
uses: helm/kind-action@v1.8.0
|
||||
with:
|
||||
wait: 120s
|
||||
|
||||
- name: Install Pulse chart
|
||||
run: |
|
||||
helm upgrade --install pulse ./deploy/helm/pulse \
|
||||
--namespace pulse \
|
||||
--create-namespace \
|
||||
--set persistence.enabled=false \
|
||||
--set server.secretEnv.create=true \
|
||||
--set server.secretEnv.data.API_TOKENS=dummy-token \
|
||||
--wait \
|
||||
--timeout 5m
|
||||
|
||||
- name: Verify deployment is available
|
||||
run: kubectl -n pulse wait --for=condition=available deployment/pulse --timeout=120s
|
||||
|
||||
- name: Hit health endpoint from inside the cluster
|
||||
run: |
|
||||
kubectl -n pulse run smoke-test \
|
||||
--rm \
|
||||
--image=curlimages/curl:8.3.0 \
|
||||
--restart=Never \
|
||||
-- curl -fsS http://pulse:7655/health
|
||||
@@ -320,6 +320,12 @@ Quick start - most settings are in the web UI:
|
||||
- **Settings → Security**: Authentication and API tokens
|
||||
- **Alerts**: Thresholds and notifications
|
||||
|
||||
### Apprise CLI Notifications
|
||||
|
||||
Pulse can broadcast grouped alerts through the [Apprise](https://github.com/caronc/apprise) CLI. Install Apprise on the Pulse host (for example with `pip install apprise`) and configure the targets under **Alerts → Notifications**. Each target URL should be a valid Apprise destination (Discord, Slack, email, SMS, etc.).
|
||||
|
||||
You can also override the CLI path and execution timeout if Apprise is installed in a non-standard location. Pulse automatically skips Apprise delivery when no targets are configured.
|
||||
|
||||
### Configuration Files
|
||||
|
||||
Pulse uses three separate configuration files with clear separation of concerns:
|
||||
|
||||
@@ -57,6 +57,13 @@ export interface Webhook {
|
||||
customFields?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface AppriseConfig {
|
||||
enabled: boolean;
|
||||
targets: string[];
|
||||
cliPath?: string;
|
||||
timeoutSeconds?: number;
|
||||
}
|
||||
|
||||
export interface NotificationTestRequest {
|
||||
type: 'email' | 'webhook';
|
||||
config?: Record<string, unknown>; // Backend expects different format than frontend types
|
||||
@@ -66,6 +73,17 @@ export interface NotificationTestRequest {
|
||||
export class NotificationsAPI {
|
||||
private static baseUrl = '/api/notifications';
|
||||
|
||||
static async getAppriseConfig(): Promise<AppriseConfig> {
|
||||
return apiFetchJSON(`${this.baseUrl}/apprise`);
|
||||
}
|
||||
|
||||
static async updateAppriseConfig(config: AppriseConfig): Promise<AppriseConfig> {
|
||||
return apiFetchJSON(`${this.baseUrl}/apprise`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(config),
|
||||
});
|
||||
}
|
||||
|
||||
// Email configuration
|
||||
static async getEmailConfig(): Promise<EmailConfig> {
|
||||
const backendConfig = await apiFetchJSON<Record<string, unknown>>(`${this.baseUrl}/email`);
|
||||
|
||||
@@ -133,7 +133,15 @@ export function ResourceTable(props: ResourceTableProps) {
|
||||
return props.resources ?? [];
|
||||
};
|
||||
|
||||
const hasRows = () => flattenResources().length > 0;
|
||||
const hasRows = () => {
|
||||
if (flattenResources().length > 0) {
|
||||
return true;
|
||||
}
|
||||
if (props.groupedResources && Object.keys(props.groupedResources).length > 0) {
|
||||
return true;
|
||||
}
|
||||
return Boolean(props.globalDefaults);
|
||||
};
|
||||
|
||||
const [activeMetricInput, setActiveMetricInput] = createSignal<{ resourceId: string; metric: string } | null>(null);
|
||||
const [showDelayRow, setShowDelayRow] = createSignal(false);
|
||||
|
||||
@@ -15,7 +15,7 @@ import type {
|
||||
DockerHost,
|
||||
DockerContainer,
|
||||
} from '@/types/api';
|
||||
import type { RawOverrideConfig, PMGThresholdDefaults } from '@/types/alerts';
|
||||
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
|
||||
import { ResourceTable, Resource, GroupHeaderMeta } from './ResourceTable';
|
||||
type OverrideType =
|
||||
| 'guest'
|
||||
@@ -169,6 +169,14 @@ interface ThresholdsTableProps {
|
||||
| Record<string, Record<string, number>>
|
||||
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
|
||||
) => void;
|
||||
snapshotDefaults: () => SnapshotAlertConfig;
|
||||
setSnapshotDefaults: (
|
||||
value:
|
||||
| SnapshotAlertConfig
|
||||
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
|
||||
) => void;
|
||||
snapshotFactoryDefaults?: SnapshotAlertConfig;
|
||||
resetSnapshotDefaults?: () => void;
|
||||
setHasUnsavedChanges: (value: boolean) => void;
|
||||
activeAlerts?: Record<string, Alert>;
|
||||
removeAlerts?: (predicate: (alert: Alert) => boolean) => void;
|
||||
@@ -200,6 +208,9 @@ interface ThresholdsTableProps {
|
||||
setDisableAllDockerHostsOffline: (value: boolean) => void;
|
||||
}
|
||||
|
||||
const DEFAULT_SNAPSHOT_WARNING = 30;
|
||||
const DEFAULT_SNAPSHOT_CRITICAL = 45;
|
||||
|
||||
export function ThresholdsTable(props: ThresholdsTableProps) {
|
||||
const navigate = useNavigate();
|
||||
const location = useLocation();
|
||||
@@ -744,6 +755,74 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
/* no-op placeholder for future scroll restoration */
|
||||
};
|
||||
|
||||
const snapshotFactoryConfig = () =>
|
||||
props.snapshotFactoryDefaults ?? {
|
||||
enabled: false,
|
||||
warningDays: DEFAULT_SNAPSHOT_WARNING,
|
||||
criticalDays: DEFAULT_SNAPSHOT_CRITICAL,
|
||||
};
|
||||
|
||||
const sanitizeSnapshotConfig = (config: SnapshotAlertConfig): SnapshotAlertConfig => {
|
||||
let warning = Math.max(0, Math.round(config.warningDays ?? 0));
|
||||
let critical = Math.max(0, Math.round(config.criticalDays ?? 0));
|
||||
|
||||
if (critical > 0 && warning > critical) {
|
||||
warning = critical;
|
||||
}
|
||||
if (critical === 0 && warning > 0) {
|
||||
critical = warning;
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: !!config.enabled,
|
||||
warningDays: warning,
|
||||
criticalDays: critical,
|
||||
};
|
||||
};
|
||||
|
||||
const updateSnapshotDefaults = (
|
||||
updater:
|
||||
| SnapshotAlertConfig
|
||||
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
|
||||
) => {
|
||||
props.setSnapshotDefaults((prev) => {
|
||||
const next =
|
||||
typeof updater === 'function'
|
||||
? (updater as (prev: SnapshotAlertConfig) => SnapshotAlertConfig)(prev)
|
||||
: { ...prev, ...updater };
|
||||
return sanitizeSnapshotConfig(next);
|
||||
});
|
||||
props.setHasUnsavedChanges(true);
|
||||
};
|
||||
|
||||
const snapshotDefaultsRecord = createMemo(() => {
|
||||
const current = props.snapshotDefaults();
|
||||
return {
|
||||
'warning days': current.warningDays ?? 0,
|
||||
'critical days': current.criticalDays ?? 0,
|
||||
};
|
||||
});
|
||||
|
||||
const snapshotFactoryDefaultsRecord = createMemo(() => {
|
||||
const factory = snapshotFactoryConfig();
|
||||
return {
|
||||
'warning days': factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING,
|
||||
'critical days': factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL,
|
||||
};
|
||||
});
|
||||
|
||||
const snapshotOverridesCount = createMemo(() => {
|
||||
const current = props.snapshotDefaults();
|
||||
const factory = snapshotFactoryConfig();
|
||||
return current.enabled !== factory.enabled ||
|
||||
(current.warningDays ?? DEFAULT_SNAPSHOT_WARNING) !==
|
||||
(factory.warningDays ?? DEFAULT_SNAPSHOT_WARNING) ||
|
||||
(current.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL) !==
|
||||
(factory.criticalDays ?? DEFAULT_SNAPSHOT_CRITICAL)
|
||||
? 1
|
||||
: 0;
|
||||
});
|
||||
|
||||
// Process guests with their overrides and group by node
|
||||
const guestsGroupedByNode = createMemo<Record<string, Resource[]>>((prev = {}) => {
|
||||
// If we're currently editing, return the previous value to avoid re-renders
|
||||
@@ -1082,16 +1161,23 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
},
|
||||
{
|
||||
key: 'storage' as const,
|
||||
label: 'Storage',
|
||||
total: props.storage?.length ?? 0,
|
||||
overrides: countOverrides(storageWithOverrides()),
|
||||
tab: 'proxmox' as const,
|
||||
},
|
||||
{
|
||||
key: 'pbs' as const,
|
||||
label: 'PBS Servers',
|
||||
total: props.pbsInstances?.length ?? 0,
|
||||
overrides: countOverrides(pbsServersWithOverrides()),
|
||||
label: 'Storage',
|
||||
total: props.storage?.length ?? 0,
|
||||
overrides: countOverrides(storageWithOverrides()),
|
||||
tab: 'proxmox' as const,
|
||||
},
|
||||
{
|
||||
key: 'snapshots' as const,
|
||||
label: 'Snapshots',
|
||||
total: 1,
|
||||
overrides: snapshotOverridesCount(),
|
||||
tab: 'proxmox' as const,
|
||||
},
|
||||
{
|
||||
key: 'pbs' as const,
|
||||
label: 'PBS Servers',
|
||||
total: props.pbsInstances?.length ?? 0,
|
||||
overrides: countOverrides(pbsServersWithOverrides()),
|
||||
tab: 'proxmox' as const,
|
||||
},
|
||||
{
|
||||
@@ -1883,6 +1969,72 @@ const dockerContainersGroupedByHost = createMemo<Record<string, Resource[]>>((pr
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
<Show when={hasSection('snapshots')}>
|
||||
<div ref={registerSection('snapshots')} class="scroll-mt-24">
|
||||
<ResourceTable
|
||||
title="Snapshot Age"
|
||||
resources={[]}
|
||||
columns={['Warning Days', 'Critical Days']}
|
||||
activeAlerts={props.activeAlerts}
|
||||
emptyMessage=""
|
||||
onEdit={startEditing}
|
||||
onSaveEdit={saveEdit}
|
||||
onCancelEdit={cancelEdit}
|
||||
onRemoveOverride={removeOverride}
|
||||
showOfflineAlertsColumn={false}
|
||||
editingId={editingId}
|
||||
editingThresholds={editingThresholds}
|
||||
setEditingThresholds={setEditingThresholds}
|
||||
formatMetricValue={formatMetricValue}
|
||||
hasActiveAlert={hasActiveAlert}
|
||||
globalDefaults={snapshotDefaultsRecord()}
|
||||
setGlobalDefaults={(value) => {
|
||||
updateSnapshotDefaults((prev) => {
|
||||
const currentRecord = {
|
||||
'warning days': prev.warningDays ?? 0,
|
||||
'critical days': prev.criticalDays ?? 0,
|
||||
};
|
||||
const nextRecord =
|
||||
typeof value === 'function'
|
||||
? value(currentRecord)
|
||||
: { ...currentRecord, ...value };
|
||||
return {
|
||||
...prev,
|
||||
warningDays:
|
||||
typeof nextRecord['warning days'] === 'number'
|
||||
? nextRecord['warning days']
|
||||
: prev.warningDays,
|
||||
criticalDays:
|
||||
typeof nextRecord['critical days'] === 'number'
|
||||
? nextRecord['critical days']
|
||||
: prev.criticalDays,
|
||||
};
|
||||
});
|
||||
}}
|
||||
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||
globalDisableFlag={() => !props.snapshotDefaults().enabled}
|
||||
onToggleGlobalDisable={() =>
|
||||
updateSnapshotDefaults((prev) => ({
|
||||
...prev,
|
||||
enabled: !prev.enabled,
|
||||
}))
|
||||
}
|
||||
factoryDefaults={snapshotFactoryDefaultsRecord()}
|
||||
onResetDefaults={
|
||||
props.resetSnapshotDefaults
|
||||
? () => {
|
||||
props.resetSnapshotDefaults?.();
|
||||
props.setHasUnsavedChanges(true);
|
||||
}
|
||||
: () =>
|
||||
updateSnapshotDefaults({
|
||||
...snapshotFactoryConfig(),
|
||||
})
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
</Show>
|
||||
|
||||
<Show when={hasSection('storage')}>
|
||||
<div ref={registerSection('storage')} class="scroll-mt-24">
|
||||
<ResourceTable
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
ThresholdsTable,
|
||||
normalizeDockerIgnoredInput,
|
||||
} from '../ThresholdsTable';
|
||||
import type { PMGThresholdDefaults } from '@/types/alerts';
|
||||
import type { PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
|
||||
|
||||
vi.mock('@solidjs/router', () => ({
|
||||
useNavigate: () => vi.fn(),
|
||||
@@ -87,6 +87,10 @@ const baseProps = () => ({
|
||||
factoryNodeDefaults: {},
|
||||
factoryDockerDefaults: {},
|
||||
factoryStorageDefault: 85,
|
||||
snapshotDefaults: () => ({ enabled: false, warningDays: 30, criticalDays: 45 }),
|
||||
setSnapshotDefaults: vi.fn(),
|
||||
snapshotFactoryDefaults: { enabled: false, warningDays: 30, criticalDays: 45 } as SnapshotAlertConfig,
|
||||
resetSnapshotDefaults: vi.fn(),
|
||||
timeThresholds: () => ({ guest: 5, node: 5, storage: 5, pbs: 5 }),
|
||||
metricTimeThresholds: () => ({}),
|
||||
setMetricTimeThresholds: vi.fn(),
|
||||
|
||||
@@ -3,18 +3,18 @@ import type { JSX } from 'solid-js';
|
||||
import { EmailProviderSelect } from '@/components/Alerts/EmailProviderSelect';
|
||||
import { WebhookConfig } from '@/components/Alerts/WebhookConfig';
|
||||
import { ThresholdsTable } from '@/components/Alerts/ThresholdsTable';
|
||||
import type { RawOverrideConfig, PMGThresholdDefaults } from '@/types/alerts';
|
||||
import type { RawOverrideConfig, PMGThresholdDefaults, SnapshotAlertConfig } from '@/types/alerts';
|
||||
import { Card } from '@/components/shared/Card';
|
||||
import { SectionHeader } from '@/components/shared/SectionHeader';
|
||||
import { SettingsPanel } from '@/components/shared/SettingsPanel';
|
||||
import { Toggle } from '@/components/shared/Toggle';
|
||||
import { formField, labelClass, controlClass, formHelpText } from '@/components/shared/Form';
|
||||
import { formField, formControl, formHelpText, labelClass, controlClass } from '@/components/shared/Form';
|
||||
import { useWebSocket } from '@/App';
|
||||
import { showSuccess, showError } from '@/utils/toast';
|
||||
import { showTooltip, hideTooltip } from '@/components/shared/Tooltip';
|
||||
import { AlertsAPI } from '@/api/alerts';
|
||||
import { NotificationsAPI, Webhook } from '@/api/notifications';
|
||||
import type { EmailConfig } from '@/api/notifications';
|
||||
import type { EmailConfig, AppriseConfig } from '@/api/notifications';
|
||||
import type { HysteresisThreshold } from '@/types/alerts';
|
||||
import type { Alert, State, VM, Container, DockerHost, DockerContainer } from '@/types/api';
|
||||
import { useNavigate, useLocation } from '@solidjs/router';
|
||||
@@ -98,6 +98,7 @@ export const tabFromPath = (
|
||||
// Store reference interfaces
|
||||
interface DestinationsRef {
|
||||
emailConfig?: () => EmailConfig;
|
||||
appriseConfig?: () => AppriseConfig;
|
||||
}
|
||||
|
||||
// Override interface for both guests and nodes
|
||||
@@ -151,6 +152,13 @@ interface UIEmailConfig {
|
||||
rateLimit: number;
|
||||
}
|
||||
|
||||
interface UIAppriseConfig {
|
||||
enabled: boolean;
|
||||
targetsText: string;
|
||||
cliPath: string;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
interface QuietHoursConfig {
|
||||
enabled: boolean;
|
||||
start: string;
|
||||
@@ -226,6 +234,22 @@ export const createDefaultGrouping = (): GroupingConfig => ({
|
||||
byGuest: false,
|
||||
});
|
||||
|
||||
const createDefaultAppriseConfig = (): UIAppriseConfig => ({
|
||||
enabled: false,
|
||||
targetsText: '',
|
||||
cliPath: 'apprise',
|
||||
timeoutSeconds: 15,
|
||||
});
|
||||
|
||||
const parseAppriseTargets = (value: string): string[] =>
|
||||
value
|
||||
.split(/\r?\n|,/)
|
||||
.map((entry) => entry.trim())
|
||||
.filter((entry, index, arr) => entry.length > 0 && arr.indexOf(entry) === index);
|
||||
|
||||
const formatAppriseTargets = (targets: string[] | undefined | null): string =>
|
||||
targets && targets.length > 0 ? targets.join('\n') : '';
|
||||
|
||||
export const normalizeMetricDelayMap = (
|
||||
input: Record<string, Record<string, number>> | undefined | null,
|
||||
): Record<string, Record<string, number>> => {
|
||||
@@ -348,11 +372,11 @@ export function Alerts() {
|
||||
>({}); // Store raw config
|
||||
|
||||
// Email configuration state moved to parent to persist across tab changes
|
||||
const [emailConfig, setEmailConfig] = createSignal<UIEmailConfig>({
|
||||
enabled: false,
|
||||
provider: '',
|
||||
server: '', // Fixed: use 'server' not 'smtpHost'
|
||||
port: 587, // Fixed: use 'port' not 'smtpPort'
|
||||
const [emailConfig, setEmailConfig] = createSignal<UIEmailConfig>({
|
||||
enabled: false,
|
||||
provider: '',
|
||||
server: '', // Fixed: use 'server' not 'smtpHost'
|
||||
port: 587, // Fixed: use 'port' not 'smtpPort'
|
||||
username: '',
|
||||
password: '',
|
||||
from: '',
|
||||
@@ -362,8 +386,12 @@ export function Alerts() {
|
||||
replyTo: '',
|
||||
maxRetries: 3,
|
||||
retryDelay: 5,
|
||||
rateLimit: 60,
|
||||
});
|
||||
rateLimit: 60,
|
||||
});
|
||||
|
||||
const [appriseConfig, setAppriseConfig] = createSignal<UIAppriseConfig>(
|
||||
createDefaultAppriseConfig(),
|
||||
);
|
||||
|
||||
// Schedule configuration state moved to parent to persist across tab changes
|
||||
const [scheduleQuietHours, setScheduleQuietHours] =
|
||||
@@ -395,6 +423,16 @@ export function Alerts() {
|
||||
} as EmailConfig;
|
||||
};
|
||||
|
||||
destinationsRef.appriseConfig = () => {
|
||||
const config = appriseConfig();
|
||||
return {
|
||||
enabled: config.enabled,
|
||||
targets: parseAppriseTargets(config.targetsText),
|
||||
cliPath: config.cliPath,
|
||||
timeoutSeconds: config.timeoutSeconds,
|
||||
} as AppriseConfig;
|
||||
};
|
||||
|
||||
// Process raw overrides config when state changes
|
||||
createEffect(() => {
|
||||
// Skip this effect if there are unsaved changes to prevent losing focus
|
||||
@@ -658,6 +696,8 @@ export function Alerts() {
|
||||
rateLimit: 60,
|
||||
});
|
||||
|
||||
setAppriseConfig(createDefaultAppriseConfig());
|
||||
|
||||
try {
|
||||
const config = await AlertsAPI.getConfig();
|
||||
|
||||
@@ -729,6 +769,29 @@ export function Alerts() {
|
||||
setMetricTimeThresholds({});
|
||||
}
|
||||
|
||||
// Load snapshot thresholds
|
||||
if (config.snapshotDefaults) {
|
||||
const enabled = Boolean(config.snapshotDefaults.enabled);
|
||||
const rawWarning = config.snapshotDefaults.warningDays ?? 30;
|
||||
const rawCritical = config.snapshotDefaults.criticalDays ?? 45;
|
||||
const safeCritical = Math.max(0, rawCritical);
|
||||
const normalizedWarning = Math.max(0, rawWarning);
|
||||
const warningDays =
|
||||
safeCritical > 0 && normalizedWarning > safeCritical ? safeCritical : normalizedWarning;
|
||||
const criticalDays = Math.max(safeCritical, warningDays);
|
||||
setSnapshotDefaults({
|
||||
enabled,
|
||||
warningDays,
|
||||
criticalDays,
|
||||
});
|
||||
} else {
|
||||
setSnapshotDefaults({
|
||||
enabled: false,
|
||||
warningDays: 30,
|
||||
criticalDays: 45,
|
||||
});
|
||||
}
|
||||
|
||||
// Load PMG thresholds
|
||||
if (config.pmgDefaults) {
|
||||
setPMGThresholds({
|
||||
@@ -867,6 +930,21 @@ export function Alerts() {
|
||||
console.error('Failed to load email configuration:', emailErr);
|
||||
}
|
||||
|
||||
try {
|
||||
const appriseData = await NotificationsAPI.getAppriseConfig();
|
||||
setAppriseConfig({
|
||||
enabled: appriseData.enabled ?? false,
|
||||
targetsText: formatAppriseTargets(appriseData.targets),
|
||||
cliPath: appriseData.cliPath || 'apprise',
|
||||
timeoutSeconds:
|
||||
typeof appriseData.timeoutSeconds === 'number' && appriseData.timeoutSeconds > 0
|
||||
? appriseData.timeoutSeconds
|
||||
: 15,
|
||||
});
|
||||
} catch (appriseErr) {
|
||||
console.error('Failed to load Apprise configuration:', appriseErr);
|
||||
}
|
||||
|
||||
if (options.notify) {
|
||||
showSuccess('Changes discarded');
|
||||
}
|
||||
@@ -911,6 +989,22 @@ export function Alerts() {
|
||||
.catch((err) => {
|
||||
console.error('Failed to reload email configuration:', err);
|
||||
});
|
||||
|
||||
NotificationsAPI.getAppriseConfig()
|
||||
.then((appriseData) => {
|
||||
setAppriseConfig({
|
||||
enabled: appriseData.enabled ?? false,
|
||||
targetsText: formatAppriseTargets(appriseData.targets),
|
||||
cliPath: appriseData.cliPath || 'apprise',
|
||||
timeoutSeconds:
|
||||
typeof appriseData.timeoutSeconds === 'number' && appriseData.timeoutSeconds > 0
|
||||
? appriseData.timeoutSeconds
|
||||
: 15,
|
||||
});
|
||||
})
|
||||
.catch((err) => {
|
||||
console.error('Failed to reload Apprise configuration:', err);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -959,6 +1053,11 @@ export function Alerts() {
|
||||
};
|
||||
|
||||
const FACTORY_STORAGE_DEFAULT = 85;
|
||||
const FACTORY_SNAPSHOT_DEFAULTS: SnapshotAlertConfig = {
|
||||
enabled: false,
|
||||
warningDays: 30,
|
||||
criticalDays: 45,
|
||||
};
|
||||
|
||||
// Threshold states - using trigger values for display
|
||||
const [guestDefaults, setGuestDefaults] = createSignal<Record<string, number | undefined>>({ ...FACTORY_GUEST_DEFAULTS });
|
||||
@@ -997,15 +1096,22 @@ export function Alerts() {
|
||||
setStorageDefault(FACTORY_STORAGE_DEFAULT);
|
||||
setHasUnsavedChanges(true);
|
||||
};
|
||||
const [timeThreshold, setTimeThreshold] = createSignal(DEFAULT_DELAY_SECONDS); // Legacy
|
||||
const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
guest: DEFAULT_DELAY_SECONDS,
|
||||
node: DEFAULT_DELAY_SECONDS,
|
||||
storage: DEFAULT_DELAY_SECONDS,
|
||||
pbs: DEFAULT_DELAY_SECONDS,
|
||||
});
|
||||
const resetSnapshotDefaults = () => {
|
||||
setSnapshotDefaults({ ...FACTORY_SNAPSHOT_DEFAULTS });
|
||||
setHasUnsavedChanges(true);
|
||||
};
|
||||
const [timeThreshold, setTimeThreshold] = createSignal(DEFAULT_DELAY_SECONDS); // Legacy
|
||||
const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
guest: DEFAULT_DELAY_SECONDS,
|
||||
node: DEFAULT_DELAY_SECONDS,
|
||||
storage: DEFAULT_DELAY_SECONDS,
|
||||
pbs: DEFAULT_DELAY_SECONDS,
|
||||
});
|
||||
const [metricTimeThresholds, setMetricTimeThresholds] =
|
||||
createSignal<Record<string, Record<string, number>>>({});
|
||||
const [snapshotDefaults, setSnapshotDefaults] = createSignal<SnapshotAlertConfig>({
|
||||
...FACTORY_SNAPSHOT_DEFAULTS,
|
||||
});
|
||||
|
||||
const [pmgThresholds, setPMGThresholds] = createSignal({
|
||||
queueTotalWarning: 500,
|
||||
@@ -1117,6 +1223,14 @@ const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
};
|
||||
};
|
||||
|
||||
const snapshotConfig = snapshotDefaults();
|
||||
const normalizedWarningDays = Math.max(0, snapshotConfig.warningDays ?? 0);
|
||||
const normalizedCriticalDays = Math.max(0, snapshotConfig.criticalDays ?? 0);
|
||||
const finalCriticalDays =
|
||||
normalizedCriticalDays > 0
|
||||
? Math.max(normalizedCriticalDays, normalizedWarningDays)
|
||||
: normalizedWarningDays;
|
||||
|
||||
const alertConfig = {
|
||||
enabled: true,
|
||||
// Global disable flags per resource type
|
||||
@@ -1168,6 +1282,11 @@ const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
timeThreshold: timeThreshold() || 0, // Legacy
|
||||
timeThresholds: timeThresholds(),
|
||||
metricTimeThresholds: normalizeMetricDelayMap(metricTimeThresholds()),
|
||||
snapshotDefaults: {
|
||||
enabled: snapshotConfig.enabled,
|
||||
warningDays: normalizedWarningDays,
|
||||
criticalDays: finalCriticalDays,
|
||||
},
|
||||
pmgDefaults: pmgThresholds(),
|
||||
// Use rawOverridesConfig which is already properly formatted with disabled flags
|
||||
overrides: rawOverridesConfig(),
|
||||
@@ -1217,6 +1336,20 @@ const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
await NotificationsAPI.updateEmailConfig(emailData);
|
||||
}
|
||||
|
||||
if (destinationsRef.appriseConfig) {
|
||||
const appriseData = destinationsRef.appriseConfig();
|
||||
const updatedApprise = await NotificationsAPI.updateAppriseConfig(appriseData);
|
||||
setAppriseConfig({
|
||||
enabled: updatedApprise.enabled ?? false,
|
||||
targetsText: formatAppriseTargets(updatedApprise.targets),
|
||||
cliPath: updatedApprise.cliPath || 'apprise',
|
||||
timeoutSeconds:
|
||||
typeof updatedApprise.timeoutSeconds === 'number' && updatedApprise.timeoutSeconds > 0
|
||||
? updatedApprise.timeoutSeconds
|
||||
: 15,
|
||||
});
|
||||
}
|
||||
|
||||
setHasUnsavedChanges(false);
|
||||
showSuccess('Configuration saved successfully!');
|
||||
} catch (err) {
|
||||
@@ -1357,13 +1490,17 @@ const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
resetDockerDefaults={resetDockerDefaults}
|
||||
resetDockerIgnoredPrefixes={resetDockerIgnoredPrefixes}
|
||||
resetStorageDefault={resetStorageDefault}
|
||||
resetSnapshotDefaults={resetSnapshotDefaults}
|
||||
factoryGuestDefaults={FACTORY_GUEST_DEFAULTS}
|
||||
factoryNodeDefaults={FACTORY_NODE_DEFAULTS}
|
||||
factoryDockerDefaults={FACTORY_DOCKER_DEFAULTS}
|
||||
factoryStorageDefault={FACTORY_STORAGE_DEFAULT}
|
||||
snapshotFactoryDefaults={FACTORY_SNAPSHOT_DEFAULTS}
|
||||
timeThresholds={timeThresholds}
|
||||
metricTimeThresholds={metricTimeThresholds}
|
||||
setMetricTimeThresholds={setMetricTimeThresholds}
|
||||
snapshotDefaults={snapshotDefaults}
|
||||
setSnapshotDefaults={setSnapshotDefaults}
|
||||
pmgThresholds={pmgThresholds}
|
||||
setPMGThresholds={setPMGThresholds}
|
||||
activeAlerts={activeAlerts}
|
||||
@@ -1404,6 +1541,8 @@ const [timeThresholds, setTimeThresholds] = createSignal({
|
||||
setHasUnsavedChanges={setHasUnsavedChanges}
|
||||
emailConfig={emailConfig}
|
||||
setEmailConfig={setEmailConfig}
|
||||
appriseConfig={appriseConfig}
|
||||
setAppriseConfig={setAppriseConfig}
|
||||
/>
|
||||
</Show>
|
||||
|
||||
@@ -1881,6 +2020,14 @@ interface ThresholdsTabProps {
|
||||
| Record<string, Record<string, number>>
|
||||
| ((prev: Record<string, Record<string, number>>) => Record<string, Record<string, number>>),
|
||||
) => void;
|
||||
snapshotDefaults: () => SnapshotAlertConfig;
|
||||
setSnapshotDefaults: (
|
||||
value:
|
||||
| SnapshotAlertConfig
|
||||
| ((prev: SnapshotAlertConfig) => SnapshotAlertConfig),
|
||||
) => void;
|
||||
snapshotFactoryDefaults: SnapshotAlertConfig;
|
||||
resetSnapshotDefaults: () => void;
|
||||
setOverrides: (value: Override[]) => void;
|
||||
setRawOverridesConfig: (value: Record<string, RawOverrideConfig>) => void;
|
||||
activeAlerts: Record<string, Alert>;
|
||||
@@ -1926,7 +2073,6 @@ interface ThresholdsTabProps {
|
||||
}
|
||||
|
||||
function ThresholdsTab(props: ThresholdsTabProps) {
|
||||
// Use the new table component for a cleaner, more information-dense layout
|
||||
return (
|
||||
<ThresholdsTable
|
||||
overrides={props.overrides}
|
||||
@@ -1958,6 +2104,10 @@ function ThresholdsTab(props: ThresholdsTabProps) {
|
||||
timeThresholds={props.timeThresholds}
|
||||
metricTimeThresholds={props.metricTimeThresholds}
|
||||
setMetricTimeThresholds={props.setMetricTimeThresholds}
|
||||
snapshotDefaults={props.snapshotDefaults}
|
||||
setSnapshotDefaults={props.setSnapshotDefaults}
|
||||
snapshotFactoryDefaults={props.snapshotFactoryDefaults}
|
||||
resetSnapshotDefaults={props.resetSnapshotDefaults}
|
||||
setHasUnsavedChanges={props.setHasUnsavedChanges}
|
||||
activeAlerts={props.activeAlerts}
|
||||
removeAlerts={props.removeAlerts}
|
||||
@@ -2005,13 +2155,18 @@ interface DestinationsTabProps {
|
||||
setHasUnsavedChanges: (value: boolean) => void;
|
||||
emailConfig: () => UIEmailConfig;
|
||||
setEmailConfig: (config: UIEmailConfig) => void;
|
||||
appriseConfig: () => UIAppriseConfig;
|
||||
setAppriseConfig: (config: UIAppriseConfig) => void;
|
||||
}
|
||||
|
||||
function DestinationsTab(props: DestinationsTabProps) {
|
||||
const [webhooks, setWebhooks] = createSignal<Webhook[]>([]);
|
||||
const [testingEmail, setTestingEmail] = createSignal(false);
|
||||
const [testingWebhook, setTestingWebhook] = createSignal<string | null>(null);
|
||||
|
||||
const appriseState = () => props.appriseConfig();
|
||||
const updateApprise = (partial: Partial<UIAppriseConfig>) => {
|
||||
props.setAppriseConfig({ ...props.appriseConfig(), ...partial });
|
||||
};
|
||||
// Load webhooks on mount (email config is now loaded in parent)
|
||||
onMount(async () => {
|
||||
try {
|
||||
@@ -2105,6 +2260,78 @@ function DestinationsTab(props: DestinationsTabProps) {
|
||||
</div>
|
||||
</SettingsPanel>
|
||||
|
||||
<SettingsPanel
|
||||
title="Apprise notifications"
|
||||
description="Relay grouped alerts through the Apprise CLI."
|
||||
action={
|
||||
<Toggle
|
||||
checked={appriseState().enabled}
|
||||
onChange={(e) => {
|
||||
updateApprise({ enabled: e.currentTarget.checked });
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
containerClass="sm:self-start"
|
||||
label={
|
||||
<span class="text-xs font-medium text-gray-600 dark:text-gray-400">
|
||||
{appriseState().enabled ? 'Enabled' : 'Disabled'}
|
||||
</span>
|
||||
}
|
||||
/>
|
||||
}
|
||||
class="min-w-0"
|
||||
bodyClass="space-y-4"
|
||||
>
|
||||
<div class="space-y-4">
|
||||
<div class={formField}>
|
||||
<label class={labelClass('text-xs uppercase tracking-[0.08em]')}>Delivery targets</label>
|
||||
<textarea
|
||||
rows={4}
|
||||
class={`${formControl} font-mono min-h-[120px]`}
|
||||
value={appriseState().targetsText}
|
||||
placeholder={`discord://token\nmailto://alerts@example.com`}
|
||||
onInput={(e) => {
|
||||
updateApprise({ targetsText: e.currentTarget.value });
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
/>
|
||||
<p class={formHelpText}>Enter one Apprise URL per line. Commas are also supported.</p>
|
||||
</div>
|
||||
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2">
|
||||
<div class={formField}>
|
||||
<label class={labelClass('text-xs uppercase tracking-[0.08em]')}>CLI path</label>
|
||||
<input
|
||||
type="text"
|
||||
value={appriseState().cliPath}
|
||||
class={formControl}
|
||||
placeholder="apprise"
|
||||
onInput={(e) => {
|
||||
updateApprise({ cliPath: e.currentTarget.value });
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
/>
|
||||
<p class={formHelpText}>Leave blank to use the default `apprise` executable.</p>
|
||||
</div>
|
||||
<div class={formField}>
|
||||
<label class={labelClass('text-xs uppercase tracking-[0.08em]')}>Timeout (seconds)</label>
|
||||
<input
|
||||
type="number"
|
||||
min="5"
|
||||
max="120"
|
||||
value={appriseState().timeoutSeconds}
|
||||
class={formControl}
|
||||
onInput={(e) => {
|
||||
const raw = e.currentTarget.valueAsNumber;
|
||||
const safe = Number.isNaN(raw) ? 15 : Math.min(120, Math.max(5, Math.trunc(raw)));
|
||||
updateApprise({ timeoutSeconds: safe });
|
||||
props.setHasUnsavedChanges(true);
|
||||
}}
|
||||
/>
|
||||
<p class={formHelpText}>Maximum time to wait for the Apprise CLI to complete.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</SettingsPanel>
|
||||
|
||||
<SettingsPanel
|
||||
title="Webhooks"
|
||||
description="Push alerts to chat apps or automation systems."
|
||||
|
||||
@@ -85,6 +85,12 @@ export interface PMGThresholdDefaults {
|
||||
quarantineGrowthCritMin?: number;
|
||||
}
|
||||
|
||||
export interface SnapshotAlertConfig {
|
||||
enabled: boolean;
|
||||
warningDays: number;
|
||||
criticalDays: number;
|
||||
}
|
||||
|
||||
export interface AlertConfig {
|
||||
enabled: boolean;
|
||||
guestDefaults: AlertThresholds;
|
||||
@@ -93,6 +99,7 @@ export interface AlertConfig {
|
||||
dockerDefaults?: DockerThresholdConfig;
|
||||
dockerIgnoredContainerPrefixes?: string[];
|
||||
pmgDefaults?: PMGThresholdDefaults;
|
||||
snapshotDefaults?: SnapshotAlertConfig;
|
||||
customRules?: CustomAlertRule[];
|
||||
overrides: Record<string, RawOverrideConfig>; // key: resource ID
|
||||
minimumDelta?: number;
|
||||
|
||||
@@ -283,6 +283,13 @@ type PMGThresholdConfig struct {
|
||||
QuarantineGrowthCritMin int `json:"quarantineGrowthCritMin"` // Minimum message growth for critical (default: 500)
|
||||
}
|
||||
|
||||
// SnapshotAlertConfig represents snapshot age alert configuration
|
||||
type SnapshotAlertConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
WarningDays int `json:"warningDays"`
|
||||
CriticalDays int `json:"criticalDays"`
|
||||
}
|
||||
|
||||
// AlertConfig represents the complete alert configuration
|
||||
type AlertConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
@@ -292,6 +299,7 @@ type AlertConfig struct {
|
||||
DockerDefaults DockerThresholdConfig `json:"dockerDefaults"`
|
||||
DockerIgnoredContainerPrefixes []string `json:"dockerIgnoredContainerPrefixes,omitempty"`
|
||||
PMGDefaults PMGThresholdConfig `json:"pmgDefaults"`
|
||||
SnapshotDefaults SnapshotAlertConfig `json:"snapshotDefaults"`
|
||||
Overrides map[string]ThresholdConfig `json:"overrides"` // keyed by resource ID
|
||||
CustomRules []CustomAlertRule `json:"customRules,omitempty"`
|
||||
Schedule ScheduleConfig `json:"schedule"`
|
||||
@@ -473,6 +481,11 @@ func NewManager() *Manager {
|
||||
QuarantineGrowthCritPct: 50, // Critical if growth ≥50%
|
||||
QuarantineGrowthCritMin: 500, // AND ≥500 messages
|
||||
},
|
||||
SnapshotDefaults: SnapshotAlertConfig{
|
||||
Enabled: false,
|
||||
WarningDays: 30,
|
||||
CriticalDays: 45,
|
||||
},
|
||||
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
MinimumDelta: 2.0, // 2% minimum change
|
||||
SuppressionWindow: 5, // 5 minutes
|
||||
@@ -680,6 +693,16 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
||||
config.PMGDefaults.QuarantineGrowthCritMin = 500
|
||||
}
|
||||
|
||||
if config.SnapshotDefaults.WarningDays < 0 {
|
||||
config.SnapshotDefaults.WarningDays = 0
|
||||
}
|
||||
if config.SnapshotDefaults.CriticalDays < 0 {
|
||||
config.SnapshotDefaults.CriticalDays = 0
|
||||
}
|
||||
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
||||
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
||||
}
|
||||
|
||||
// Ensure minimums for other important fields
|
||||
if config.MinimumDelta <= 0 {
|
||||
config.MinimumDelta = 2.0
|
||||
@@ -738,6 +761,10 @@ func (m *Manager) UpdateConfig(config AlertConfig) {
|
||||
m.config.Overrides[id] = override
|
||||
}
|
||||
|
||||
if !m.config.SnapshotDefaults.Enabled {
|
||||
m.clearSnapshotAlertsForInstanceLocked("")
|
||||
}
|
||||
|
||||
m.applyGlobalOfflineSettingsLocked()
|
||||
|
||||
log.Info().
|
||||
@@ -2729,6 +2756,216 @@ func (m *Manager) CheckStorage(storage models.Storage) {
|
||||
}
|
||||
}
|
||||
|
||||
func buildSnapshotGuestKey(instance, node string, vmid int) string {
|
||||
instance = strings.TrimSpace(instance)
|
||||
node = strings.TrimSpace(node)
|
||||
if instance == "" {
|
||||
instance = node
|
||||
}
|
||||
if instance == node {
|
||||
return fmt.Sprintf("%s-%d", node, vmid)
|
||||
}
|
||||
return fmt.Sprintf("%s-%s-%d", instance, node, vmid)
|
||||
}
|
||||
|
||||
// CheckSnapshotsForInstance evaluates guest snapshots for age-based alerts.
|
||||
func (m *Manager) CheckSnapshotsForInstance(instanceName string, snapshots []models.GuestSnapshot, guestNames map[string]string) {
|
||||
m.mu.RLock()
|
||||
enabled := m.config.Enabled
|
||||
snapshotCfg := m.config.SnapshotDefaults
|
||||
m.mu.RUnlock()
|
||||
|
||||
if !enabled {
|
||||
return
|
||||
}
|
||||
|
||||
if !snapshotCfg.Enabled {
|
||||
m.clearSnapshotAlertsForInstance(instanceName)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
validAlerts := make(map[string]struct{})
|
||||
|
||||
for _, snapshot := range snapshots {
|
||||
if instanceName != "" && snapshot.Instance != "" && snapshot.Instance != instanceName {
|
||||
continue
|
||||
}
|
||||
if snapshot.Time.IsZero() {
|
||||
continue
|
||||
}
|
||||
|
||||
ageHours := now.Sub(snapshot.Time).Hours()
|
||||
if ageHours < 0 {
|
||||
continue
|
||||
}
|
||||
ageDays := ageHours / 24
|
||||
|
||||
var level AlertLevel
|
||||
var threshold int
|
||||
|
||||
if snapshotCfg.CriticalDays > 0 && ageDays >= float64(snapshotCfg.CriticalDays) {
|
||||
level = AlertLevelCritical
|
||||
threshold = snapshotCfg.CriticalDays
|
||||
} else if snapshotCfg.WarningDays > 0 && ageDays >= float64(snapshotCfg.WarningDays) {
|
||||
level = AlertLevelWarning
|
||||
threshold = snapshotCfg.WarningDays
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
alertID := fmt.Sprintf("snapshot-age-%s", snapshot.ID)
|
||||
validAlerts[alertID] = struct{}{}
|
||||
|
||||
guestKey := buildSnapshotGuestKey(snapshot.Instance, snapshot.Node, snapshot.VMID)
|
||||
guestName := strings.TrimSpace(guestNames[guestKey])
|
||||
|
||||
guestType := "VM"
|
||||
if strings.EqualFold(snapshot.Type, "lxc") {
|
||||
guestType = "Container"
|
||||
}
|
||||
|
||||
if guestName == "" {
|
||||
switch guestType {
|
||||
case "Container":
|
||||
guestName = fmt.Sprintf("CT %d", snapshot.VMID)
|
||||
default:
|
||||
guestName = fmt.Sprintf("VM %d", snapshot.VMID)
|
||||
}
|
||||
}
|
||||
|
||||
snapshotName := strings.TrimSpace(snapshot.Name)
|
||||
if snapshotName == "" {
|
||||
snapshotName = "(unnamed)"
|
||||
}
|
||||
|
||||
ageDaysRounded := math.Round(ageDays*10) / 10
|
||||
message := fmt.Sprintf(
|
||||
"%s snapshot '%s' for %s is %.1f days old on %s (threshold: %d days)",
|
||||
guestType,
|
||||
snapshotName,
|
||||
guestName,
|
||||
ageDaysRounded,
|
||||
snapshot.Node,
|
||||
threshold,
|
||||
)
|
||||
|
||||
thresholdTime := snapshot.Time.Add(time.Duration(threshold) * 24 * time.Hour)
|
||||
if thresholdTime.After(now) {
|
||||
thresholdTime = now
|
||||
}
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
"snapshotName": snapshot.Name,
|
||||
"snapshotCreatedAt": snapshot.Time,
|
||||
"snapshotAgeDays": ageDays,
|
||||
"snapshotAgeHours": ageHours,
|
||||
"guestName": guestName,
|
||||
"guestType": guestType,
|
||||
"guestInstance": snapshot.Instance,
|
||||
"guestNode": snapshot.Node,
|
||||
"guestVmid": snapshot.VMID,
|
||||
"thresholdDays": threshold,
|
||||
}
|
||||
|
||||
resourceName := fmt.Sprintf("%s snapshot '%s'", guestName, snapshotName)
|
||||
|
||||
m.mu.Lock()
|
||||
if existing, exists := m.activeAlerts[alertID]; exists {
|
||||
existing.LastSeen = now
|
||||
existing.Level = level
|
||||
existing.Value = ageDays
|
||||
existing.Threshold = float64(threshold)
|
||||
existing.Message = message
|
||||
existing.ResourceName = resourceName
|
||||
if existing.Metadata == nil {
|
||||
existing.Metadata = make(map[string]interface{})
|
||||
}
|
||||
for k, v := range metadata {
|
||||
existing.Metadata[k] = v
|
||||
}
|
||||
m.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
|
||||
alert := &Alert{
|
||||
ID: alertID,
|
||||
Type: "snapshot-age",
|
||||
Level: level,
|
||||
ResourceID: snapshot.ID,
|
||||
ResourceName: resourceName,
|
||||
Node: snapshot.Node,
|
||||
Instance: snapshot.Instance,
|
||||
Message: message,
|
||||
Value: ageDays,
|
||||
Threshold: float64(threshold),
|
||||
StartTime: thresholdTime,
|
||||
LastSeen: now,
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
m.preserveAlertState(alertID, alert)
|
||||
|
||||
m.activeAlerts[alertID] = alert
|
||||
m.recentAlerts[alertID] = alert
|
||||
m.historyManager.AddAlert(*alert)
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Error().Interface("panic", r).Msg("Panic in SaveActiveAlerts goroutine (snapshot)")
|
||||
}
|
||||
}()
|
||||
if err := m.SaveActiveAlerts(); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to save active alerts after snapshot alert creation")
|
||||
}
|
||||
}()
|
||||
|
||||
if !m.checkRateLimit(alertID) {
|
||||
m.mu.Unlock()
|
||||
log.Debug().
|
||||
Str("alertID", alertID).
|
||||
Str("guest", guestName).
|
||||
Msg("Snapshot alert suppressed due to rate limit")
|
||||
continue
|
||||
}
|
||||
|
||||
if m.onAlert != nil {
|
||||
nowCopy := now
|
||||
alert.LastNotified = &nowCopy
|
||||
if m.dispatchAlert(alert, true) {
|
||||
log.Info().
|
||||
Str("alertID", alertID).
|
||||
Str("guest", guestName).
|
||||
Msg("Snapshot age alert dispatched")
|
||||
} else {
|
||||
alert.LastNotified = nil
|
||||
}
|
||||
} else {
|
||||
log.Warn().
|
||||
Str("alertID", alertID).
|
||||
Msg("Snapshot age alert created but no onAlert callback set")
|
||||
}
|
||||
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
for alertID, alert := range m.activeAlerts {
|
||||
if alert == nil || alert.Type != "snapshot-age" {
|
||||
continue
|
||||
}
|
||||
if instanceName != "" && alert.Instance != instanceName {
|
||||
continue
|
||||
}
|
||||
if _, ok := validAlerts[alertID]; ok {
|
||||
continue
|
||||
}
|
||||
m.clearAlertNoLock(alertID)
|
||||
}
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
// checkZFSPoolHealth checks ZFS pool for errors and degraded state
|
||||
func (m *Manager) checkZFSPoolHealth(storage models.Storage) {
|
||||
pool := storage.ZFSPool
|
||||
@@ -6208,3 +6445,21 @@ func (m *Manager) clearAlertNoLock(alertID string) {
|
||||
Str("alertID", alertID).
|
||||
Msg("Alert cleared")
|
||||
}
|
||||
|
||||
func (m *Manager) clearSnapshotAlertsForInstance(instance string) {
|
||||
m.mu.Lock()
|
||||
m.clearSnapshotAlertsForInstanceLocked(instance)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func (m *Manager) clearSnapshotAlertsForInstanceLocked(instance string) {
|
||||
for alertID, alert := range m.activeAlerts {
|
||||
if alert == nil || alert.Type != "snapshot-age" {
|
||||
continue
|
||||
}
|
||||
if instance != "" && alert.Instance != instance {
|
||||
continue
|
||||
}
|
||||
m.clearAlertNoLock(alertID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ func TestCheckGuestSkipsAlertsWhenMetricDisabled(t *testing.T) {
|
||||
GuestDefaults: ThresholdConfig{
|
||||
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
},
|
||||
TimeThreshold: 0,
|
||||
TimeThreshold: 0,
|
||||
TimeThresholds: map[string]int{},
|
||||
NodeDefaults: ThresholdConfig{
|
||||
CPU: &HysteresisThreshold{Trigger: 80, Clear: 75},
|
||||
@@ -212,6 +212,67 @@ func TestHandleDockerHostRemovedClearsAlertsAndTracking(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckSnapshotsForInstanceCreatesAndClearsAlerts(t *testing.T) {
|
||||
m := NewManager()
|
||||
m.ClearActiveAlerts()
|
||||
|
||||
cfg := AlertConfig{
|
||||
Enabled: true,
|
||||
StorageDefault: HysteresisThreshold{Trigger: 85, Clear: 80},
|
||||
SnapshotDefaults: SnapshotAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 7,
|
||||
CriticalDays: 14,
|
||||
},
|
||||
Overrides: make(map[string]ThresholdConfig),
|
||||
}
|
||||
m.UpdateConfig(cfg)
|
||||
m.mu.Lock()
|
||||
m.config.TimeThreshold = 0
|
||||
m.config.TimeThresholds = map[string]int{}
|
||||
m.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
snapshots := []models.GuestSnapshot{
|
||||
{
|
||||
ID: "inst-node-100-weekly",
|
||||
Name: "weekly",
|
||||
Node: "node",
|
||||
Instance: "inst",
|
||||
Type: "qemu",
|
||||
VMID: 100,
|
||||
Time: now.Add(-15 * 24 * time.Hour),
|
||||
},
|
||||
}
|
||||
guestNames := map[string]string{
|
||||
"inst-node-100": "app-server",
|
||||
}
|
||||
|
||||
m.CheckSnapshotsForInstance("inst", snapshots, guestNames)
|
||||
|
||||
m.mu.RLock()
|
||||
alert, exists := m.activeAlerts["snapshot-age-inst-node-100-weekly"]
|
||||
m.mu.RUnlock()
|
||||
if !exists {
|
||||
t.Fatalf("expected snapshot age alert to be created")
|
||||
}
|
||||
if alert.Level != AlertLevelCritical {
|
||||
t.Fatalf("expected critical level for old snapshot, got %s", alert.Level)
|
||||
}
|
||||
if alert.ResourceName != "app-server snapshot 'weekly'" {
|
||||
t.Fatalf("unexpected resource name: %s", alert.ResourceName)
|
||||
}
|
||||
|
||||
m.CheckSnapshotsForInstance("inst", nil, guestNames)
|
||||
|
||||
m.mu.RLock()
|
||||
_, exists = m.activeAlerts["snapshot-age-inst-node-100-weekly"]
|
||||
m.mu.RUnlock()
|
||||
if exists {
|
||||
t.Fatalf("expected snapshot alert to be cleared when snapshot missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDockerHostIgnoresContainersByPrefix(t *testing.T) {
|
||||
m := NewManager()
|
||||
|
||||
|
||||
@@ -87,6 +87,51 @@ func (h *NotificationHandlers) UpdateEmailConfig(w http.ResponseWriter, r *http.
|
||||
json.NewEncoder(w).Encode(map[string]string{"status": "success"})
|
||||
}
|
||||
|
||||
// GetAppriseConfig returns the current Apprise configuration.
|
||||
func (h *NotificationHandlers) GetAppriseConfig(w http.ResponseWriter, r *http.Request) {
|
||||
config := h.monitor.GetNotificationManager().GetAppriseConfig()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(config); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateAppriseConfig updates the Apprise configuration.
|
||||
func (h *NotificationHandlers) UpdateAppriseConfig(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var config notifications.AppriseConfig
|
||||
if err := json.Unmarshal(body, &config); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Bool("enabled", config.Enabled).
|
||||
Int("targetCount", len(config.Targets)).
|
||||
Str("cliPath", config.CLIPath).
|
||||
Int("timeoutSeconds", config.TimeoutSeconds).
|
||||
Msg("Parsed Apprise configuration update")
|
||||
|
||||
h.monitor.GetNotificationManager().SetAppriseConfig(config)
|
||||
|
||||
if err := h.monitor.GetConfigPersistence().SaveAppriseConfig(config); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to save Apprise configuration")
|
||||
}
|
||||
|
||||
normalized := h.monitor.GetNotificationManager().GetAppriseConfig()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(normalized); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to encode Apprise configuration response")
|
||||
}
|
||||
}
|
||||
|
||||
// GetWebhooks returns all webhook configurations
|
||||
func (h *NotificationHandlers) GetWebhooks(w http.ResponseWriter, r *http.Request) {
|
||||
webhooks := h.monitor.GetNotificationManager().GetWebhooks()
|
||||
@@ -479,6 +524,10 @@ func (h *NotificationHandlers) HandleNotifications(w http.ResponseWriter, r *htt
|
||||
h.GetEmailConfig(w, r)
|
||||
case path == "/email" && r.Method == http.MethodPut:
|
||||
h.UpdateEmailConfig(w, r)
|
||||
case path == "/apprise" && r.Method == http.MethodGet:
|
||||
h.GetAppriseConfig(w, r)
|
||||
case path == "/apprise" && r.Method == http.MethodPut:
|
||||
h.UpdateAppriseConfig(w, r)
|
||||
case path == "/webhooks" && r.Method == http.MethodGet:
|
||||
h.GetWebhooks(w, r)
|
||||
case path == "/webhooks" && r.Method == http.MethodPost:
|
||||
|
||||
@@ -25,6 +25,7 @@ type ExportData struct {
|
||||
Alerts alerts.AlertConfig `json:"alerts"`
|
||||
Email notifications.EmailConfig `json:"email"`
|
||||
Webhooks []notifications.WebhookConfig `json:"webhooks"`
|
||||
Apprise notifications.AppriseConfig `json:"apprise"`
|
||||
System SystemSettings `json:"system"`
|
||||
GuestMetadata map[string]*GuestMetadata `json:"guestMetadata,omitempty"`
|
||||
OIDC *OIDCConfig `json:"oidc,omitempty"`
|
||||
@@ -55,6 +56,11 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
|
||||
return "", fmt.Errorf("failed to load email config: %w", err)
|
||||
}
|
||||
|
||||
appriseConfig, err := c.LoadAppriseConfig()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to load Apprise config: %w", err)
|
||||
}
|
||||
|
||||
webhooks, err := c.LoadWebhooks()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to load webhooks: %w", err)
|
||||
@@ -90,6 +96,7 @@ func (c *ConfigPersistence) ExportConfig(passphrase string) (string, error) {
|
||||
Alerts: *alertConfig,
|
||||
Email: *emailConfig,
|
||||
Webhooks: webhooks,
|
||||
Apprise: *appriseConfig,
|
||||
System: *systemSettings,
|
||||
GuestMetadata: guestMetadata,
|
||||
OIDC: oidcConfig,
|
||||
@@ -153,6 +160,10 @@ func (c *ConfigPersistence) ImportConfig(encryptedData string, passphrase string
|
||||
return fmt.Errorf("failed to import email config: %w", err)
|
||||
}
|
||||
|
||||
if err := c.SaveAppriseConfig(exportData.Apprise); err != nil {
|
||||
return fmt.Errorf("failed to import Apprise config: %w", err)
|
||||
}
|
||||
|
||||
if err := c.SaveWebhooks(exportData.Webhooks); err != nil {
|
||||
return fmt.Errorf("failed to import webhooks: %w", err)
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ type ConfigPersistence struct {
|
||||
alertFile string
|
||||
emailFile string
|
||||
webhookFile string
|
||||
appriseFile string
|
||||
nodesFile string
|
||||
systemFile string
|
||||
oidcFile string
|
||||
@@ -49,6 +50,7 @@ func NewConfigPersistence(configDir string) *ConfigPersistence {
|
||||
alertFile: filepath.Join(configDir, "alerts.json"),
|
||||
emailFile: filepath.Join(configDir, "email.enc"),
|
||||
webhookFile: filepath.Join(configDir, "webhooks.enc"),
|
||||
appriseFile: filepath.Join(configDir, "apprise.enc"),
|
||||
nodesFile: filepath.Join(configDir, "nodes.enc"),
|
||||
systemFile: filepath.Join(configDir, "system.json"),
|
||||
oidcFile: filepath.Join(configDir, "oidc.enc"),
|
||||
@@ -183,6 +185,15 @@ func (c *ConfigPersistence) SaveAlertConfig(config alerts.AlertConfig) error {
|
||||
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
|
||||
config.TimeThresholds["all"] = config.TimeThreshold
|
||||
}
|
||||
if config.SnapshotDefaults.WarningDays < 0 {
|
||||
config.SnapshotDefaults.WarningDays = 0
|
||||
}
|
||||
if config.SnapshotDefaults.CriticalDays < 0 {
|
||||
config.SnapshotDefaults.CriticalDays = 0
|
||||
}
|
||||
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
||||
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
||||
}
|
||||
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
|
||||
|
||||
data, err := json.MarshalIndent(config, "", " ")
|
||||
@@ -235,7 +246,12 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
|
||||
MinimumDelta: 2.0,
|
||||
SuppressionWindow: 5,
|
||||
HysteresisMargin: 5.0,
|
||||
Overrides: make(map[string]alerts.ThresholdConfig),
|
||||
SnapshotDefaults: alerts.SnapshotAlertConfig{
|
||||
Enabled: false,
|
||||
WarningDays: 30,
|
||||
CriticalDays: 45,
|
||||
},
|
||||
Overrides: make(map[string]alerts.ThresholdConfig),
|
||||
}, nil
|
||||
}
|
||||
return nil, err
|
||||
@@ -285,6 +301,15 @@ func (c *ConfigPersistence) LoadAlertConfig() (*alerts.AlertConfig, error) {
|
||||
if delay, ok := config.TimeThresholds["all"]; ok && delay <= 0 {
|
||||
config.TimeThresholds["all"] = config.TimeThreshold
|
||||
}
|
||||
if config.SnapshotDefaults.WarningDays < 0 {
|
||||
config.SnapshotDefaults.WarningDays = 0
|
||||
}
|
||||
if config.SnapshotDefaults.CriticalDays < 0 {
|
||||
config.SnapshotDefaults.CriticalDays = 0
|
||||
}
|
||||
if config.SnapshotDefaults.CriticalDays > 0 && config.SnapshotDefaults.WarningDays > config.SnapshotDefaults.CriticalDays {
|
||||
config.SnapshotDefaults.WarningDays = config.SnapshotDefaults.CriticalDays
|
||||
}
|
||||
config.MetricTimeThresholds = alerts.NormalizeMetricTimeThresholds(config.MetricTimeThresholds)
|
||||
config.DockerIgnoredContainerPrefixes = alerts.NormalizeDockerIgnoredPrefixes(config.DockerIgnoredContainerPrefixes)
|
||||
|
||||
@@ -386,6 +411,82 @@ func (c *ConfigPersistence) LoadEmailConfig() (*notifications.EmailConfig, error
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
// SaveAppriseConfig saves Apprise configuration to file (encrypted if available)
|
||||
func (c *ConfigPersistence) SaveAppriseConfig(config notifications.AppriseConfig) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
config = notifications.NormalizeAppriseConfig(config)
|
||||
|
||||
data, err := json.MarshalIndent(config, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := c.EnsureConfigDir(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if c.crypto != nil {
|
||||
encrypted, err := c.crypto.Encrypt(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = encrypted
|
||||
}
|
||||
|
||||
if err := os.WriteFile(c.appriseFile, data, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info().
|
||||
Str("file", c.appriseFile).
|
||||
Bool("encrypted", c.crypto != nil).
|
||||
Msg("Apprise configuration saved")
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadAppriseConfig loads Apprise configuration from file (decrypts if encrypted)
|
||||
func (c *ConfigPersistence) LoadAppriseConfig() (*notifications.AppriseConfig, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
data, err := os.ReadFile(c.appriseFile)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
defaultCfg := notifications.AppriseConfig{
|
||||
Enabled: false,
|
||||
Targets: []string{},
|
||||
CLIPath: "apprise",
|
||||
TimeoutSeconds: 15,
|
||||
}
|
||||
return &defaultCfg, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if c.crypto != nil {
|
||||
decrypted, err := c.crypto.Decrypt(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data = decrypted
|
||||
}
|
||||
|
||||
var config notifications.AppriseConfig
|
||||
if err := json.Unmarshal(data, &config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
normalized := notifications.NormalizeAppriseConfig(config)
|
||||
|
||||
log.Info().
|
||||
Str("file", c.appriseFile).
|
||||
Bool("encrypted", c.crypto != nil).
|
||||
Msg("Apprise configuration loaded")
|
||||
return &normalized, nil
|
||||
}
|
||||
|
||||
// SaveWebhooks saves webhook configurations to file
|
||||
func (c *ConfigPersistence) SaveWebhooks(webhooks []notifications.WebhookConfig) error {
|
||||
c.mu.Lock()
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
||||
)
|
||||
|
||||
func TestSaveAlertConfig_PreservesStorageOverrideHysteresis(t *testing.T) {
|
||||
@@ -134,6 +135,11 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
|
||||
TimeThreshold: 0,
|
||||
TimeThresholds: map[string]int{"guest": 0, "node": 0},
|
||||
DockerIgnoredContainerPrefixes: []string{" Runner "},
|
||||
SnapshotDefaults: alerts.SnapshotAlertConfig{
|
||||
Enabled: true,
|
||||
WarningDays: 20,
|
||||
CriticalDays: 10,
|
||||
},
|
||||
NodeDefaults: alerts.ThresholdConfig{
|
||||
Temperature: &alerts.HysteresisThreshold{Trigger: 0, Clear: 0},
|
||||
},
|
||||
@@ -172,4 +178,67 @@ func TestLoadAlertConfigAppliesDefaults(t *testing.T) {
|
||||
if !reflect.DeepEqual(loaded.DockerIgnoredContainerPrefixes, expectedPrefixes) {
|
||||
t.Fatalf("expected normalized prefixes %v, got %v", expectedPrefixes, loaded.DockerIgnoredContainerPrefixes)
|
||||
}
|
||||
if loaded.SnapshotDefaults.Enabled != true {
|
||||
t.Fatalf("expected snapshot defaults to preserve enabled state")
|
||||
}
|
||||
if loaded.SnapshotDefaults.WarningDays != 10 {
|
||||
t.Fatalf("expected snapshot warning days normalized to critical, got %d", loaded.SnapshotDefaults.WarningDays)
|
||||
}
|
||||
if loaded.SnapshotDefaults.CriticalDays != 10 {
|
||||
t.Fatalf("expected snapshot critical days preserved at 10, got %d", loaded.SnapshotDefaults.CriticalDays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppriseConfigPersistence(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
cp := config.NewConfigPersistence(tempDir)
|
||||
if err := cp.EnsureConfigDir(); err != nil {
|
||||
t.Fatalf("EnsureConfigDir: %v", err)
|
||||
}
|
||||
|
||||
cfg := notifications.AppriseConfig{
|
||||
Enabled: true,
|
||||
Targets: []string{" discord://token ", "", "mailto://alerts@example.com"},
|
||||
CLIPath: " /usr/local/bin/apprise ",
|
||||
TimeoutSeconds: 3,
|
||||
}
|
||||
|
||||
if err := cp.SaveAppriseConfig(cfg); err != nil {
|
||||
t.Fatalf("SaveAppriseConfig: %v", err)
|
||||
}
|
||||
|
||||
loaded, err := cp.LoadAppriseConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAppriseConfig: %v", err)
|
||||
}
|
||||
|
||||
if !loaded.Enabled {
|
||||
t.Fatalf("expected config to remain enabled")
|
||||
}
|
||||
|
||||
expectedTargets := []string{"discord://token", "mailto://alerts@example.com"}
|
||||
if !reflect.DeepEqual(loaded.Targets, expectedTargets) {
|
||||
t.Fatalf("unexpected targets: got %v want %v", loaded.Targets, expectedTargets)
|
||||
}
|
||||
|
||||
if loaded.CLIPath != "/usr/local/bin/apprise" {
|
||||
t.Fatalf("expected CLI path to be trimmed, got %q", loaded.CLIPath)
|
||||
}
|
||||
|
||||
if loaded.TimeoutSeconds != 5 {
|
||||
t.Fatalf("expected timeout normalized to minimum 5 seconds, got %d", loaded.TimeoutSeconds)
|
||||
}
|
||||
|
||||
// Clearing targets should disable the config on next load
|
||||
if err := cp.SaveAppriseConfig(notifications.AppriseConfig{Enabled: true}); err != nil {
|
||||
t.Fatalf("SaveAppriseConfig empty: %v", err)
|
||||
}
|
||||
|
||||
empty, err := cp.LoadAppriseConfig()
|
||||
if err != nil {
|
||||
t.Fatalf("LoadAppriseConfig empty: %v", err)
|
||||
}
|
||||
if empty.Enabled {
|
||||
t.Fatalf("expected disabled configuration when no targets stored")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1363,6 +1363,12 @@ func New(cfg *config.Config) (*Monitor, error) {
|
||||
log.Warn().Err(err).Msg("Failed to load email configuration")
|
||||
}
|
||||
|
||||
if appriseConfig, err := m.configPersist.LoadAppriseConfig(); err == nil {
|
||||
m.notificationMgr.SetAppriseConfig(*appriseConfig)
|
||||
} else {
|
||||
log.Warn().Err(err).Msg("Failed to load Apprise configuration")
|
||||
}
|
||||
|
||||
// Migrate webhooks if needed (from unencrypted to encrypted)
|
||||
if err := m.configPersist.MigrateWebhooksIfNeeded(); err != nil {
|
||||
log.Warn().Err(err).Msg("Failed to migrate webhooks")
|
||||
@@ -5686,6 +5692,21 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
|
||||
guestKey := func(instance, node string, vmid int) string {
|
||||
if instance == node {
|
||||
return fmt.Sprintf("%s-%d", node, vmid)
|
||||
}
|
||||
return fmt.Sprintf("%s-%s-%d", instance, node, vmid)
|
||||
}
|
||||
|
||||
guestNames := make(map[string]string, len(vms)+len(containers))
|
||||
for _, vm := range vms {
|
||||
guestNames[guestKey(instanceName, vm.Node, vm.VMID)] = vm.Name
|
||||
}
|
||||
for _, ct := range containers {
|
||||
guestNames[guestKey(instanceName, ct.Node, ct.VMID)] = ct.Name
|
||||
}
|
||||
|
||||
activeGuests := 0
|
||||
for _, vm := range vms {
|
||||
if !vm.Template {
|
||||
@@ -5857,6 +5878,10 @@ func (m *Monitor) pollGuestSnapshots(ctx context.Context, instanceName string, c
|
||||
// Update state with guest snapshots for this instance
|
||||
m.state.UpdateGuestSnapshotsForInstance(instanceName, allSnapshots)
|
||||
|
||||
if m.alertManager != nil {
|
||||
m.alertManager.CheckSnapshotsForInstance(instanceName, allSnapshots, guestNames)
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("instance", instanceName).
|
||||
Int("count", len(allSnapshots)).
|
||||
|
||||
@@ -2,11 +2,13 @@ package notifications
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/template"
|
||||
@@ -103,6 +105,7 @@ type NotificationManager struct {
|
||||
mu sync.RWMutex
|
||||
emailConfig EmailConfig
|
||||
webhooks []WebhookConfig
|
||||
appriseConfig AppriseConfig
|
||||
enabled bool
|
||||
cooldown time.Duration
|
||||
lastNotified map[string]notificationRecord
|
||||
@@ -114,8 +117,11 @@ type NotificationManager struct {
|
||||
groupByGuest bool
|
||||
webhookHistory []WebhookDelivery // Keep last 100 webhook deliveries for debugging
|
||||
webhookRateLimits map[string]*webhookRateLimit // Track rate limits per webhook URL
|
||||
appriseExec appriseExecFunc
|
||||
}
|
||||
|
||||
type appriseExecFunc func(ctx context.Context, path string, args []string) ([]byte, error)
|
||||
|
||||
// copyEmailConfig returns a defensive copy of EmailConfig including its slices to avoid data races.
|
||||
func copyEmailConfig(cfg EmailConfig) EmailConfig {
|
||||
copy := cfg
|
||||
@@ -154,6 +160,58 @@ func copyWebhookConfigs(webhooks []WebhookConfig) []WebhookConfig {
|
||||
return copies
|
||||
}
|
||||
|
||||
func copyAppriseConfig(cfg AppriseConfig) AppriseConfig {
|
||||
copy := cfg
|
||||
if len(cfg.Targets) > 0 {
|
||||
copy.Targets = append([]string(nil), cfg.Targets...)
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
// NormalizeAppriseConfig cleans and normalizes Apprise configuration values.
|
||||
func NormalizeAppriseConfig(cfg AppriseConfig) AppriseConfig {
|
||||
normalized := cfg
|
||||
normalized.CLIPath = strings.TrimSpace(normalized.CLIPath)
|
||||
if normalized.CLIPath == "" {
|
||||
normalized.CLIPath = "apprise"
|
||||
}
|
||||
|
||||
if normalized.TimeoutSeconds <= 0 {
|
||||
normalized.TimeoutSeconds = 15
|
||||
} else if normalized.TimeoutSeconds > 120 {
|
||||
normalized.TimeoutSeconds = 120
|
||||
} else if normalized.TimeoutSeconds < 5 {
|
||||
normalized.TimeoutSeconds = 5
|
||||
}
|
||||
|
||||
cleanTargets := make([]string, 0, len(normalized.Targets))
|
||||
seen := make(map[string]struct{}, len(normalized.Targets))
|
||||
for _, target := range normalized.Targets {
|
||||
trimmed := strings.TrimSpace(target)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
lower := strings.ToLower(trimmed)
|
||||
if _, exists := seen[lower]; exists {
|
||||
continue
|
||||
}
|
||||
seen[lower] = struct{}{}
|
||||
cleanTargets = append(cleanTargets, trimmed)
|
||||
}
|
||||
|
||||
normalized.Targets = cleanTargets
|
||||
if len(cleanTargets) == 0 {
|
||||
normalized.Enabled = false
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
func defaultAppriseExec(ctx context.Context, path string, args []string) ([]byte, error) {
|
||||
cmd := exec.CommandContext(ctx, path, args...)
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
type notificationRecord struct {
|
||||
lastSent time.Time
|
||||
alertStart time.Time
|
||||
@@ -200,6 +258,14 @@ type WebhookConfig struct {
|
||||
CustomFields map[string]string `json:"customFields,omitempty"`
|
||||
}
|
||||
|
||||
// AppriseConfig holds Apprise CLI notification settings.
|
||||
type AppriseConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Targets []string `json:"targets"`
|
||||
CLIPath string `json:"cliPath,omitempty"`
|
||||
TimeoutSeconds int `json:"timeoutSeconds,omitempty"`
|
||||
}
|
||||
|
||||
// NewNotificationManager creates a new notification manager
|
||||
func NewNotificationManager(publicURL string) *NotificationManager {
|
||||
if publicURL != "" {
|
||||
@@ -208,10 +274,16 @@ func NewNotificationManager(publicURL string) *NotificationManager {
|
||||
log.Info().Msg("NotificationManager initialized without public URL - webhook links may not work")
|
||||
}
|
||||
return &NotificationManager{
|
||||
enabled: true,
|
||||
cooldown: 5 * time.Minute,
|
||||
lastNotified: make(map[string]notificationRecord),
|
||||
webhooks: []WebhookConfig{},
|
||||
enabled: true,
|
||||
cooldown: 5 * time.Minute,
|
||||
lastNotified: make(map[string]notificationRecord),
|
||||
webhooks: []WebhookConfig{},
|
||||
appriseConfig: AppriseConfig{
|
||||
Enabled: false,
|
||||
Targets: []string{},
|
||||
CLIPath: "apprise",
|
||||
TimeoutSeconds: 15,
|
||||
},
|
||||
groupWindow: 30 * time.Second,
|
||||
pendingAlerts: make([]*alerts.Alert, 0),
|
||||
groupByNode: true,
|
||||
@@ -219,6 +291,7 @@ func NewNotificationManager(publicURL string) *NotificationManager {
|
||||
webhookHistory: make([]WebhookDelivery, 0, WebhookHistoryMaxSize),
|
||||
webhookRateLimits: make(map[string]*webhookRateLimit),
|
||||
publicURL: publicURL,
|
||||
appriseExec: defaultAppriseExec,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,6 +302,20 @@ func (n *NotificationManager) SetEmailConfig(config EmailConfig) {
|
||||
n.emailConfig = config
|
||||
}
|
||||
|
||||
// SetAppriseConfig updates Apprise configuration.
|
||||
func (n *NotificationManager) SetAppriseConfig(config AppriseConfig) {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
n.appriseConfig = NormalizeAppriseConfig(config)
|
||||
}
|
||||
|
||||
// GetAppriseConfig returns a copy of the Apprise configuration.
|
||||
func (n *NotificationManager) GetAppriseConfig() AppriseConfig {
|
||||
n.mu.RLock()
|
||||
defer n.mu.RUnlock()
|
||||
return copyAppriseConfig(n.appriseConfig)
|
||||
}
|
||||
|
||||
// SetCooldown updates the cooldown duration
|
||||
func (n *NotificationManager) SetCooldown(minutes int) {
|
||||
n.mu.Lock()
|
||||
@@ -434,6 +521,7 @@ func (n *NotificationManager) sendGroupedAlerts() {
|
||||
// Snapshot configuration while holding the lock to avoid races with concurrent updates
|
||||
emailConfig := copyEmailConfig(n.emailConfig)
|
||||
webhooks := copyWebhookConfigs(n.webhooks)
|
||||
appriseConfig := copyAppriseConfig(n.appriseConfig)
|
||||
|
||||
// Send notifications using the captured snapshots outside the lock to avoid blocking writers
|
||||
if emailConfig.Enabled {
|
||||
@@ -457,6 +545,10 @@ func (n *NotificationManager) sendGroupedAlerts() {
|
||||
}
|
||||
}
|
||||
|
||||
if appriseConfig.Enabled && len(appriseConfig.Targets) > 0 {
|
||||
go n.sendGroupedApprise(appriseConfig, alertsToSend)
|
||||
}
|
||||
|
||||
// Update last notified time for all alerts
|
||||
now := time.Now()
|
||||
for _, alert := range alertsToSend {
|
||||
@@ -480,6 +572,76 @@ func (n *NotificationManager) sendGroupedEmail(config EmailConfig, alertList []*
|
||||
n.sendHTMLEmail(subject, htmlBody, textBody, config)
|
||||
}
|
||||
|
||||
func (n *NotificationManager) sendGroupedApprise(config AppriseConfig, alertList []*alerts.Alert) {
|
||||
if len(alertList) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
cfg := NormalizeAppriseConfig(config)
|
||||
if !cfg.Enabled || len(cfg.Targets) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
primary := alertList[0]
|
||||
alertCount := len(alertList)
|
||||
|
||||
title := fmt.Sprintf("Pulse alert: %s", primary.ResourceName)
|
||||
if alertCount > 1 {
|
||||
title = fmt.Sprintf("Pulse alerts (%d)", alertCount)
|
||||
}
|
||||
|
||||
var bodyBuilder strings.Builder
|
||||
bodyBuilder.WriteString(primary.Message)
|
||||
bodyBuilder.WriteString("\n\n")
|
||||
|
||||
for _, alert := range alertList {
|
||||
bodyBuilder.WriteString(fmt.Sprintf("[%s] %s", strings.ToUpper(string(alert.Level)), alert.ResourceName))
|
||||
bodyBuilder.WriteString(fmt.Sprintf(" — value %.2f (threshold %.2f)\n", alert.Value, alert.Threshold))
|
||||
if alert.Node != "" {
|
||||
bodyBuilder.WriteString(fmt.Sprintf("Node: %s\n", alert.Node))
|
||||
}
|
||||
if alert.Instance != "" && alert.Instance != alert.Node {
|
||||
bodyBuilder.WriteString(fmt.Sprintf("Instance: %s\n", alert.Instance))
|
||||
}
|
||||
bodyBuilder.WriteString("\n")
|
||||
}
|
||||
|
||||
if n.publicURL != "" {
|
||||
bodyBuilder.WriteString("Dashboard: " + n.publicURL + "\n")
|
||||
}
|
||||
|
||||
body := bodyBuilder.String()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(cfg.TimeoutSeconds)*time.Second)
|
||||
defer cancel()
|
||||
|
||||
args := []string{"-t", title, "-b", body}
|
||||
args = append(args, cfg.Targets...)
|
||||
|
||||
execFn := n.appriseExec
|
||||
if execFn == nil {
|
||||
execFn = defaultAppriseExec
|
||||
}
|
||||
|
||||
output, err := execFn(ctx, cfg.CLIPath, args)
|
||||
if err != nil {
|
||||
log.Warn().
|
||||
Err(err).
|
||||
Str("cliPath", cfg.CLIPath).
|
||||
Strs("targets", cfg.Targets).
|
||||
Msg("Failed to send Apprise notification")
|
||||
return
|
||||
}
|
||||
|
||||
if len(output) > 0 {
|
||||
log.Debug().
|
||||
Str("cliPath", cfg.CLIPath).
|
||||
Strs("targets", cfg.Targets).
|
||||
Str("output", string(output)).
|
||||
Msg("Apprise CLI output")
|
||||
}
|
||||
}
|
||||
|
||||
// sendEmail sends an email notification
|
||||
func (n *NotificationManager) sendEmail(alert *alerts.Alert) {
|
||||
n.mu.RLock()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -17,6 +18,96 @@ func flushPending(n *NotificationManager) {
|
||||
n.sendGroupedAlerts()
|
||||
}
|
||||
|
||||
func TestNormalizeAppriseConfig(t *testing.T) {
|
||||
original := AppriseConfig{
|
||||
Enabled: true,
|
||||
Targets: []string{" discord://token ", "", "DISCORD://TOKEN"},
|
||||
CLIPath: " ",
|
||||
TimeoutSeconds: -5,
|
||||
}
|
||||
|
||||
normalized := NormalizeAppriseConfig(original)
|
||||
|
||||
if normalized.CLIPath != "apprise" {
|
||||
t.Fatalf("expected default CLI path 'apprise', got %q", normalized.CLIPath)
|
||||
}
|
||||
|
||||
if normalized.TimeoutSeconds != 15 {
|
||||
t.Fatalf("expected timeout of 15 seconds, got %d", normalized.TimeoutSeconds)
|
||||
}
|
||||
|
||||
if !normalized.Enabled {
|
||||
t.Fatalf("expected config to remain enabled when targets exist")
|
||||
}
|
||||
|
||||
if len(normalized.Targets) != 1 || normalized.Targets[0] != "discord://token" {
|
||||
t.Fatalf("unexpected targets normalization result: %#v", normalized.Targets)
|
||||
}
|
||||
|
||||
// When all targets removed, enabled should reset to false
|
||||
empty := NormalizeAppriseConfig(AppriseConfig{Enabled: true})
|
||||
if empty.Enabled {
|
||||
t.Fatalf("expected enabled to be false when no targets configured")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendGroupedAppriseInvokesExecutor(t *testing.T) {
|
||||
nm := NewNotificationManager("")
|
||||
nm.SetGroupingWindow(0)
|
||||
nm.SetEmailConfig(EmailConfig{Enabled: false})
|
||||
|
||||
done := make(chan struct{})
|
||||
var capturedArgs []string
|
||||
|
||||
nm.appriseExec = func(ctx context.Context, path string, args []string) ([]byte, error) {
|
||||
if path != "apprise" {
|
||||
t.Fatalf("expected CLI path 'apprise', got %q", path)
|
||||
}
|
||||
capturedArgs = append([]string(nil), args...)
|
||||
close(done)
|
||||
return []byte("success"), nil
|
||||
}
|
||||
|
||||
nm.SetAppriseConfig(AppriseConfig{
|
||||
Enabled: true,
|
||||
Targets: []string{"discord://token"},
|
||||
TimeoutSeconds: 10,
|
||||
})
|
||||
|
||||
alert := &alerts.Alert{
|
||||
ID: "test",
|
||||
Type: "cpu",
|
||||
Level: alerts.AlertLevelCritical,
|
||||
ResourceID: "vm-100",
|
||||
ResourceName: "vm-100",
|
||||
Message: "CPU usage high",
|
||||
Value: 95,
|
||||
Threshold: 90,
|
||||
StartTime: time.Now().Add(-time.Minute),
|
||||
LastSeen: time.Now(),
|
||||
}
|
||||
|
||||
nm.mu.Lock()
|
||||
nm.pendingAlerts = append(nm.pendingAlerts, alert)
|
||||
nm.mu.Unlock()
|
||||
|
||||
nm.sendGroupedAlerts()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatalf("timed out waiting for Apprise executor to run")
|
||||
}
|
||||
|
||||
if len(capturedArgs) == 0 {
|
||||
t.Fatalf("expected Apprise executor to receive arguments")
|
||||
}
|
||||
|
||||
if capturedArgs[len(capturedArgs)-1] != "discord://token" {
|
||||
t.Fatalf("expected target URL as last argument, got %v", capturedArgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotificationCooldownAllowsNewAlertInstance(t *testing.T) {
|
||||
nm := NewNotificationManager("")
|
||||
nm.SetCooldown(1) // 1 minute cooldown
|
||||
|
||||
Reference in New Issue
Block a user