Repository: kopia/kopia
Pull request #4234 (2024-11-06 18:50:09)

feat(server): add logging of failed and successful login attempts
A minimal commit to add support for monitoring, troubleshooting, and to allow using tools like crowdsec or fail2ban. This to ease the comfort a bit on opening up access to a kopia server over internet.

When/if change is merged or a similar is created I can create instructions and templates for using said tools to protect a kopia server instance running om Linux.

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/kopia/kopia/pull/4234?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) Report
Attention: Patch coverage is `50.00000%` with `3 lines` in your changes missing coverage. Please review.
> Project coverage is 76.06%. Comparing base [(`cb455c6`)](https://app.codecov.io/gh/kopia/kopia/commit/cb455c699a04e54cb47e25a676d7d65063ce883e?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) to head [(`7ace684`)](https://app.codecov.io/gh/kopia/kopia/commit/7ace684c568a3cfc98a91411595dc8df9f6e925d?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia).
> Report is 337 commits behind head on master.

| [Files with missing lines](https://app.codecov.io/gh/kopia/kopia/pull/4234?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) | Patch % | Lines |
|---|---|---|
| [internal/server/server.go](https://app.codecov.io/gh/kopia/kopia/pull/4234?src=pr&el=tree&filepath=internal%2Fserver%2Fserver.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia#diff-aW50ZXJuYWwvc2VydmVyL3NlcnZlci5nbw==) | 50.00% | [3 Missing :warning: ](https://app.codecov.io/gh/kopia/kopia/pull/4234?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #4234      +/-   ##
==========================================
+ Coverage   75.86%   76.06%   +0.20%     
==========================================
  Files         470      506      +36     
  Lines       37301    38834    +1533     
==========================================
+ Hits        28299    29540    +1241     
- Misses       7071     7333     +262     
- Partials     1931     1961      +30     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/kopia/kopia/pull/4234?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=kopia).

linust: Renamed PR to match regexp. (Missed this routine)

Patch:
diff --git a/cli/app.go b/cli/app.go
index 43083575..b9bf1fe1 100644
--- a/cli/app.go
+++ b/cli/app.go
@@ -16,9 +16,13 @@
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/kopia/kopia/internal/apiclient"
+	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/internal/gather"
 	"github.com/kopia/kopia/internal/passwordpersist"
 	"github.com/kopia/kopia/internal/releasable"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/logging"
@@ -88,6 +92,7 @@ type appServices interface {
 	repositoryConfigFileName() string
 	getProgress() *cliProgress
 	getRestoreProgress() RestoreProgress
+	notificationTemplateOptions() notifytemplate.Options
 
 	stdout() io.Writer
 	Stderr() io.Writer
@@ -112,6 +117,7 @@ type advancedAppServices interface {
 	getPasswordFromFlags(ctx context.Context, isCreate, allowPersistent bool) (string, error)
 	optionsFromFlags(ctx context.Context) *repo.Options
 	runAppWithContext(command *kingpin.CmdClause, callback func(ctx context.Context) error) error
+	enableErrorNotifications() bool
 }
 
 // App contains per-invocation flags and state of Kopia CLI.
@@ -139,6 +145,8 @@ type App struct {
 	upgradeOwnerID      string
 	doNotWaitForUpgrade bool
 
+	errorNotifications bool
+
 	currentAction         string
 	onExitCallbacks       []func()
 	onFatalErrorCallbacks []func(err error)
@@ -240,6 +248,10 @@ func (c *App) passwordPersistenceStrategy() passwordpersist.Strategy {
 	return passwordpersist.File()
 }
 
+func (c *App) enableErrorNotifications() bool {
+	return c.errorNotifications
+}
+
 func (c *App) setup(app *kingpin.Application) {
 	app.PreAction(func(pc *kingpin.ParseContext) error {
 		if sc := pc.SelectedCommand; sc != nil {
@@ -274,6 +286,7 @@ func (c *App) setup(app *kingpin.Application) {
 	app.Flag("dump-allocator-stats", "Dump allocator stats at the end of execution.").Hidden().Envar(c.EnvName("KOPIA_DUMP_ALLOCATOR_STATS")).BoolVar(&c.dumpAllocatorStats)
 	app.Flag("upgrade-owner-id", "Repository format upgrade owner-id.").Hidden().Envar(c.EnvName("KOPIA_REPO_UPGRADE_OWNER_ID")).StringVar(&c.upgradeOwnerID)
 	app.Flag("upgrade-no-block", "Do not block when repository format upgrade is in progress, instead exit with a message.").Hidden().Default("false").Envar(c.EnvName("KOPIA_REPO_UPGRADE_NO_BLOCK")).BoolVar(&c.doNotWaitForUpgrade)
+	app.Flag("error-notifications", "Send notification on errors").Hidden().Envar(c.EnvName("KOPIA_SEND_ERROR_NOTIFICATIONS")).Default("true").BoolVar(&c.errorNotifications)
 
 	if c.enableTestOnlyFlags() {
 		app.Flag("ignore-missing-required-features", "Open repository despite missing features (VERY DANGEROUS, ONLY FOR TESTING)").Hidden().BoolVar(&c.testonlyIgnoreMissingRequiredFeatures)
@@ -562,6 +575,8 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos
 			return errors.Wrap(err, "open repository")
 		}
 
+		t0 := clock.Now()
+
 		err = act(ctx, rep)
 
 		if rep != nil && err == nil && !mode.disableMaintenance {
@@ -570,6 +585,17 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos
 			}
 		}
 
+		if err != nil && rep != nil && c.errorNotifications {
+			notification.Send(ctx, rep, "generic-error", notifydata.NewErrorInfo(
+				c.currentActionName(),
+				c.currentActionName(),
+				t0,
+				clock.Now(),
+				err), notification.SeverityError,
+				c.notificationTemplateOptions(),
+			)
+		}
+
 		if rep != nil && mode.mustBeConnected {
 			if cerr := rep.Close(ctx); cerr != nil {
 				return errors.Wrap(cerr, "unable to close repository")
@@ -647,6 +673,11 @@ func (c *App) advancedCommand(ctx context.Context) {
 	}
 }
 
+func (c *App) notificationTemplateOptions() notifytemplate.Options {
+	// perhaps make this configurable in the future
+	return notifytemplate.DefaultOptions
+}
+
 func init() {
 	kingpin.EnableFileExpansion = false
 }
diff --git a/cli/command_notification_template_set.go b/cli/command_notification_template_set.go
index 82d5b6b1..5c1ba85b 100644
--- a/cli/command_notification_template_set.go
+++ b/cli/command_notification_template_set.go
@@ -77,7 +77,7 @@ func (c *commandNotificationTemplateSet) launchEditor(ctx context.Context, rep r
 	var lastUpdated string
 
 	if err := editor.EditLoop(ctx, "template.md", s, false, func(updated string) error {
-		_, err := notifytemplate.ParseTemplate(updated)
+		_, err := notifytemplate.ParseTemplate(updated, notifytemplate.DefaultOptions)
 		if err == nil {
 			lastUpdated = updated
 			return nil
diff --git a/cli/command_server_start.go b/cli/command_server_start.go
index 98ff726a..cb509490 100644
--- a/cli/command_server_start.go
+++ b/cli/command_server_start.go
@@ -161,6 +161,9 @@ func (c *commandServerStart) serverStartOptions(ctx context.Context) (*server.Op
 		DebugScheduler:         c.debugScheduler,
 		MinMaintenanceInterval: c.minMaintenanceInterval,
 		DisableCSRFTokenChecks: c.disableCSRFTokenChecks,
+
+		EnableErrorNotifications: c.svc.enableErrorNotifications(),
+		NotifyTemplateOptions:    c.svc.notificationTemplateOptions(),
 	}, nil
 }
 
diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go
index 26700cbe..7f34127e 100644
--- a/cli/command_snapshot_create.go
+++ b/cli/command_snapshot_create.go
@@ -12,6 +12,8 @@
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/fs/virtualfs"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/kopia/kopia/snapshot/policy"
@@ -127,6 +129,8 @@ func (c *commandSnapshotCreate) run(ctx context.Context, rep repo.RepositoryWrit
 		return err
 	}
 
+	var st notifydata.MultiSnapshotStatus
+
 	for _, snapshotDir := range sources {
 		if u.IsCanceled() {
 			log(ctx).Info("Upload canceled")
@@ -138,11 +142,13 @@ func (c *commandSnapshotCreate) run(ctx context.Context, rep repo.RepositoryWrit
 			finalErrors = append(finalErrors, fmt.Sprintf("failed to prepare source: %s", err))
 		}
 
-		if err := c.snapshotSingleSource(ctx, fsEntry, setManual, rep, u, sourceInfo, tags); err != nil {
+		if err := c.snapshotSingleSource(ctx, fsEntry, setManual, rep, u, sourceInfo, tags, &st); err != nil {
 			finalErrors = append(finalErrors, err.Error())
 		}
 	}
 
+	notification.Send(ctx, rep, "snapshot-report", st, notification.SeverityReport, c.svc.notificationTemplateOptions())
+
 	// ensure we flush at least once in the session to properly close all pending buffers,
 	// otherwise the session will be reported as memory leak.
 	// by default the wrapper function does not flush on errors, which is what we want to do always.
@@ -259,27 +265,48 @@ func startTimeAfterEndTime(startTime, endTime time.Time) bool {
 		startTime.After(endTime)
 }
 
-//nolint:gocyclo
-func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntry fs.Entry, setManual bool, rep repo.RepositoryWriter, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo, tags map[string]string) error {
+//nolint:gocyclo,funlen
+func (c *commandSnapshotCreate) snapshotSingleSource(
+	ctx context.Context,
+	fsEntry fs.Entry,
+	setManual bool,
+	rep repo.RepositoryWriter,
+	u *snapshotfs.Uploader,
+	sourceInfo snapshot.SourceInfo,
+	tags map[string]string,
+	st *notifydata.MultiSnapshotStatus,
+) (finalErr error) {
 	log(ctx).Infof("Snapshotting %v ...", sourceInfo)
 
-	var err error
+	var mwe notifydata.ManifestWithError
 
-	previous, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil)
-	if err != nil {
-		return err
+	mwe.Source = sourceInfo
+
+	st.Snapshots = append(st.Snapshots, &mwe)
+
+	defer func() {
+		if finalErr != nil {
+			mwe.Error = finalErr.Error()
+		}
+	}()
+
+	var previous []*snapshot.Manifest
+
+	previous, finalErr = findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil)
+	if finalErr != nil {
+		return finalErr
 	}
 
-	policyTree, err := policy.TreeForSource(ctx, rep, sourceInfo)
-	if err != nil {
-		return errors.Wrap(err, "unable to get policy tree")
+	policyTree, finalErr := policy.TreeForSource(ctx, rep, sourceInfo)
+	if finalErr != nil {
+		return errors.Wrap(finalErr, "unable to get policy tree")
 	}
 
-	manifest, err := u.Upload(ctx, fsEntry, policyTree, sourceInfo, previous...)
-	if err != nil {
+	manifest, finalErr := u.Upload(ctx, fsEntry, policyTree, sourceInfo, previous...)
+	if finalErr != nil {
 		// fail-fast uploads will fail here without recording a manifest, other uploads will
 		// possibly fail later.
-		return errors.Wrap(err, "upload error")
+		return errors.Wrap(finalErr, "upload error")
 	}
 
 	manifest.Description = c.snapshotCreateDescription
@@ -308,6 +335,8 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntr
 		manifest.EndTime = fs.UTCTimestampFromTime(endTimeOverride)
 	}
 
+	mwe.Manifest = *manifest
+
 	ignoreIdenticalSnapshot := policyTree.EffectivePolicy().RetentionPolicy.IgnoreIdenticalSnapshots.OrDefault(false)
 	if ignoreIdenticalSnapshot && len(previous) > 0 {
 		if previous[0].RootObjectID() == manifest.RootObjectID() {
@@ -316,17 +345,17 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntr
 		}
 	}
 
-	if _, err = snapshot.SaveSnapshot(ctx, rep, manifest); err != nil {
-		return errors.Wrap(err, "cannot save manifest")
+	if _, finalErr = snapshot.SaveSnapshot(ctx, rep, manifest); finalErr != nil {
+		return errors.Wrap(finalErr, "cannot save manifest")
 	}
 
-	if _, err = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true); err != nil {
-		return errors.Wrap(err, "unable to apply retention policy")
+	if _, finalErr = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true); finalErr != nil {
+		return errors.Wrap(finalErr, "unable to apply retention policy")
 	}
 
 	if setManual {
-		if err = policy.SetManual(ctx, rep, sourceInfo); err != nil {
-			return errors.Wrap(err, "unable to set manual field in scheduling policy for source")
+		if finalErr = policy.SetManual(ctx, rep, sourceInfo); finalErr != nil {
+			return errors.Wrap(finalErr, "unable to set manual field in scheduling policy for source")
 		}
 	}
 
diff --git a/go.mod b/go.mod
index 9a5d9745..e8df41ba 100644
--- a/go.mod
+++ b/go.mod
@@ -56,17 +56,17 @@ require (
 	go.opentelemetry.io/otel/trace v1.31.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.28.0
+	golang.org/x/crypto v0.29.0
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	golang.org/x/mod v0.21.0
-	golang.org/x/net v0.30.0
-	golang.org/x/oauth2 v0.23.0
-	golang.org/x/sync v0.8.0
-	golang.org/x/sys v0.26.0
-	golang.org/x/term v0.25.0
-	golang.org/x/text v0.19.0
-	google.golang.org/api v0.204.0
-	google.golang.org/grpc v1.67.1
+	golang.org/x/mod v0.22.0
+	golang.org/x/net v0.31.0
+	golang.org/x/oauth2 v0.24.0
+	golang.org/x/sync v0.9.0
+	golang.org/x/sys v0.27.0
+	golang.org/x/term v0.26.0
+	golang.org/x/text v0.20.0
+	google.golang.org/api v0.205.0
+	google.golang.org/grpc v1.68.0
 	google.golang.org/protobuf v1.35.1
 	gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216
 )
@@ -75,7 +75,7 @@ require (
 	al.essio.dev/pkg/shellescape v1.5.1 // indirect
 	cel.dev/expr v0.16.1 // indirect
 	cloud.google.com/go v0.116.0 // indirect
-	cloud.google.com/go/auth v0.10.0 // indirect
+	cloud.google.com/go/auth v0.10.1 // indirect
 	cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
 	cloud.google.com/go/compute/metadata v0.5.2 // indirect
 	cloud.google.com/go/iam v1.2.1 // indirect
diff --git a/go.sum b/go.sum
index 93263d73..2f90b847 100644
--- a/go.sum
+++ b/go.sum
@@ -5,8 +5,8 @@ cel.dev/expr v0.16.1/go.mod h1:AsGA5zb3WruAEQeQng1RZdGEXmBj0jvMWh6l5SnNuC8=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE=
 cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U=
-cloud.google.com/go/auth v0.10.0 h1:tWlkvFAh+wwTOzXIjrwM64karR1iTBZ/GRr0S/DULYo=
-cloud.google.com/go/auth v0.10.0/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
+cloud.google.com/go/auth v0.10.1 h1:TnK46qldSfHWt2a0b/hciaiVJsmDXWy9FqyUan0uYiI=
+cloud.google.com/go/auth v0.10.1/go.mod h1:xxA5AqpDrvS+Gkmo9RqrGGRh6WSNKKOXhY3zNOr38tI=
 cloud.google.com/go/auth/oauth2adapt v0.2.5 h1:2p29+dePqsCHPP1bqDJcKj4qxRyYCcbzKpFyKGt3MTk=
 cloud.google.com/go/auth/oauth2adapt v0.2.5/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=
 cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
@@ -350,8 +350,8 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
-golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
-golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
+golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
+golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
@@ -360,8 +360,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
-golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
+golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -373,18 +373,18 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
-golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
-golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
+golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -400,23 +400,24 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
 golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
-golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
-golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
+golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU=
+golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
 golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
 golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -429,8 +430,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.204.0 h1:3PjmQQEDkR/ENVZZwIYB4W/KzYtN8OrqnNcHWpeR8E4=
-google.golang.org/api v0.204.0/go.mod h1:69y8QSoKIbL9F94bWgWAq6wGqGwyjBgi2y8rAK8zLag=
+google.golang.org/api v0.205.0 h1:LFaxkAIpDb/GsrWV20dMMo5MR0h8UARTbn24LmD+0Pg=
+google.golang.org/api v0.205.0/go.mod h1:NrK1EMqO8Xk6l6QwRAmrXXg2v6dzukhlOyvkYtnvUuc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -447,8 +448,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
-google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
+google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0=
+google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA=
 google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a h1:UIpYSuWdWHSzjwcAFRLjKcPXFZVVLXGEM23W+NWqipw=
 google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a/go.mod h1:9i1T9n4ZinTUZGgzENMi8MDDgbGC5mqTS75JAv6xN3A=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
diff --git a/internal/server/grpc_session.go b/internal/server/grpc_session.go
index 3779ea26..e532b288 100644
--- a/internal/server/grpc_session.go
+++ b/internal/server/grpc_session.go
@@ -132,7 +132,7 @@ func (s *Server) Session(srv grpcapi.KopiaRepository_SessionServer) error {
 			go func() {
 				defer s.grpcServerState.sem.Release(1)
 
-				handleSessionRequest(ctx, dw, authz, usernameAtHostname, req, func(resp *grpcapi.SessionResponse) {
+				s.handleSessionRequest(ctx, dw, authz, usernameAtHostname, req, func(resp *grpcapi.SessionResponse) {
 					if err := s.send(srv, req.GetRequestId(), resp); err != nil {
 						select {
 						case lastErr <- err:
@@ -149,7 +149,7 @@ func (s *Server) Session(srv grpcapi.KopiaRepository_SessionServer) error {
 
 var tracer = otel.Tracer("kopia/grpc")
 
-func handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, authz auth.AuthorizationInfo, usernameAtHostname string, req *grpcapi.SessionRequest, respond func(*grpcapi.SessionResponse)) {
+func (s *Server) handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, authz auth.AuthorizationInfo, usernameAtHostname string, req *grpcapi.SessionRequest, respond func(*grpcapi.SessionResponse)) {
 	if req.GetTraceContext() != nil {
 		var tc propagation.TraceContext
 		ctx = tc.Extract(ctx, propagation.MapCarrier(req.GetTraceContext()))
@@ -187,7 +187,7 @@ func handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, a
 		respond(handleApplyRetentionPolicyRequest(ctx, dw, authz, usernameAtHostname, inner.ApplyRetentionPolicy))
 
 	case *grpcapi.SessionRequest_SendNotification:
-		respond(handleSendNotificationRequest(ctx, dw, authz, inner.SendNotification))
+		respond(s.handleSendNotificationRequest(ctx, dw, authz, inner.SendNotification))
 
 	case *grpcapi.SessionRequest_InitializeSession:
 		respond(errorResponse(errors.New("InitializeSession must be the first request in a session")))
@@ -488,7 +488,7 @@ func handleApplyRetentionPolicyRequest(ctx context.Context, rep repo.RepositoryW
 	}
 }
 
-func handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWriter, authz auth.AuthorizationInfo, req *grpcapi.SendNotificationRequest) *grpcapi.SessionResponse {
+func (s *Server) handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWriter, authz auth.AuthorizationInfo, req *grpcapi.SendNotificationRequest) *grpcapi.SessionResponse {
 	ctx, span := tracer.Start(ctx, "GRPCSession.SendNotification")
 	defer span.End()
 
@@ -499,7 +499,8 @@ func handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWrite
 	if err := notification.SendInternal(ctx, rep,
 		req.GetTemplateName(),
 		json.RawMessage(req.GetEventArgs()),
-		notification.Severity(req.GetSeverity())); err != nil {
+		notification.Severity(req.GetSeverity()),
+		s.options.NotifyTemplateOptions); err != nil {
 		return errorResponse(err)
 	}
 
diff --git a/internal/server/server.go b/internal/server/server.go
index a0e4e516..208c5048 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -26,6 +26,7 @@
 	"github.com/kopia/kopia/internal/scheduler"
 	"github.com/kopia/kopia/internal/serverapi"
 	"github.com/kopia/kopia/internal/uitask"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/logging"
 	"github.com/kopia/kopia/repo/maintenance"
@@ -207,6 +208,9 @@ func isAuthenticated(rc requestContext) bool {
 		rc.w.Header().Set("WWW-Authenticate", `Basic realm="Kopia"`)
 		http.Error(rc.w, "Access denied.\n", http.StatusUnauthorized)
 
+		// Log failed authentication attempt
+		log(rc.req.Context()).Warnf("failed login attempt by client %s for user %s", rc.req.RemoteAddr, username)
+
 		return false
 	}
 
@@ -222,6 +226,9 @@ func isAuthenticated(rc requestContext) bool {
 			Expires: now.Add(kopiaAuthCookieTTL),
 			Path:    "/",
 		})
+
+		// Log successful authentication
+		log(rc.req.Context()).Infof("successful login by client %s for user %s", rc.req.RemoteAddr, username)
 	}
 
 	return true
@@ -535,6 +542,14 @@ func (s *Server) endUpload(ctx context.Context, src snapshot.SourceInfo) {
 	s.parallelSnapshotsChanged.Signal()
 }
 
+func (s *Server) enableErrorNotifications() bool {
+	return s.options.EnableErrorNotifications
+}
+
+func (s *Server) notificationTemplateOptions() notifytemplate.Options {
+	return s.options.NotifyTemplateOptions
+}
+
 // SetRepository sets the repository (nil is allowed and indicates server that is not
 // connected to the repository).
 func (s *Server) SetRepository(ctx context.Context, rep repo.Repository) error {
@@ -785,23 +800,25 @@ func (s *Server) ServeStaticFiles(m *mux.Router, fs http.FileSystem) {
 
 // Options encompasses all API server options.
 type Options struct {
-	ConfigFile             string
-	ConnectOptions         *repo.ConnectOptions
-	RefreshInterval        time.Duration
-	MaxConcurrency         int
-	Authenticator          auth.Authenticator
-	Authorizer             auth.Authorizer
-	PasswordPersist        passwordpersist.Strategy
-	AuthCookieSigningKey   string
-	LogRequests            bool
-	UIUser                 string // name of the user allowed to access the UI API
-	UIPreferencesFile      string // name of the JSON file storing UI preferences
-	ServerControlUser      string // name of the user allowed to access the server control API
-	DisableCSRFTokenChecks bool
-	PersistentLogs         bool
-	UITitlePrefix          string
-	DebugScheduler         bool
-	MinMaintenanceInterval time.Duration
+	ConfigFile               string
+	ConnectOptions           *repo.ConnectOptions
+	RefreshInterval          time.Duration
+	MaxConcurrency           int
+	Authenticator            auth.Authenticator
+	Authorizer               auth.Authorizer
+	PasswordPersist          passwordpersist.Strategy
+	AuthCookieSigningKey     string
+	LogRequests              bool
+	UIUser                   string // name of the user allowed to access the UI API
+	UIPreferencesFile        string // name of the JSON file storing UI preferences
+	ServerControlUser        string // name of the user allowed to access the server control API
+	DisableCSRFTokenChecks   bool
+	PersistentLogs           bool
+	UITitlePrefix            string
+	DebugScheduler           bool
+	MinMaintenanceInterval   time.Duration
+	EnableErrorNotifications bool
+	NotifyTemplateOptions    notifytemplate.Options
 }
 
 // InitRepositoryFunc is a function that attempts to connect to/open repository.
diff --git a/internal/server/server_maintenance.go b/internal/server/server_maintenance.go
index 8faedbcd..1703d1e8 100644
--- a/internal/server/server_maintenance.go
+++ b/internal/server/server_maintenance.go
@@ -8,6 +8,9 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/internal/clock"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/maintenance"
 )
@@ -32,6 +35,8 @@ type srvMaintenance struct {
 type maintenanceManagerServerInterface interface {
 	runMaintenanceTask(ctx context.Context, dr repo.DirectRepository) error
 	refreshScheduler(reason string)
+	enableErrorNotifications() bool
+	notificationTemplateOptions() notifytemplate.Options
 }
 
 func (s *srvMaintenance) trigger() {
@@ -138,9 +143,21 @@ func startMaintenanceManager(
 
 				m.beforeRun()
 
+				t0 := clock.Now()
+
 				if err := srv.runMaintenanceTask(mctx, rep); err != nil {
 					log(ctx).Debugw("maintenance task failed", "err", err)
 					m.afterFailedRun()
+
+					if srv.enableErrorNotifications() {
+						notification.Send(ctx,
+							rep,
+							"generic-error",
+							notifydata.NewErrorInfo("Maintenance", "Scheduled Maintenance", t0, clock.Now(), err),
+							notification.SeverityError,
+							srv.notificationTemplateOptions(),
+						)
+					}
 				}
 
 				m.refresh(mctx, true)
diff --git a/notification/notification_send.go b/notification/notification_send.go
index 1a4c2b92..200f34d1 100644
--- a/notification/notification_send.go
+++ b/notification/notification_send.go
@@ -105,7 +105,7 @@ func notificationSendersFromRepo(ctx context.Context, rep repo.Repository, sever
 
 // Send sends a notification for the given event.
 // Any errors encountered during the process are logged.
-func Send(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity) {
+func Send(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) {
 	// if we're connected to a repository server, send the notification there.
 	if rem, ok := rep.(repo.RemoteNotifications); ok {
 		jsonData, err := json.Marshal(eventArgs)
@@ -122,13 +122,13 @@ func Send(ctx context.Context, rep repo.Repository, templateName string, eventAr
 		return
 	}
 
-	if err := SendInternal(ctx, rep, templateName, eventArgs, sev); err != nil {
+	if err := SendInternal(ctx, rep, templateName, eventArgs, sev, opt); err != nil {
 		log(ctx).Warnw("unable to send notification", "err", err)
 	}
 }
 
 // SendInternal sends a notification for the given event and returns an error.
-func SendInternal(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity) error {
+func SendInternal(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) error {
 	senders, err := notificationSendersFromRepo(ctx, rep, sev)
 	if err != nil {
 		return errors.Wrap(err, "unable to get notification senders")
@@ -137,7 +137,7 @@ func SendInternal(ctx context.Context, rep repo.Repository, templateName string,
 	var resultErr error
 
 	for _, s := range senders {
-		if err := SendTo(ctx, rep, s, templateName, eventArgs, sev); err != nil {
+		if err := SendTo(ctx, rep, s, templateName, eventArgs, sev, opt); err != nil {
 			resultErr = multierr.Append(resultErr, err)
 		}
 	}
@@ -166,7 +166,7 @@ func MakeTemplateArgs(eventArgs any) TemplateArgs {
 }
 
 // SendTo sends a notification to the given sender.
-func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateName string, eventArgs any, sev Severity) error {
+func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) error {
 	// execute template
 	var bodyBuf bytes.Buffer
 
@@ -175,7 +175,7 @@ func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateN
 		return errors.Wrap(err, "unable to resolve notification template")
 	}
 
-	t, err := notifytemplate.ParseTemplate(tmpl)
+	t, err := notifytemplate.ParseTemplate(tmpl, opt)
 	if err != nil {
 		return errors.Wrap(err, "unable to parse notification template")
 	}
@@ -205,5 +205,5 @@ func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateN
 func SendTestNotification(ctx context.Context, rep repo.Repository, s sender.Sender) error {
 	log(ctx).Infof("Sending test notification to %v", s.Summary())
 
-	return SendTo(ctx, rep, s, notifytemplate.TestNotification, struct{}{}, SeveritySuccess)
+	return SendTo(ctx, rep, s, notifytemplate.TestNotification, struct{}{}, SeveritySuccess, notifytemplate.DefaultOptions)
 }
diff --git a/notification/notifydata/doc.go b/notification/notifydata/doc.go
new file mode 100644
index 00000000..26449422
--- /dev/null
+++ b/notification/notifydata/doc.go
@@ -0,0 +1,2 @@
+// Package notifydata contains the data structures used by the notification package.
+package notifydata
diff --git a/notification/notifydata/error_info.go b/notification/notifydata/error_info.go
new file mode 100644
index 00000000..4002a061
--- /dev/null
+++ b/notification/notifydata/error_info.go
@@ -0,0 +1,43 @@
+package notifydata
+
+import (
+	"fmt"
+	"time"
+)
+
+// ErrorInfo represents information about errors.
+type ErrorInfo struct {
+	Operation        string    `json:"operation"`
+	OperationDetails string    `json:"operationDetails"`
+	StartTime        time.Time `json:"start"`
+	EndTime          time.Time `json:"end"`
+	ErrorMessage     string    `json:"error"`
+	ErrorDetails     string    `json:"errorDetails"`
+}
+
+// StartTimestamp returns the start time of the operation that caused the error.
+func (e *ErrorInfo) StartTimestamp() time.Time {
+	return e.StartTime.Truncate(time.Second)
+}
+
+// EndTimestamp returns the end time of the operation that caused the error.
+func (e *ErrorInfo) EndTimestamp() time.Time {
+	return e.EndTime.Truncate(time.Second)
+}
+
+// Duration returns the duration of the operation.
+func (e *ErrorInfo) Duration() time.Duration {
+	return e.EndTimestamp().Sub(e.StartTimestamp())
+}
+
+// NewErrorInfo creates a new ErrorInfo.
+func NewErrorInfo(operation, operationDetails string, startTime, endTime time.Time, err error) *ErrorInfo {
+	return &ErrorInfo{
+		Operation:        operation,
+		OperationDetails: operationDetails,
+		StartTime:        startTime,
+		EndTime:          endTime,
+		ErrorMessage:     fmt.Sprintf("%v", err),
+		ErrorDetails:     fmt.Sprintf("%+v", err),
+	}
+}
diff --git a/notification/notifydata/multi_snapshot_status.go b/notification/notifydata/multi_snapshot_status.go
new file mode 100644
index 00000000..4543ea89
--- /dev/null
+++ b/notification/notifydata/multi_snapshot_status.go
@@ -0,0 +1,96 @@
+package notifydata
+
+import (
+	"time"
+
+	"github.com/kopia/kopia/snapshot"
+)
+
+// ManifestWithError represents information about the snapshot manifest with optional error.
+type ManifestWithError struct {
+	snapshot.Manifest `json:"manifest"` // may not be filled out if there was an error, Manifst.Source is always set.
+
+	Error string `json:"error"` // will be present if there was an error
+}
+
+// StartTimestamp returns the start time of the snapshot.
+func (m *ManifestWithError) StartTimestamp() time.Time {
+	return m.StartTime.ToTime().UTC().Truncate(time.Second)
+}
+
+// EndTimestamp returns the end time of the snapshot.
+func (m *ManifestWithError) EndTimestamp() time.Time {
+	return m.EndTime.ToTime().UTC().Truncate(time.Second)
+}
+
+// TotalSize returns the total size of the snapshot in bytes.
+func (m *ManifestWithError) TotalSize() int64 {
+	if m.RootEntry == nil {
+		return 0
+	}
+
+	if m.RootEntry.DirSummary != nil {
+		return m.RootEntry.DirSummary.TotalFileSize
+	}
+
+	return m.RootEntry.FileSize
+}
+
+// TotalFiles returns the total number of files in the snapshot.
+func (m *ManifestWithError) TotalFiles() int64 {
+	if m.RootEntry == nil {
+		return 0
+	}
+
+	if m.RootEntry.DirSummary != nil {
+		return m.RootEntry.DirSummary.TotalFileCount
+	}
+
+	return 1
+}
+
+// TotalDirs returns the total number of directories in the snapshot.
+func (m *ManifestWithError) TotalDirs() int64 {
+	if m.RootEntry == nil {
+		return 0
+	}
+
+	if m.RootEntry.DirSummary != nil {
+		return m.RootEntry.DirSummary.TotalDirCount
+	}
+
+	return 0
+}
+
+// Duration returns the duration of the snapshot.
+func (m *ManifestWithError) Duration() time.Duration {
+	return time.Duration(m.EndTime - m.StartTime)
+}
+
+// StatusCode returns the status code of the manifest.
+func (m *ManifestWithError) StatusCode() string {
+	if m.Error != "" {
+		return "fatal"
+	}
+
+	if m.Manifest.IncompleteReason != "" {
+		return "incomplete"
+	}
+
+	if m.Manifest.RootEntry != nil && m.Manifest.RootEntry.DirSummary != nil {
+		if m.Manifest.RootEntry.DirSummary.FatalErrorCount > 0 {
+			return "fatal"
+		}
+
+		if m.Manifest.RootEntry.DirSummary.IgnoredErrorCount > 0 {
+			return "error"
+		}
+	}
+
+	return "ok"
+}
+
+// MultiSnapshotStatus represents the status of multiple snapshots.
+type MultiSnapshotStatus struct {
+	Snapshots []*ManifestWithError `json:"snapshots"`
+}
diff --git a/notification/notifytemplate/embeddedtemplate.go b/notification/notifytemplate/embeddedtemplate.go
index 9e3841f8..0c24cb12 100644
--- a/notification/notifytemplate/embeddedtemplate.go
+++ b/notification/notifytemplate/embeddedtemplate.go
@@ -3,12 +3,15 @@
 
 import (
 	"embed"
+	"slices"
+	"sort"
 	"text/template"
 	"time"
 
 	"github.com/pkg/errors"
 
-	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/internal/units"
+	"github.com/kopia/kopia/notification/notifydata"
 )
 
 //go:embed "*.html"
@@ -20,22 +23,41 @@
 	TestNotification = "test-notification"
 )
 
-// Functions is a map of functions that can be used in templates.
+// Options provides options for template rendering.
+type Options struct {
+	Timezone   *time.Location
+	TimeFormat string
+}
+
+// functions is a map of functions that can be used in templates.
+func functions(opt Options) template.FuncMap {
+	if opt.Timezone == nil {
+		opt.Timezone = time.Local
+	}
+
+	if opt.TimeFormat == "" {
+		opt.TimeFormat = time.RFC1123Z
+	}
+
+	return template.FuncMap{
+		"bytes": units.BytesString[int64],
+		"sortSnapshotManifestsByName": func(man []*notifydata.ManifestWithError) []*notifydata.ManifestWithError {
+			res := slices.Clone(man)
+			sort.Slice(res, func(i, j int) bool {
+				return res[i].Source.String() < res[j].Source.String()
+			})
+			return res
+		},
+		"formatTime": func(t time.Time) string {
+			return t.In(opt.Timezone).Format(opt.TimeFormat)
+		},
+	}
+}
+
+// DefaultOptions is the default set of options.
 //
 //nolint:gochecknoglobals
-var Functions = template.FuncMap{
-	"toTime": func(t any) time.Time {
-		if t, ok := t.(time.Time); ok {
-			return t
-		}
-
-		if t, ok := t.(fs.UTCTimestamp); ok {
-			return t.ToTime()
-		}
-
-		return time.Time{}
-	},
-}
+var DefaultOptions = Options{}
 
 // GetEmbeddedTemplate returns embedded template by name.
 func GetEmbeddedTemplate(templateName string) (string, error) {
@@ -61,7 +83,7 @@ func SupportedTemplates() []string {
 }
 
 // ParseTemplate parses a named template.
-func ParseTemplate(tmpl string) (*template.Template, error) {
+func ParseTemplate(tmpl string, opt Options) (*template.Template, error) {
 	//nolint:wrapcheck
-	return template.New("template").Funcs(Functions).Parse(tmpl)
+	return template.New("template").Funcs(functions(opt)).Parse(tmpl)
 }
diff --git a/notification/notifytemplate/generic-error.html b/notification/notifytemplate/generic-error.html
new file mode 100644
index 00000000..7287c11a
--- /dev/null
+++ b/notification/notifytemplate/generic-error.html
@@ -0,0 +1,20 @@
+Subject: Kopia has encountered an error during {{ .EventArgs.Operation }} on {{.Hostname}}
+
+<!doctype html>
+<html>
+<head>
+</head>
+<body>
+
+<p><b>Operation:</b> {{ .EventArgs.OperationDetails }}</p>
+<p><b>Started:</b> {{ .EventArgs.StartTimestamp | formatTime }}</p>
+<p><b>Finished:</b> {{ .EventArgs.EndTimestamp | formatTime }} ({{ .EventArgs.Duration }})</p>
+
+<p><b>Message:</b> {{ .EventArgs.ErrorMessage }}</p>
+
+<pre>{{ .EventArgs.ErrorDetails }}</pre>
+
+<p>Generated at {{ .EventTime | formatTime }} by <a href="https://kopia.io">Kopia {{ .KopiaBuildVersion }}</a>.</p>
+
+</body>
+</html>
diff --git a/notification/notifytemplate/generic-error.txt b/notification/notifytemplate/generic-error.txt
new file mode 100644
index 00000000..eea8ace7
--- /dev/null
+++ b/notification/notifytemplate/generic-error.txt
@@ -0,0 +1,11 @@
+Subject: Kopia has encountered an error during {{ .EventArgs.Operation }} on {{.Hostname}}
+
+Operation: {{ .EventArgs.OperationDetails }}
+Started:   {{ .EventArgs.StartTimestamp | formatTime }}
+Finished:  {{ .EventArgs.EndTimestamp | formatTime }} ({{ .EventArgs.Duration }})
+
+{{ .EventArgs.ErrorDetails }}
+
+Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}.
+
+https://kopia.io/
\ No newline at end of file
diff --git a/notification/notifytemplate/snapshot-report.html b/notification/notifytemplate/snapshot-report.html
new file mode 100644
index 00000000..272d4330
--- /dev/null
+++ b/notification/notifytemplate/snapshot-report.html
@@ -0,0 +1,103 @@
+Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}}
+
+<!doctype html>
+<html>
+<head>
+<style type="text/css">
+    table {
+        width: 100%;
+        border-collapse: collapse;
+    }
+
+    thead tr {
+        background-color: #f2f2f2;
+    }
+
+    th, td {
+        border: 1px solid black;
+        padding: 2px;
+        text-align: left;
+    }
+
+    th {
+        background-color: #f2f2f2;
+    }
+
+    code {
+        background-color: #f2f2f2;
+        padding: 2px;
+    }
+
+    span.path {
+        font-family: monospace;
+        color: #344652;
+        font-weight: bold;
+    }
+
+    tr.snapshotstatus-fatal {
+        background-color: #fde9e4;
+    }
+
+    tr.snapshotstatus-error {
+        background-color: #fcffba;
+    }
+
+    tr.snapshotstatus-incomplete {
+        background-color: #8a8c7e;
+    }
+</style>
+</head>
+<body>
+<table border="1">
+<thead>
+    <tr>
+        <th>Source</th>
+        <th>Started</th>
+        <th>Duration</th>
+        <th>Total Size</th>
+        <th>Total Files</th>
+        <th>Total Directories</th>
+    </tr>
+</thead>
+{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}
+<tr class="snapshotstatus-{{ .StatusCode }}">
+<td><span class="path">{{ .Source.Path }}</span></td>
+<td>{{ .StartTimestamp | formatTime }}</td>
+<td>{{ .Duration }}</td>
+<td>{{ .TotalSize | bytes }}</td>
+<td>{{ .TotalFiles }}</td>
+<td>{{ .TotalDirs }}</td>
+</tr>
+
+{{ if .Error }}
+<tr class="snapshotstatus-{{ .StatusCode }}">
+    <td colspan="6">
+        <b style="color:red">Error:</b> {{ .Error }}
+    </td>
+</tr>
+{{ end }}
+
+{{ if .RootEntry }}
+{{ if .RootEntry.DirSummary }}
+{{ if .RootEntry.DirSummary.FailedEntries }}
+<tr class="snapshotstatus-{{ .StatusCode }}">
+    <td colspan="6">
+        <b style="color:red">Failed Entries:</b>
+        <ul>
+        {{ range .RootEntry.DirSummary.FailedEntries }}
+        <li><span class="path">{{.EntryPath}}</span>: {{.Error}}</li>
+        {{ end }}
+        </ul>
+    </td>
+</tr>
+{{ end }}
+{{ end }}
+{{ end }}
+
+{{ end }}
+</table>
+
+<p>Generated at {{ .EventTime | formatTime }} by <a href="https://kopia.io">Kopia {{ .KopiaBuildVersion }}</a>.</p>
+
+</body>
+</html>
diff --git a/notification/notifytemplate/snapshot-report.txt b/notification/notifytemplate/snapshot-report.txt
new file mode 100644
index 00000000..212c85d0
--- /dev/null
+++ b/notification/notifytemplate/snapshot-report.txt
@@ -0,0 +1,16 @@
+Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}}
+
+{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}Path: {{ .Source.Path }}
+
+  Status: {{ .StatusCode }}
+  Start: {{ .StartTimestamp | formatTime  }} Duration: {{ .Duration }}
+  Size: {{ .TotalSize | bytes }}, {{ .TotalFiles }} files, {{ .TotalDirs }} directories.
+{{ if .Error }}  Error: {{ .Error }}
+{{ end }}{{ if .RootEntry }}{{ if .RootEntry.DirSummary }}{{ if .RootEntry.DirSummary.FailedEntries }}
+  Failed Entries:
+{{ range .RootEntry.DirSummary.FailedEntries }}
+  - {{.EntryPath}}: {{.Error}}{{ end }}{{ end }}{{ end }}
+{{ end }}
+{{ end }}Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}.
+
+https://kopia.io/
\ No newline at end of file


Test patch:
diff --git a/internal/server/server_maintenance_test.go b/internal/server/server_maintenance_test.go
index cade0639..1d478549 100644
--- a/internal/server/server_maintenance_test.go
+++ b/internal/server/server_maintenance_test.go
@@ -12,6 +12,7 @@
 
 	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/internal/repotesting"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/maintenance"
 )
@@ -47,6 +48,14 @@ func (s *testServer) refreshScheduler(reason string) {
 	s.refreshSchedulerCount.Add(1)
 }
 
+func (s *testServer) enableErrorNotifications() bool {
+	return false
+}
+
+func (s *testServer) notificationTemplateOptions() notifytemplate.Options {
+	return notifytemplate.DefaultOptions
+}
+
 func TestServerMaintenance(t *testing.T) {
 	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
 
diff --git a/internal/server/server_test.go b/internal/server/server_test.go
index 36182e99..4a6a8402 100644
--- a/internal/server/server_test.go
+++ b/internal/server/server_test.go
@@ -290,7 +290,7 @@ func remoteRepositoryNotificationTest(t *testing.T, ctx context.Context, rep rep
 	}))
 	require.NoError(t, rw.Flush(ctx))
 
-	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError)
+	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError, notifytemplate.DefaultOptions)
 	require.Equal(t, int32(1), numRequestsReceived.Load())
 
 	// another webhook which fails
@@ -307,7 +307,7 @@ func remoteRepositoryNotificationTest(t *testing.T, ctx context.Context, rep rep
 	}))
 
 	require.NoError(t, rw.Flush(ctx))
-	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError)
+	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError, notifytemplate.DefaultOptions)
 	require.Equal(t, int32(1), numRequestsReceived.Load())
 }
 
diff --git a/notification/notifytemplate/notifytemplate_test.go b/notification/notifytemplate/notifytemplate_test.go
new file mode 100644
index 00000000..182c98fb
--- /dev/null
+++ b/notification/notifytemplate/notifytemplate_test.go
@@ -0,0 +1,118 @@
+package notifytemplate_test
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
+	"github.com/kopia/kopia/notification/notifytemplate"
+	"github.com/kopia/kopia/snapshot"
+)
+
+var defaultTestOptions = notifytemplate.Options{
+	Timezone: time.UTC,
+}
+
+var altTestOptions = notifytemplate.Options{
+	Timezone:   time.FixedZone("PST", -8*60*60),
+	TimeFormat: time.RFC1123,
+}
+
+func TestNotifyTemplate_generic_error(t *testing.T) {
+	args := notification.MakeTemplateArgs(&notifydata.ErrorInfo{
+		Operation:        "Some Operation",
+		OperationDetails: "Some Operation Details",
+		ErrorMessage:     "error message",
+		ErrorDetails:     "error details",
+		StartTime:        time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC),
+		EndTime:          time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC),
+	})
+
+	args.EventTime = time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC)
+	args.Hostname = "some-host"
+
+	verifyTemplate(t, "generic-error.txt", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "generic-error.html", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "generic-error.txt", ".alt", args, altTestOptions)
+	verifyTemplate(t, "generic-error.html", ".alt", args, altTestOptions)
+}
+
+func TestNotifyTemplate_snapshot_report(t *testing.T) {
+	args := notification.MakeTemplateArgs(&notifydata.MultiSnapshotStatus{
+		Snapshots: []*notifydata.ManifestWithError{
+			{
+				Manifest: snapshot.Manifest{
+					Source:    snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"},
+					StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()),
+					EndTime:   fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC).UnixNano()),
+					RootEntry: &snapshot.DirEntry{
+						DirSummary: &fs.DirectorySummary{
+							TotalFileCount: 123,
+							TotalFileSize:  456,
+							TotalDirCount:  33,
+							FailedEntries: []*fs.EntryWithError{
+								{
+									EntryPath: "/some/path",
+									Error:     "some error",
+								},
+								{
+									EntryPath: "/some/path2",
+									Error:     "some error",
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				Error: "some top-level error",
+				Manifest: snapshot.Manifest{
+					Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/other/path"},
+				},
+			},
+		},
+	})
+
+	args.EventTime = time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC)
+	args.Hostname = "some-host"
+
+	verifyTemplate(t, "snapshot-report.txt", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "snapshot-report.html", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "snapshot-report.txt", ".alt", args, altTestOptions)
+	verifyTemplate(t, "snapshot-report.html", ".alt", args, altTestOptions)
+}
+
+func verifyTemplate(t *testing.T, embeddedTemplateName, expectedSuffix string, args interface{}, opt notifytemplate.Options) {
+	t.Helper()
+
+	tmpl, err := notifytemplate.GetEmbeddedTemplate(embeddedTemplateName)
+	require.NoError(t, err)
+
+	tt, err := notifytemplate.ParseTemplate(tmpl, opt)
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+
+	require.NoError(t, tt.Execute(&buf, args))
+
+	actualFileName := filepath.Join("testdata", embeddedTemplateName+expectedSuffix+".actual")
+	require.NoError(t, os.WriteFile(actualFileName, buf.Bytes(), 0o644))
+
+	expectedFileName := filepath.Join("testdata", embeddedTemplateName+expectedSuffix+".expected")
+
+	wantBytes, err := os.ReadFile(expectedFileName)
+	require.NoError(t, err)
+
+	want := string(wantBytes)
+
+	require.Equal(t, want, buf.String())
+
+	require.NoError(t, os.Remove(actualFileName))
+}
diff --git a/notification/notifytemplate/test-notification.html b/notification/notifytemplate/test-notification.html
index d8823c65..a9aaf6ea 100644
--- a/notification/notifytemplate/test-notification.html
+++ b/notification/notifytemplate/test-notification.html
@@ -1,4 +1,4 @@
-Subject: Test notification from Kopia at {{ .EventTime }}
+Subject: Test notification from Kopia at {{ .EventTime | formatTime }}
 
 <p>This is a test notification from Kopia.</p>
 
diff --git a/notification/notifytemplate/test-notification.txt b/notification/notifytemplate/test-notification.txt
index e364e269..6624b03e 100644
--- a/notification/notifytemplate/test-notification.txt
+++ b/notification/notifytemplate/test-notification.txt
@@ -1,4 +1,4 @@
-Subject: Test notification from Kopia at {{ .EventTime }}
+Subject: Test notification from Kopia at {{ .EventTime | formatTime }}
 
 This is a test notification from Kopia.
 
diff --git a/notification/notifytemplate/testdata/.gitignore b/notification/notifytemplate/testdata/.gitignore
new file mode 100644
index 00000000..e69de29b
diff --git a/notification/notifytemplate/testdata/generic-error.html.alt.expected b/notification/notifytemplate/testdata/generic-error.html.alt.expected
new file mode 100644
index 00000000..0d8f0013
--- /dev/null
+++ b/notification/notifytemplate/testdata/generic-error.html.alt.expected
@@ -0,0 +1,20 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+<!doctype html>
+<html>
+<head>
+</head>
+<body>
+
+<p><b>Operation:</b> Some Operation Details</p>
+<p><b>Started:</b> Wed, 01 Jan 2020 19:04:05 PST</p>
+<p><b>Finished:</b> Wed, 01 Jan 2020 19:04:06 PST (1s)</p>
+
+<p><b>Message:</b> error message</p>
+
+<pre>error details</pre>
+
+<p>Generated at Wed, 01 Jan 2020 19:04:05 PST by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>
diff --git a/notification/notifytemplate/testdata/generic-error.html.default.expected b/notification/notifytemplate/testdata/generic-error.html.default.expected
new file mode 100644
index 00000000..9faf8d9d
--- /dev/null
+++ b/notification/notifytemplate/testdata/generic-error.html.default.expected
@@ -0,0 +1,20 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+<!doctype html>
+<html>
+<head>
+</head>
+<body>
+
+<p><b>Operation:</b> Some Operation Details</p>
+<p><b>Started:</b> Thu, 02 Jan 2020 03:04:05 +0000</p>
+<p><b>Finished:</b> Thu, 02 Jan 2020 03:04:06 +0000 (1s)</p>
+
+<p><b>Message:</b> error message</p>
+
+<pre>error details</pre>
+
+<p>Generated at Thu, 02 Jan 2020 03:04:05 +0000 by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>
diff --git a/notification/notifytemplate/testdata/generic-error.txt.alt.expected b/notification/notifytemplate/testdata/generic-error.txt.alt.expected
new file mode 100644
index 00000000..170cdcdc
--- /dev/null
+++ b/notification/notifytemplate/testdata/generic-error.txt.alt.expected
@@ -0,0 +1,11 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+Operation: Some Operation Details
+Started:   Wed, 01 Jan 2020 19:04:05 PST
+Finished:  Wed, 01 Jan 2020 19:04:06 PST (1s)
+
+error details
+
+Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial.
+
+https://kopia.io/
\ No newline at end of file
diff --git a/notification/notifytemplate/testdata/generic-error.txt.default.expected b/notification/notifytemplate/testdata/generic-error.txt.default.expected
new file mode 100644
index 00000000..4f03e75b
--- /dev/null
+++ b/notification/notifytemplate/testdata/generic-error.txt.default.expected
@@ -0,0 +1,11 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+Operation: Some Operation Details
+Started:   Thu, 02 Jan 2020 03:04:05 +0000
+Finished:  Thu, 02 Jan 2020 03:04:06 +0000 (1s)
+
+error details
+
+Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial.
+
+https://kopia.io/
\ No newline at end of file
diff --git a/notification/notifytemplate/testdata/snapshot-report.html.alt.expected b/notification/notifytemplate/testdata/snapshot-report.html.alt.expected
new file mode 100644
index 00000000..11647e19
--- /dev/null
+++ b/notification/notifytemplate/testdata/snapshot-report.html.alt.expected
@@ -0,0 +1,119 @@
+Subject: Kopia created 2 snapshots on some-host
+
+<!doctype html>
+<html>
+<head>
+<style type="text/css">
+    table {
+        width: 100%;
+        border-collapse: collapse;
+    }
+
+    thead tr {
+        background-color: #f2f2f2;
+    }
+
+    th, td {
+        border: 1px solid black;
+        padding: 2px;
+        text-align: left;
+    }
+
+    th {
+        background-color: #f2f2f2;
+    }
+
+    code {
+        background-color: #f2f2f2;
+        padding: 2px;
+    }
+
+    span.path {
+        font-family: monospace;
+        color: #344652;
+        font-weight: bold;
+    }
+
+    tr.snapshotstatus-fatal {
+        background-color: #fde9e4;
+    }
+
+    tr.snapshotstatus-error {
+        background-color: #fcffba;
+    }
+
+    tr.snapshotstatus-incomplete {
+        background-color: #8a8c7e;
+    }
+</style>
+</head>
+<body>
+<table border="1">
+<thead>
+    <tr>
+        <th>Source</th>
+        <th>Started</th>
+        <th>Duration</th>
+        <th>Total Size</th>
+        <th>Total Files</th>
+        <th>Total Directories</th>
+    </tr>
+</thead>
+
+<tr class="snapshotstatus-fatal">
+<td><span class="path">/some/other/path</span></td>
+<td>Wed, 31 Dec 1969 16:00:00 PST</td>
+<td>0s</td>
+<td>0 B</td>
+<td>0</td>
+<td>0</td>
+</tr>
+
+
+<tr class="snapshotstatus-fatal">
+    <td colspan="6">
+        <b style="color:red">Error:</b> some top-level error
+    </td>
+</tr>
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+<td><span class="path">/some/path</span></td>
+<td>Wed, 01 Jan 2020 19:04:05 PST</td>
+<td>1.000000001s</td>
+<td>456 B</td>
+<td>123</td>
+<td>33</td>
+</tr>
+
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+    <td colspan="6">
+        <b style="color:red">Failed Entries:</b>
+        <ul>
+        
+        <li><span class="path">/some/path</span>: some error</li>
+        
+        <li><span class="path">/some/path2</span>: some error</li>
+        
+        </ul>
+    </td>
+</tr>
+
+
+
+
+
+</table>
+
+<p>Generated at Wed, 01 Jan 2020 19:04:05 PST by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>
diff --git a/notification/notifytemplate/testdata/snapshot-report.html.default.expected b/notification/notifytemplate/testdata/snapshot-report.html.default.expected
new file mode 100644
index 00000000..5cf75fb2
--- /dev/null
+++ b/notification/notifytemplate/testdata/snapshot-report.html.default.expected
@@ -0,0 +1,119 @@
+Subject: Kopia created 2 snapshots on some-host
+
+<!doctype html>
+<html>
+<head>
+<style type="text/css">
+    table {
+        width: 100%;
+        border-collapse: collapse;
+    }
+
+    thead tr {
+        background-color: #f2f2f2;
+    }
+
+    th, td {
+        border: 1px solid black;
+        padding: 2px;
+        text-align: left;
+    }
+
+    th {
+        background-color: #f2f2f2;
+    }
+
+    code {
+        background-color: #f2f2f2;
+        padding: 2px;
+    }
+
+    span.path {
+        font-family: monospace;
+        color: #344652;
+        font-weight: bold;
+    }
+
+    tr.snapshotstatus-fatal {
+        background-color: #fde9e4;
+    }
+
+    tr.snapshotstatus-error {
+        background-color: #fcffba;
+    }
+
+    tr.snapshotstatus-incomplete {
+        background-color: #8a8c7e;
+    }
+</style>
+</head>
+<body>
+<table border="1">
+<thead>
+    <tr>
+        <th>Source</th>
+        <th>Started</th>
+        <th>Duration</th>
+        <th>Total Size</th>
+        <th>Total Files</th>
+        <th>Total Directories</th>
+    </tr>
+</thead>
+
+<tr class="snapshotstatus-fatal">
+<td><span class="path">/some/other/path</span></td>
+<td>Thu, 01 Jan 1970 00:00:00 +0000</td>
+<td>0s</td>
+<td>0 B</td>
+<td>0</td>
+<td>0</td>
+</tr>
+
+
+<tr class="snapshotstatus-fatal">
+    <td colspan="6">
+        <b style="color:red">Error:</b> some top-level error
+    </td>
+</tr>
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+<td><span class="path">/some/path</span></td>
+<td>Thu, 02 Jan 2020 03:04:05 +0000</td>
+<td>1.000000001s</td>
+<td>456 B</td>
+<td>123</td>
+<td>33</td>
+</tr>
+
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+    <td colspan="6">
+        <b style="color:red">Failed Entries:</b>
+        <ul>
+        
+        <li><span class="path">/some/path</span>: some error</li>
+        
+        <li><span class="path">/some/path2</span>: some error</li>
+        
+        </ul>
+    </td>
+</tr>
+
+
+
+
+
+</table>
+
+<p>Generated at Thu, 02 Jan 2020 03:04:05 +0000 by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>
diff --git a/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected b/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected
new file mode 100644
index 00000000..46de1240
--- /dev/null
+++ b/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected
@@ -0,0 +1,23 @@
+Subject: Kopia created 2 snapshots on some-host
+
+Path: /some/other/path
+
+  Status: fatal
+  Start: Wed, 31 Dec 1969 16:00:00 PST Duration: 0s
+  Size: 0 B, 0 files, 0 directories.
+  Error: some top-level error
+
+Path: /some/path
+
+  Status: ok
+  Start: Wed, 01 Jan 2020 19:04:05 PST Duration: 1.000000001s
+  Size: 456 B, 123 files, 33 directories.
+
+  Failed Entries:
+
+  - /some/path: some error
+  - /some/path2: some error
+
+Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial.
+
+https://kopia.io/
\ No newline at end of file
diff --git a/notification/notifytemplate/testdata/snapshot-report.txt.default.expected b/notification/notifytemplate/testdata/snapshot-report.txt.default.expected
new file mode 100644
index 00000000..b11a4e06
--- /dev/null
+++ b/notification/notifytemplate/testdata/snapshot-report.txt.default.expected
@@ -0,0 +1,23 @@
+Subject: Kopia created 2 snapshots on some-host
+
+Path: /some/other/path
+
+  Status: fatal
+  Start: Thu, 01 Jan 1970 00:00:00 +0000 Duration: 0s
+  Size: 0 B, 0 files, 0 directories.
+  Error: some top-level error
+
+Path: /some/path
+
+  Status: ok
+  Start: Thu, 02 Jan 2020 03:04:05 +0000 Duration: 1.000000001s
+  Size: 456 B, 123 files, 33 directories.
+
+  Failed Entries:
+
+  - /some/path: some error
+  - /some/path2: some error
+
+Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial.
+
+https://kopia.io/
\ No newline at end of file