Repository: cyverse-de/timelord
Pull request #17: Terminate jobs only if they're in the cluster
Author: login: johnworth
Created: 2025-04-18T17:11:52Z | Merged: 2025-04-18T21:57:09Z

Along with some minor refactoring, this PR should fix the following:
* Timelord attempting to terminate a job repeatedly because it's not in the cluster though the database says it's running.
* Timelord attempting to send a notification repeatedly when it attempts to terminate a job that isn't in the cluster.

Timelord should now use the `GET /vice/admin/listing` endpoint in `app-exposer` to get a list of the analyses running in the cluster. If the analysis to be terminated is not listed by that endpoint, then timelord will use the `POST /{uuid}/status` in `job-status-listener` to set the status of the job to `Completed`.

Timelord has also been changed to not send a notification if the database is out of sync as described above. This should prevent it from spamming users endlessly.


Diff:
diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml
index ca4a48e..f8e7e42 100644
--- a/.github/workflows/golangci-lint.yml
+++ b/.github/workflows/golangci-lint.yml
@@ -8,6 +8,6 @@ on:
 
 jobs:
   call-workflow-passing-data:
-    uses: cyverse-de/github-workflows/.github/workflows/golangci-lint.yml@v0.0.4
+    uses: cyverse-de/github-workflows/.github/workflows/golangci-lint.yml@v0.2.0
     with:
-      go-version: 1.21
+      go-version: 1.23
diff --git a/analyses.go b/analyses.go
index 8e7490b..98b108a 100644
--- a/analyses.go
+++ b/analyses.go
@@ -6,11 +6,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
 	"net/url"
-	"path/filepath"
-	"strings"
 	"time"
 
 	"github.com/cyverse-de/messaging/v9"
@@ -51,6 +47,8 @@ type Job struct {
 	PeriodicPeriod int    `json:"periodic_period"`
 }
 
+type Analysis = Job
+
 func (j *Job) accessURL() (string, error) {
 	if VICEURI == "" {
 		return "", nil
@@ -93,7 +91,17 @@ func getRemainingDuration(j *Job) (string, error) {
 	return fmt.Sprintf("%d:%02d", h, m), nil
 }
 
-func jobFromRow(ctx context.Context, dedb *sql.DB, rows *sql.Rows) (Job, error) {
+type TimelordAnalyses struct {
+	dedb *sql.DB
+}
+
+func NewTimelordAnalyses(dedb *sql.DB) *TimelordAnalyses {
+	return &TimelordAnalyses{
+		dedb,
+	}
+}
+
+func (a *TimelordAnalyses) jobFromRow(ctx context.Context, rows *sql.Rows) (Job, error) {
 	var (
 		err            error
 		job            Job
@@ -135,7 +143,7 @@ func jobFromRow(ctx context.Context, dedb *sql.DB, rows *sql.Rows) (Job, error)
 		job.StartDate = startDate.Time.Format(TimestampFromDBFormat)
 	}
 
-	job.ExternalID, err = getExternalID(ctx, dedb, job.ID)
+	job.ExternalID, err = a.getExternalID(ctx, job.ID)
 	if err != nil {
 		return job, err
 	}
@@ -148,14 +156,14 @@ select job_steps.external_id
  where job_steps.job_id = $1
  limit 1`
 
-func getExternalID(ctx context.Context, dedb *sql.DB, jobID string) (string, error) {
+func (a *TimelordAnalyses) getExternalID(ctx context.Context, jobID string) (string, error) {
 	var (
 		err        error
 		row        *sql.Row
 		externalID string
 	)
 
-	row = dedb.QueryRowContext(
+	row = a.dedb.QueryRowContext(
 		ctx,
 		externalIDsQuery,
 		jobID,
@@ -190,13 +198,13 @@ select jobs.id,
 
 // JobsToKill returns a list of running jobs that are past their expiration date
 // and can be killed off. 'api' should be the base URL for the analyses service.
-func JobsToKill(ctx context.Context, dedb *sql.DB) ([]Job, error) {
+func (a *TimelordAnalyses) JobsToKill(ctx context.Context) ([]Job, error) {
 	var (
 		err  error
 		rows *sql.Rows
 	)
 
-	if rows, err = dedb.QueryContext(
+	if rows, err = a.dedb.QueryContext(
 		ctx,
 		jobsToKillQuery,
 		"Running",
@@ -209,7 +217,7 @@ func JobsToKill(ctx context.Context, dedb *sql.DB) ([]Job, error) {
 	jobs := []Job{}
 
 	for rows.Next() {
-		job, err := jobFromRow(ctx, dedb, rows)
+		job, err := a.jobFromRow(ctx, rows)
 		if err != nil {
 			return nil, err
 		}
@@ -250,13 +258,13 @@ SELECT jobs.id,
 `
 
 // JobPeriodicWarnings returns a list of running jobs that may need periodic notifications to be sent
-func JobPeriodicWarnings(ctx context.Context, dedb *sql.DB) ([]Job, error) {
+func (a *TimelordAnalyses) JobPeriodicWarnings(ctx context.Context) ([]Job, error) {
 	var (
 		err  error
 		rows *sql.Rows
 	)
 
-	if rows, err = dedb.QueryContext(
+	if rows, err = a.dedb.QueryContext(
 		ctx,
 		periodicWarningsQuery,
 		"Running",
@@ -268,7 +276,7 @@ func JobPeriodicWarnings(ctx context.Context, dedb *sql.DB) ([]Job, error) {
 	jobs := []Job{}
 
 	for rows.Next() {
-		job, err := jobFromRow(ctx, dedb, rows)
+		job, err := a.jobFromRow(ctx, rows)
 		if err != nil {
 			return nil, err
 		}
@@ -309,7 +317,7 @@ select jobs.id,
 // JobKillWarnings returns a list of running jobs that are set to be killed
 // within the number of minutes specified. 'api' should be the base URL for the
 // analyses service.
-func JobKillWarnings(ctx context.Context, dedb *sql.DB, minutes int64) ([]Job, error) {
+func (a *TimelordAnalyses) JobKillWarnings(ctx context.Context, minutes int64) ([]Job, error) {
 	var (
 		err  error
 		rows *sql.Rows
@@ -320,7 +328,7 @@ func JobKillWarnings(ctx context.Context, dedb *sql.DB, minutes int64) ([]Job, e
 	// nowtimestamp := now.Format(fmtstring)
 	// futuretimestamp := now.Add(time.Duration(minutes) * time.Minute).Format(fmtstring)
 
-	if rows, err = dedb.QueryContext(
+	if rows, err = a.dedb.QueryContext(
 		ctx,
 		jobWarningsQuery,
 		"Running",
@@ -334,7 +342,7 @@ func JobKillWarnings(ctx context.Context, dedb *sql.DB, minutes int64) ([]Job, e
 	jobs := []Job{}
 
 	for rows.Next() {
-		job, err := jobFromRow(ctx, dedb, rows)
+		job, err := a.jobFromRow(ctx, rows)
 		if err != nil {
 			return nil, err
 		}
@@ -349,116 +357,6 @@ func JobKillWarnings(ctx context.Context, dedb *sql.DB, minutes int64) ([]Job, e
 	return jobs, nil
 }
 
-// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
-type JobKiller struct {
-	K8sEnabled     bool   // whether or not the VICE apps are running k8s
-	AppsBase       string // base URL for the apps service
-	AppExposerBase string // base URL for the app-exposer serivce
-}
-
-// KillJob uses either the apps or app-exposer APIs to kill a VICE job.
-func (j *JobKiller) KillJob(ctx context.Context, dedb *sql.DB, job *Job) error {
-	if j.K8sEnabled {
-		return j.killK8sJob(ctx, dedb, job)
-	}
-	return j.killCondorJob(ctx, job.ID, job.User)
-
-}
-
-// killCondorJob uses the provided API at the base URL to kill a running job. This
-// will probably be to the apps service. jobID should be the UUID for the Job,
-// typically returned in the ID field by the analyses service. The username
-// should be the short username for the user that launched the job.
-func (j *JobKiller) killCondorJob(ctx context.Context, jobID, username string) error {
-	apiURL, err := url.Parse(j.AppsBase)
-	if err != nil {
-		return err
-	}
-
-	apiURL.Path = filepath.Join(apiURL.Path, "analyses", jobID, "stop")
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL.String(), nil)
-	if err != nil {
-		return err
-	}
-
-	var shortusername string
-	userparts := strings.Split(username, "@")
-	if len(userparts) > 1 {
-		shortusername = userparts[0]
-	} else {
-		shortusername = username
-	}
-	q := req.URL.Query()
-	q.Add("user", shortusername)
-	req.URL.RawQuery = q.Encode()
-
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return fmt.Errorf("response status code for POST %s was %d as %s", apiURL.String(), resp.StatusCode, username)
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	log.Infof("response from %s was: %s", req.URL, string(body))
-	return nil
-}
-
-// killK8sJob uses the app-exposer API to make a job save its outputs and exit.
-// JobID should be the external_id (AKA invocationID) for the job.
-func (j *JobKiller) killK8sJob(ctx context.Context, dedb *sql.DB, job *Job) error {
-	var err error
-
-	origAPIURL, err := url.Parse(j.AppExposerBase)
-	if err != nil {
-		return err
-	}
-
-	externalID := job.ExternalID
-
-	var apiURL *url.URL
-	apiURL, err = url.Parse(origAPIURL.String()) // lol
-	if err != nil {
-		return errors.Wrapf(err, "error parsing URL %s while processing external-id %s", origAPIURL.String(), externalID)
-	}
-
-	apiURL.Path = filepath.Join(apiURL.Path, "vice", externalID, "save-and-exit")
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL.String(), nil)
-	if err != nil {
-		return errors.Wrapf(err, "error creating save-and-exit request for external-id %s", externalID)
-	}
-
-	resp, err := httpClient.Do(req)
-	if err != nil {
-		return errors.Wrapf(err, "error calling save-and-exit for external-id %s", externalID)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return fmt.Errorf("response status code for POST %s was %d", apiURL.String(), resp.StatusCode)
-	}
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return errors.Wrapf(err, "error reading response body of save-and-exit call for external-id %s", externalID)
-	}
-
-	log.Infof("response from %s was: %s", req.URL, string(body))
-
-	resp.Body.Close()
-
-	return nil
-}
-
 const jobByExternalIDQuery = `
 select jobs.id,
        jobs.app_id,
@@ -481,7 +379,7 @@ select jobs.id,
   join job_steps on jobs.id = job_steps.job_id
  where job_steps.external_id = $1`
 
-func lookupByExternalID(ctx context.Context, dedb *sql.DB, externalID string) (*Job, error) {
+func (a *TimelordAnalyses) lookupByExternalID(ctx context.Context, externalID string) (*Job, error) {
 	var (
 		err            error
 		job            *Job
@@ -492,7 +390,7 @@ func lookupByExternalID(ctx context.Context, dedb *sql.DB, externalID string) (*
 
 	job = &Job{}
 
-	if err = dedb.QueryRowContext(ctx, jobByExternalIDQuery, externalID).Scan(
+	if err = a.dedb.QueryRowContext(ctx, jobByExternalIDQuery, externalID).Scan(
 		&job.ID,
 		&job.AppID,
 		&job.UserID,
@@ -530,10 +428,10 @@ func generateSubdomain(userID, externalID string) string {
 
 const setSubdomainMutation = `update only jobs set subdomain = $1 where id = $2`
 
-func setSubdomain(ctx context.Context, dedb *sql.DB, analysisID, subdomain string) error {
+func (a *TimelordAnalyses) setSubdomain(ctx context.Context, analysisID, subdomain string) error {
 	var err error
 
-	if _, err = dedb.ExecContext(ctx, setSubdomainMutation, subdomain, analysisID); err != nil {
+	if _, err = a.dedb.ExecContext(ctx, setSubdomainMutation, subdomain, analysisID); err != nil {
 		return errors.Wrapf(err, "error setting subdomain for job %s to %s", analysisID, subdomain)
 	}
 
@@ -544,13 +442,13 @@ const setPlannedEndDateMutation = `update only jobs set planned_end_date = $1 wh
 
 // setPlannedEndDate takes in context, db, a job ID, and a number of milliseconds since the epoch and sets that value as the planned end date for that job
 // previously, we were passing around semi-mangled timestamps, and needed to correct for having read in a local timestamp as a UTC one. This should no longer be necessary, but is noted here in case bugs crop up
-func setPlannedEndDate(ctx context.Context, dedb *sql.DB, id string, millisSinceEpoch int64) error {
+func (a *TimelordAnalyses) setPlannedEndDate(ctx context.Context, id string, millisSinceEpoch int64) error {
 	var err error
 
 	plannedEndDate := time.UnixMilli(millisSinceEpoch).
 		Format("2006-01-02 15:04:05.000000-07")
 
-	if _, err = dedb.ExecContext(ctx, setPlannedEndDateMutation, plannedEndDate, id); err != nil {
+	if _, err = a.dedb.ExecContext(ctx, setPlannedEndDateMutation, plannedEndDate, id); err != nil {
 		return errors.Wrapf(err, "error setting planned_end_date to %s for job %s", plannedEndDate, id)
 	}
 
@@ -566,14 +464,14 @@ SELECT t.name
     ON s.job_type_id = t.id
  WHERE j.id = $1`
 
-func isInteractive(ctx context.Context, dedb *sql.DB, id string) (bool, error) {
+func (a *TimelordAnalyses) isInteractive(ctx context.Context, id string) (bool, error) {
 	var (
 		err      error
 		rows     *sql.Rows
 		jobTypes []string
 	)
 
-	if rows, err = dedb.QueryContext(ctx, stepTypeQuery, id); err != nil {
+	if rows, err = a.dedb.QueryContext(ctx, stepTypeQuery, id); err != nil {
 		return false, err
 	}
 	defer rows.Close()
@@ -603,12 +501,12 @@ SELECT user_id
  WHERE id = $1
 `
 
-func getUserIDForJob(ctx context.Context, dedb *sql.DB, analysisID string) (string, error) {
+func (a *TimelordAnalyses) getUserIDForJob(ctx context.Context, analysisID string) (string, error) {
 	var (
 		err    error
 		userID string
 	)
-	if err = dedb.QueryRowContext(ctx, getUserIDQuery, analysisID).Scan(&userID); err != nil {
+	if err = a.dedb.QueryRowContext(ctx, getUserIDQuery, analysisID).Scan(&userID); err != nil {
 		return "", err
 	}
 	return userID, nil
@@ -625,22 +523,22 @@ SELECT sum(CASE WHEN tools.time_limit_seconds > 0 THEN tools.time_limit_seconds
  WHERE jobs.id = $1
 `
 
-func getTimeLimit(ctx context.Context, dedb *sql.DB, analysisID string) (int64, error) {
+func (a *TimelordAnalyses) getTimeLimit(ctx context.Context, analysisID string) (int64, error) {
 	var (
 		err              error
 		timeLimitSeconds int64
 	)
-	if err = dedb.QueryRowContext(ctx, getTimeLimitQuery, analysisID).Scan(&timeLimitSeconds); err != nil {
+	if err = a.dedb.QueryRowContext(ctx, getTimeLimitQuery, analysisID).Scan(&timeLimitSeconds); err != nil {
 		return 0, err
 	}
 	return timeLimitSeconds, nil
 }
 
 // EnsureSubdomain makes sure the provided job has a subdomain set in the DB, returning it
-func EnsureSubdomain(ctx context.Context, dedb *sql.DB, analysis *Job) (string, error) {
+func (a *TimelordAnalyses) EnsureSubdomain(ctx context.Context, analysis *Job) (string, error) {
 	if analysis.Subdomain == "" {
 		// make sure to use analysis.ID, not external ID here.
-		userID, err := getUserIDForJob(ctx, dedb, analysis.ID)
+		userID, err := a.getUserIDForJob(ctx, analysis.ID)
 		if err != nil {
 			return "", errors.Wrapf(err, "error getting userID for job %s", analysis.ID)
 		} else {
@@ -651,7 +549,7 @@ func EnsureSubdomain(ctx context.Context, dedb *sql.DB, analysis *Job) (string,
 
 			log.Infof("generated subdomain for analysis %s is %s, based on user ID %s and invocation ID %s", analysis.ID, subdomain, userID, analysis.ExternalID)
 
-			if err = setSubdomain(ctx, dedb, analysis.ID, subdomain); err != nil {
+			if err = a.setSubdomain(ctx, analysis.ID, subdomain); err != nil {
 				return "", errors.Wrapf(err, "error setting subdomain for analysis '%s' to '%s'", analysis.ID, subdomain)
 			}
 			return subdomain, nil
@@ -661,7 +559,7 @@ func EnsureSubdomain(ctx context.Context, dedb *sql.DB, analysis *Job) (string,
 	}
 }
 
-func EnsurePlannedEndDate(ctx context.Context, dedb *sql.DB, analysis *Job) error {
+func (a *TimelordAnalyses) EnsurePlannedEndDate(ctx context.Context, analysis *Job) error {
 	// Check to see if the planned_end_date is set for the analysis
 	if analysis.PlannedEndDate != "" {
 		log.Infof("planned end date for %s is set to %s, nothing to do", analysis.ID, analysis.PlannedEndDate)
@@ -674,7 +572,7 @@ func EnsurePlannedEndDate(ctx context.Context, dedb *sql.DB, analysis *Job) erro
 	}
 	sdnano := startDate.UnixNano()
 
-	timeLimitSeconds, err := getTimeLimit(ctx, dedb, analysis.ID)
+	timeLimitSeconds, err := a.getTimeLimit(ctx, analysis.ID)
 	if err != nil {
 		return errors.Wrapf(err, "error fetching time limit for analysis %s", analysis.ID)
 	}
@@ -682,7 +580,7 @@ func EnsurePlannedEndDate(ctx context.Context, dedb *sql.DB, analysis *Job) erro
 	// StartDate is in milliseconds, so convert it to nanoseconds, add correct number of seconds,
 	// then convert back to milliseconds.
 	endDate := time.Unix(0, sdnano).Add(time.Duration(timeLimitSeconds)*time.Second).UnixNano() / 1000000
-	if err = setPlannedEndDate(ctx, dedb, analysis.ID, endDate); err != nil {
+	if err = a.setPlannedEndDate(ctx, analysis.ID, endDate); err != nil {
 		return errors.Wrapf(err, "error setting planned end date for analysis '%s' to '%d'", analysis.ID, endDate)
 	}
 	return nil
@@ -691,7 +589,7 @@ func EnsurePlannedEndDate(ctx context.Context, dedb *sql.DB, analysis *Job) erro
 // CreateMessageHandler returns a function that can be used by the messaging
 // package to handle job status messages. The handler will set the planned
 // end date for an analysis if it's not already set.
-func CreateMessageHandler(dedb *sql.DB) func(context.Context, amqp.Delivery) {
+func (a *TimelordAnalyses) CreateMessageHandler() func(context.Context, amqp.Delivery) {
 	return func(ctx context.Context, delivery amqp.Delivery) {
 		var err error
 		msgLog := log.WithFields(log.Fields{"context": "message handler"})
@@ -715,14 +613,14 @@ func CreateMessageHandler(dedb *sql.DB) func(context.Context, amqp.Delivery) {
 		externalID = update.Job.InvocationID
 		msgLog = msgLog.WithFields(log.Fields{"externalID": externalID})
 
-		analysis, err := lookupByExternalID(ctx, dedb, externalID)
+		analysis, err := a.lookupByExternalID(ctx, externalID)
 		if err != nil {
 			msgLog.Error(errors.Wrapf(err, "error looking up analysis by external ID '%s'", externalID))
 			return
 		}
 		msgLog = msgLog.WithFields(log.Fields{"ID": analysis.ID})
 
-		analysisIsInteractive, err := isInteractive(ctx, dedb, analysis.ID)
+		analysisIsInteractive, err := a.isInteractive(ctx, analysis.ID)
 		if err != nil {
 			msgLog.Error(errors.Wrapf(err, "error looking up interactive status for analysis %s", analysis.ID))
 			return
@@ -741,13 +639,13 @@ func CreateMessageHandler(dedb *sql.DB) func(context.Context, amqp.Delivery) {
 		msgLog.Infof("job status update for %s was %s", analysis.ID, update.State)
 
 		// Set the subdomain
-		subdomain, err := EnsureSubdomain(ctx, dedb, analysis)
+		subdomain, err := a.EnsureSubdomain(ctx, analysis)
 		if err != nil {
 			msgLog.Error(errors.Wrap(err, "error ensuring subdomain for analysis"))
 		}
 		msgLog = msgLog.WithFields(log.Fields{"subdomain": subdomain})
 
-		err = EnsurePlannedEndDate(ctx, dedb, analysis)
+		err = a.EnsurePlannedEndDate(ctx, analysis)
 		if err != nil {
 			msgLog.Error(errors.Wrap(err, "error ensuring planned end date for analysis"))
 		}
diff --git a/analysis_termination.go b/analysis_termination.go
new file mode 100644
index 0000000..bf82f7b
--- /dev/null
+++ b/analysis_termination.go
@@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // client for the app-exposer service
+	dedb       *sql.DB
+	jslURL     *url.URL // the URL for the job-status-listener service
+}
+
+func NewJobKiller(k8sEnabled bool, appsBase, aeBase string, dedb *sql.DB, jslURL *url.URL) (*JobKiller, error) {
+	appExposer, err := NewAppExposer(aeBase)
+	if err != nil {
+		return nil, err
+	}
+	return &JobKiller{
+		K8sEnabled: k8sEnabled,
+		AppsBase:   appsBase,
+		appExposer: appExposer,
+		dedb:       dedb,
+		jslURL:     jslURL,
+	}, nil
+}
+
+// KillJob uses either the apps or app-exposer APIs to kill a VICE job.
+func (j *JobKiller) KillJob(ctx context.Context, job *Job) error {
+	if j.K8sEnabled {
+		return j.killK8sJob(ctx, job)
+	}
+	return j.killBatchAnalysis(ctx, job.ID, job.User)
+
+}
+
+// killBatchAnalysis uses the provided API at the base URL to kill a running job. This
+// will probably be to the apps service. jobID should be the UUID for the Job,
+// typically returned in the ID field by the analyses service. The username
+// should be the short username for the user that launched the job.
+func (j *JobKiller) killBatchAnalysis(ctx context.Context, jobID, username string) error {
+	return j.appExposer.StopAnalyses(ctx, jobID, username)
+}
+
+func (j *JobKiller) requestAnalysisTermination(ctx context.Context, job *Job) error {
+	return j.appExposer.VICESaveAndExit(ctx, job)
+}
+
+func (j *JobKiller) sendCompletedStatus(ctx context.Context, job *Job) error {
+	updateEndpoint := *j.jslURL
+	updateEndpoint.Path = path.Join(updateEndpoint.Path, job.ExternalID, "status")
+
+	postBody := map[string]string{
+		"Hostname": "timelord",
+		"Message":  "Set state to Completed",
+		"State":    "Completed",
+	}
+
+	postJSON, err := json.Marshal(postBody)
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, updateEndpoint.String(), bytes.NewBuffer(postJSON))
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return fmt.Errorf("status code from job-status-listener was %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+// checkForAnalysisInCluster returns whether the analysis is present in cluster in some
+// state. Calls out to app-exposer.
+func (j *JobKiller) checkForAnalysisInCluster(ctx context.Context, analysis *Analysis) (bool, error) {
+	var (
+		err   error
+		found bool
+	)
+
+	listing, err := j.appExposer.AdminListing(ctx, analysis)
+	if err != nil {
+		return found, err
+	}
+
+	if len(listing.Deployments) > 0 {
+		found = true
+	}
+
+	return found, nil
+}
+
+// killK8sJob uses the app-exposer API to make a job save its outputs and exit.
+// JobID should be the external_id (AKA invocationID) for the job.
+func (j *JobKiller) killK8sJob(ctx context.Context, job *Job) error {
+	return j.requestAnalysisTermination(ctx, job)
+}
diff --git a/app_exposer.go b/app_exposer.go
new file mode 100644
index 0000000..dd6e9bd
--- /dev/null
+++ b/app_exposer.go
@@ -0,0 +1,217 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+
+	corev1 "k8s.io/api/core/v1"
+	netv1 "k8s.io/api/networking/v1"
+)
+
+// MetaInfo contains useful information provided by multiple resource types.
+type MetaInfo struct {
+	Name              string `json:"name"`
+	Namespace         string `json:"namespace"`
+	AnalysisName      string `json:"analysisName"`
+	AppName           string `json:"appName"`
+	AppID             string `json:"appID"`
+	ExternalID        string `json:"externalID"`
+	UserID            string `json:"userID"`
+	Username          string `json:"username"`
+	CreationTimestamp string `json:"creationTimestamp"`
+}
+
+// DeploymentInfo contains information returned about a Deployment.
+type DeploymentInfo struct {
+	MetaInfo
+	Image   string   `json:"image"`
+	Command []string `json:"command"`
+	Port    int32    `json:"port"`
+	User    int64    `json:"user"`
+	Group   int64    `json:"group"`
+}
+
+// PodInfo tracks information about the pods for a VICE analysis.
+type PodInfo struct {
+	MetaInfo
+	Phase                 string                   `json:"phase"`
+	Message               string                   `json:"message"`
+	Reason                string                   `json:"reason"`
+	ContainerStatuses     []corev1.ContainerStatus `json:"containerStatuses"`
+	InitContainerStatuses []corev1.ContainerStatus `json:"initContainerStatuses"`
+}
+
+// ConfigMapInfo contains useful info about a config map.
+type ConfigMapInfo struct {
+	MetaInfo
+	Data map[string]string `json:"data"`
+}
+
+// ServiceInfoPort contains information about a service's Port.
+type ServiceInfoPort struct {
+	Name           string `json:"name"`
+	NodePort       int32  `json:"nodePort"`
+	TargetPort     int32  `json:"targetPort"`
+	TargetPortName string `json:"targetPortName"`
+	Port           int32  `json:"port"`
+	Protocol       string `json:"protocol"`
+}
+
+// ServiceInfo contains info about a service
+type ServiceInfo struct {
+	MetaInfo
+	Ports []ServiceInfoPort `json:"ports"`
+}
+
+// IngressInfo contains useful Ingress VICE info.
+type IngressInfo struct {
+	MetaInfo
+	DefaultBackend string              `json:"defaultBackend"`
+	Rules          []netv1.IngressRule `json:"rules"`
+}
+
+// ResourceInfo contains all of the k8s resource information about a running VICE analysis
+// that we know of and care about.
+type ResourceInfo struct {
+	Deployments []DeploymentInfo `json:"deployments"`
+	Pods        []PodInfo        `json:"pods"`
+	ConfigMaps  []ConfigMapInfo  `json:"configMaps"`
+	Services    []ServiceInfo    `json:"services"`
+	Ingresses   []IngressInfo    `json:"ingresses"`
+}
+
+type AppExposer struct {
+	baseURL *url.URL
+}
+
+// NewAppExposer returns a newly created *AppExposer
+func NewAppExposer(baseURL string) (*AppExposer, error) {
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, err
+	}
+
+	ae := &AppExposer{
+		baseURL: u,
+	}
+
+	return ae, nil
+}
+
+func (ae *AppExposer) StopAnalyses(ctx context.Context, jobID, username string) error {
+	callURL := *ae.baseURL
+
+	callURL.Path = filepath.Join(callURL.Path, "analyses", jobID, "stop")
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, callURL.String(), nil)
+	if err != nil {
+		return err
+	}
+
+	var shortusername string
+	userparts := strings.Split(username, "@")
+	if len(userparts) > 1 {
+		shortusername = userparts[0]
+	} else {
+		shortusername = username
+	}
+	q := req.URL.Query()
+	q.Add("user", shortusername)
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return fmt.Errorf("response status code for POST %s was %d as %s", callURL.String(), resp.StatusCode, username)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("response from %s was: %s", req.URL, string(body))
+	return nil
+}
+
+func (ae *AppExposer) VICESaveAndExit(ctx context.Context, analysis *Analysis) error {
+	var err error
+
+	externalID := analysis.ExternalID
+
+	apiURL := *ae.baseURL
+	apiURL.Path = filepath.Join(apiURL.Path, "vice", externalID, "save-and-exit")
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL.String(), nil)
+	if err != nil {
+		return errors.Wrapf(err, "error creating save-and-exit request for external-id %s", externalID)
+	}
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return errors.Wrapf(err, "error calling save-and-exit for external-id %s", externalID)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return fmt.Errorf("response status code for POST %s was %d", apiURL.String(), resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return errors.Wrapf(err, "error reading response body of save-and-exit call for external-id %s", externalID)
+	}
+
+	log.Infof("response from %s was: %s", req.URL, string(body))
+
+	resp.Body.Close()
+
+	return nil
+}
+
+func (ae *AppExposer) AdminListing(ctx context.Context, analysis *Analysis) (*ResourceInfo, error) {
+	var (
+		err error
+		req *http.Request
+	)
+
+	api := *ae.baseURL
+
+	api.Path = filepath.Join(api.Path, "vice", "admin", "listing")
+
+	req, err = http.NewRequestWithContext(ctx, http.MethodGet, api.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	q := req.URL.Query()
+	q.Set("external-id", analysis.ExternalID)
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return nil, fmt.Errorf("status code in response was %d", resp.StatusCode)
+	}
+
+	var resourceInfo ResourceInfo
+	err = json.NewDecoder(resp.Body).Decode(&resourceInfo)
+	return &resourceInfo, err
+}
diff --git a/go.mod b/go.mod
index fbf0cc7..3f53051 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,8 @@
 module github.com/cyverse-de/timelord
 
-go 1.21
+go 1.23.0
+
+toolchain go1.23.7
 
 require (
 	github.com/cyverse-de/configurate v0.0.0-20190318152107-8f767cb828d9
@@ -15,30 +17,49 @@ require (
 	github.com/streadway/amqp v1.0.1-0.20200716223359-e6b33f460591
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.31.0
 	go.opentelemetry.io/otel v1.6.3
+	k8s.io/api v0.32.3
 )
 
 require (
 	github.com/cyverse-de/model/v6 v6.0.1 // indirect
 	github.com/felixge/httpsnoop v1.0.2 // indirect
 	github.com/fsnotify/fsnotify v1.4.7 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.5.0 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cast v1.3.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.12 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/otel/exporters/jaeger v1.6.1 // indirect
 	go.opentelemetry.io/otel/metric v0.29.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.6.1 // indirect
 	go.opentelemetry.io/otel/trace v1.6.3 // indirect
-	golang.org/x/sys v0.1.0 // indirect
-	golang.org/x/text v0.3.8 // indirect
+	golang.org/x/crypto v0.37.0 // indirect
+	golang.org/x/net v0.39.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
+	golang.org/x/term v0.31.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.2.4 // indirect
+	k8s.io/apimachinery v0.32.3 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )
diff --git a/go.sum b/go.sum
index c0bfa2b..eddb797 100644
--- a/go.sum
+++ b/go.sum
@@ -14,6 +14,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyverse-de/configurate v0.0.0-20190318152107-8f767cb828d9 h1:GXDamvpoo7b4k+cEk8ZA06HhHnz86wiJod5Q3zZO9Tg=
 github.com/cyverse-de/configurate v0.0.0-20190318152107-8f767cb828d9/go.mod h1:QMZ4G8bX5f0vKiH9+/2JqV687mN1byJ18tjZwIJIagI=
 github.com/cyverse-de/dbutil v1.0.1 h1:aCfckMIIJcPGZw9kJ5a1sJSji03/swkCsC7iwD5cX9A=
@@ -25,26 +26,32 @@ github.com/cyverse-de/messaging/v9 v9.1.5/go.mod h1:bGZRCqilJTPBmWrgsyuupi3KRgkj
 github.com/cyverse-de/model/v6 v6.0.1 h1:6c0Tkl0hIbaJj7paI872QAPyvX9bhZNH93o37zDZG0s=
 github.com/cyverse-de/model/v6 v6.0.1/go.mod h1:u9qbFn7B6/1HlwwuyVriIxfGy2hh5t9vFB4+k61X7YE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
 github.com/felixge/httpsnoop v1.0.2/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -52,10 +59,15 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
 github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -63,17 +75,22 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
 github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -82,6 +99,11 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -90,8 +112,9 @@ github.com/pelletier/go-toml v1.5.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -102,6 +125,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/sanyokbig/pqinterval v1.1.2 h1:RzHMPdRMNvSZSDE+Qr20fFWSfBkKPFrLdFhzqmF0VnY=
 github.com/sanyokbig/pqinterval v1.1.2/go.mod h1:jJvMjZaZFVqNTNVCd90zcFOkmbJgjxlWWkpu9/VeUFs=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -130,15 +155,20 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.10/go.mod h1:SVTZcEiaaEsE84gE7dYuteSc4oklkYHIFE4EBu+DiNQ=
 github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.12 h1:uRAKt7s1rB8cabFTcBpfrtlvagCo/9c2bnYC8ckTdLA=
 github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.12/go.mod h1:aTozMqfRG7Jj9uXdh3WG1Wnoc83dZqvleg6oRmBHCa0=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.31.0 h1:woM+Mb4d0A+Dxa3rYPenSN5ZeS9qHUvE8rlObiLRXTY=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.31.0/go.mod h1:PFmBsWbldL1kiWZk9+0LBZz2brhByaGsvp6pRICMlPE=
@@ -168,51 +198,101 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
+golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
-golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
+golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls=
+k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k=
+k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U=
+k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro=
+k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
+sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
diff --git a/main.go b/main.go
index df1359c..9834954 100644
--- a/main.go
+++ b/main.go
@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"database/sql"
 	"flag"
 	"fmt"
 	"io"
@@ -244,8 +243,8 @@ func ensureNotifRecord(ctx context.Context, vicedb *VICEDatabaser, job Job) erro
 
 const maxAttempts = 3
 
-func sendWarning(ctx context.Context, db *sql.DB, vicedb *VICEDatabaser, warningInterval int64, warningKey string) {
-	jobs, err := JobKillWarnings(ctx, db, warningInterval)
+func sendWarning(ctx context.Context, a *TimelordAnalyses, vicedb *VICEDatabaser, warningInterval int64, warningKey string) {
+	jobs, err := a.JobKillWarnings(ctx, warningInterval)
 	if err != nil {
 		log.Error(err)
 	} else {
@@ -313,9 +312,9 @@ func sendWarning(ctx context.Context, db *sql.DB, vicedb *VICEDatabaser, warning
 	}
 }
 
-func sendPeriodic(ctx context.Context, db *sql.DB, vicedb *VICEDatabaser) {
+func sendPeriodic(ctx context.Context, a *TimelordAnalyses, vicedb *VICEDatabaser) {
 	// fetch jobs which periodic updates might apply to
-	jobs, err := JobPeriodicWarnings(ctx, db)
+	jobs, err := a.JobPeriodicWarnings(ctx)
 
 	// loop over them and check if they have notif_statuses info
 	if err != nil {
@@ -329,7 +328,7 @@ func sendPeriodic(ctx context.Context, db *sql.DB, vicedb *VICEDatabaser) {
 				periodDuration      time.Duration
 			)
 
-			if err = EnsurePlannedEndDate(ctx, db, &j); err != nil {
+			if err = a.EnsurePlannedEndDate(ctx, &j); err != nil {
 				log.Error(errors.Wrapf(err, "Error ensuring a planned end date for job %s", j.ID))
 			}
 
@@ -394,6 +393,7 @@ func main() {
 		configPath      = flag.String("config", "/etc/iplant/de/jobservices.yml", "The path to the YAML config file.")
 		expvarPort      = flag.String("port", "60000", "The path to listen for expvar requests on.")
 		appExposerBase  = flag.String("app-exposer", "http://app-exposer", "The base URL for the app-exposer service.")
+		jslBase         = flag.String("job-status-listener", "http://job-status-listener", "The base URL for the job-status-listener service.")
 		killNotifKey    = flag.String("kill-notif-key", "killnotifsent", "The key for the annotation detailing whether the notification about job termination was sent.")
 		warningInterval = flag.Int64("warning-interval", 60, "The number of minutes in advance to warn users about job kills.")
 		warningSentKey  = flag.String("warning-sent-key", warningSentKey, "The key for the annotation detailing whether the job termination warning was sent.")
@@ -508,20 +508,26 @@ func main() {
 
 	go amqpclient.Listen()
 
+	jslURL, err := url.Parse(*jslBase)
+	if err != nil {
+		log.Fatal("could not parse --job-status-listener")
+	}
+
+	a := NewTimelordAnalyses(db)
+
 	amqpclient.AddConsumer(
 		exchange,
 		exchangeType,
 		"timelord",
 		messaging.UpdatesKey,
-		CreateMessageHandler(db),
+		a.CreateMessageHandler(),
 		100,
 	)
 	log.Info("done configuring messaging support")
 
-	jobKiller := &JobKiller{
-		K8sEnabled:     k8sEnabled,
-		AppsBase:       appsBase,
-		AppExposerBase: *appExposerBase,
+	jobKiller, err := NewJobKiller(k8sEnabled, appsBase, *appExposerBase, db, jslURL)
+	if err != nil {
+		log.Fatal(err)
 	}
 
 	go func() {
@@ -531,15 +537,15 @@ func main() {
 			ctx, span := otel.Tracer(otelName).Start(context.Background(), "job killer iteration")
 
 			// 1 hour warning
-			sendWarning(ctx, db, vicedb, *warningInterval, *warningSentKey)
+			sendWarning(ctx, a, vicedb, *warningInterval, *warningSentKey)
 
 			// 1 day warning
-			sendWarning(ctx, db, vicedb, 1440, oneDayWarningKey)
+			sendWarning(ctx, a, vicedb, 1440, oneDayWarningKey)
 
 			// periodic warnings
-			sendPeriodic(ctx, db, vicedb)
+			sendPeriodic(ctx, a, vicedb)
 
-			jl, err = JobsToKill(ctx, db)
+			jl, err = a.JobsToKill(ctx)
 			if err != nil {
 				log.Error(errors.Wrap(err, "error getting list of jobs to kill"))
 				span.End()
@@ -549,6 +555,31 @@ func main() {
 			for _, j := range jl {
 				log.Infof("handling analysis kill logic for external ID: %s, ID: %s", j.ExternalID, j.ID)
 
+				log.Infof("checking for analysis in the cluster by external ID: %s", j.ExternalID)
+				// Look for the analysis to terminate in the cluster.
+				found, err := jobKiller.checkForAnalysisInCluster(ctx, &j)
+				if err != nil {
+					log.Error(err)
+				}
+
+				// If the analysis to terminate isn't in the cluster, mark it as completed through
+				// job-status-listener.
+				if !found {
+					log.Infof("analysis with external ID %s was not found in the cluster", j.ExternalID)
+
+					if err = jobKiller.sendCompletedStatus(ctx, &j); err != nil {
+						log.Errorf("error sending 'Completed' status for analysis with external ID %s: %s", j.ExternalID, err)
+					}
+
+					log.Debugf("sent 'Completed' status update for analysis with external ID %s", j.ExternalID)
+
+					// If the job is listed in the database but isn't in the cluster, we don't want to send
+					// notifications when marking it as completed, so do a continue here.
+					continue
+				}
+
+				log.Infof("analysis with external ID %s was found in the cluster", j.ExternalID)
+
 				if err = ensureNotifRecord(ctx, vicedb, j); err != nil {
 					log.Error(err)
 					span.End()
@@ -566,32 +597,40 @@ func main() {
 
 				log.Debugf("before kill job for external ID: %s, analysis ID: %s", j.ExternalID, j.ID)
 
-				err = jobKiller.KillJob(ctx, db, &j)
+				// Always try to kill the job if it's in the list of jobs to kill.
+				err = jobKiller.KillJob(ctx, &j)
 				if err != nil {
+					// Log the error, but don't return and don't do a continue. The issue may
+					// fix itself on the next attempt and the user should still be warned if
+					// they haven't been already.
 					log.Error(errors.Wrapf(err, "error terminating analysis '%s'", j.ID))
-				} else {
+				}
 
+				// Don't send kill warnings if warning has already been sent. Don't block kill
+				// attempts if the notification failed or if the kill warning was not sent.
+				if !notifStatuses.KillWarningSent {
 					err = SendKillNotification(ctx, &j, *killNotifKey)
 					if err != nil {
 						log.Error(errors.Wrapf(err, "error sending notification that %s has been terminated", j.ID))
 					}
-				}
 
-				if err != nil {
-					notifStatuses.KillWarningFailureCount = notifStatuses.KillWarningFailureCount + 1
+					// the err here only refers to the error possibly returned by the SendKillNotification call.
+					if err != nil {
+						notifStatuses.KillWarningFailureCount = notifStatuses.KillWarningFailureCount + 1
 
-					if err = vicedb.SetKillWarningFailureCount(ctx, &j, notifStatuses.KillWarningFailureCount); err != nil {
-						log.Error(err)
-						span.End()
-						continue
+						if err = vicedb.SetKillWarningFailureCount(ctx, &j, notifStatuses.KillWarningFailureCount); err != nil {
+							log.Error(err)
+							span.End()
+							continue
+						}
 					}
-				}
 
-				if err == nil || notifStatuses.KillWarningFailureCount >= maxAttempts {
-					if err = vicedb.SetKillWarningSent(ctx, &j, true); err != nil {
-						log.Error(err)
-						span.End()
-						continue
+					if err == nil || notifStatuses.KillWarningFailureCount >= maxAttempts {
+						if err = vicedb.SetKillWarningSent(ctx, &j, true); err != nil {
+							log.Error(err)
+							span.End()
+							continue
+						}
 					}
 				}
 			}


Reviews:
user: login: slr71
body: This looks good to me. I just had some small suggestions.

You might be able to fix the lint warnings by updating the GitHub workflow that does the lint check:

```
name: golangci-lint
on:
  push:
    branches:
      - master
      - main
  pull_request:

jobs:
  call-workflow-passing-data:
    uses: cyverse-de/github-workflows/.github/workflows/golangci-lint.yml@v0.2.0
    with:
      go-version: 1.23
```
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-18T19:53:23Z
user: login: johnworth
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T21:52:50Z
user: login: johnworth
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T21:53:57Z
user: login: slr71
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-18T21:54:29Z
user: login: johnworth
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T21:56:30Z

Review comments:
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
path: analysis_termination.go
user: login: slr71
body: This is minor, but the comment could be updated to indicate that this is a client rather than a base URL.
created_at: 2025-04-18T18:50:06Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
+	dedb       *sql.DB
+	jslURL     *url.URL
path: analysis_termination.go
user: login: slr71
body: Is this the URL for `job-status-listener`?
created_at: 2025-04-18T18:52:10Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
+	dedb       *sql.DB
+	jslURL     *url.URL
+}
+
+func NewJobKiller(k8sEnabled bool, appsBase, aeBase string, dedb *sql.DB, jslURL *url.URL) (*JobKiller, error) {
+	appExposer, err := NewAppExposer(aeBase)
+	if err != nil {
+		return nil, err
+	}
+	return &JobKiller{
+		K8sEnabled: k8sEnabled,
+		AppsBase:   appsBase,
+		appExposer: appExposer,
+		dedb:       dedb,
+		jslURL:     jslURL,
+	}, nil
+}
+
+// KillJob uses either the apps or app-exposer APIs to kill a VICE job.
+func (j *JobKiller) KillJob(ctx context.Context, job *Job) error {
+	if j.K8sEnabled {
+		return j.killK8sJob(ctx, job)
+	}
+	return j.killCondorJob(ctx, job.ID, job.User)
+
+}
+
+// killCondorJob uses the provided API at the base URL to kill a running job. This
+// will probably be to the apps service. jobID should be the UUID for the Job,
+// typically returned in the ID field by the analyses service. The username
+// should be the short username for the user that launched the job.
+func (j *JobKiller) killCondorJob(ctx context.Context, jobID, username string) error {
path: analysis_termination.go
user: login: slr71
body: I feel a little pedantic for saying this, but I'd kind of like to see this function renamed to something like `killExecutableJob` or `killNonInteractiveJob`  or something like that since the jobs aren't running in Condor anymore.
created_at: 2025-04-18T18:58:18Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
+	dedb       *sql.DB
+	jslURL     *url.URL
+}
+
+func NewJobKiller(k8sEnabled bool, appsBase, aeBase string, dedb *sql.DB, jslURL *url.URL) (*JobKiller, error) {
+	appExposer, err := NewAppExposer(aeBase)
+	if err != nil {
+		return nil, err
+	}
+	return &JobKiller{
+		K8sEnabled: k8sEnabled,
+		AppsBase:   appsBase,
+		appExposer: appExposer,
+		dedb:       dedb,
+		jslURL:     jslURL,
+	}, nil
+}
+
+// KillJob uses either the apps or app-exposer APIs to kill a VICE job.
+func (j *JobKiller) KillJob(ctx context.Context, job *Job) error {
+	if j.K8sEnabled {
+		return j.killK8sJob(ctx, job)
+	}
+	return j.killCondorJob(ctx, job.ID, job.User)
+
+}
+
+// killCondorJob uses the provided API at the base URL to kill a running job. This
+// will probably be to the apps service. jobID should be the UUID for the Job,
+// typically returned in the ID field by the analyses service. The username
+// should be the short username for the user that launched the job.
+func (j *JobKiller) killCondorJob(ctx context.Context, jobID, username string) error {
+	return j.appExposer.StopAnalyses(ctx, jobID, username)
+}
+
+func (j *JobKiller) requestAnalysisTermination(ctx context.Context, job *Job) error {
+	return j.appExposer.VICESaveAndExit(ctx, job)
+}
path: analysis_termination.go
user: login: slr71
body: I like the idea of having this stuff encapsulated in the `app-exposer` client. 👍 
created_at: 2025-04-18T19:00:09Z
author_association: MEMBER
diff_hunk: @@ -549,6 +555,31 @@ func main() {
 			for _, j := range jl {
 				log.Infof("handling analysis kill logic for external ID: %s, ID: %s", j.ExternalID, j.ID)
 
+				log.Infof("checking for analysis in the cluster by external ID: %s", j.ExternalID)
+				// Look for the analysis to terminate in the cluster.
+				found, err := jobKiller.checkForAnalysisInCluster(ctx, &j)
+				if err != nil {
+					log.Error(err)
+				}
+
+				// If the analysis to terminate isn't in the cluster, mark it as completed through
+				// job-status-listener.
+				if !found {
+					log.Infof("analysis with external ID %s was not found in the cluster", j.ExternalID)
+
+					if err = jobKiller.sendCompletedStatus(ctx, &j); err != nil {
+						log.Errorf("error sending 'Completed' status for analysis with external ID %s: %s", j.ExternalID, err)
+					}
+
+					log.Debugf("sent 'Completed' status update for analysis with external ID %s", j.ExternalID)
+
+					// If the job is listed in the database but isn't in the cluster, we don't want to send
+					// notifications when marking it as completed, so do a continue here.
+					continue
+				}
path: main.go
user: login: slr71
body: 👍 
created_at: 2025-04-18T19:46:57Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
path: analysis_termination.go
user: login: johnworth
body: This should be fixed now. Sorry about that.
created_at: 2025-04-18T21:52:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
+	dedb       *sql.DB
+	jslURL     *url.URL
path: analysis_termination.go
user: login: johnworth
body: Yup, I just added a comment for it.
created_at: 2025-04-18T21:53:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
path: analysis_termination.go
user: login: slr71
body: No worries at all. I just figured I'd mention it.
created_at: 2025-04-18T21:54:29Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+)
+
+// JobKiller is responsible for killing jobs either in HTCondor or in K8s.
+type JobKiller struct {
+	K8sEnabled bool        // whether or not the VICE apps are running k8s
+	AppsBase   string      // base URL for the apps service
+	appExposer *AppExposer // base URL for the app-exposer serivce
+	dedb       *sql.DB
+	jslURL     *url.URL
+}
+
+func NewJobKiller(k8sEnabled bool, appsBase, aeBase string, dedb *sql.DB, jslURL *url.URL) (*JobKiller, error) {
+	appExposer, err := NewAppExposer(aeBase)
+	if err != nil {
+		return nil, err
+	}
+	return &JobKiller{
+		K8sEnabled: k8sEnabled,
+		AppsBase:   appsBase,
+		appExposer: appExposer,
+		dedb:       dedb,
+		jslURL:     jslURL,
+	}, nil
+}
+
+// KillJob uses either the apps or app-exposer APIs to kill a VICE job.
+func (j *JobKiller) KillJob(ctx context.Context, job *Job) error {
+	if j.K8sEnabled {
+		return j.killK8sJob(ctx, job)
+	}
+	return j.killCondorJob(ctx, job.ID, job.User)
+
+}
+
+// killCondorJob uses the provided API at the base URL to kill a running job. This
+// will probably be to the apps service. jobID should be the UUID for the Job,
+// typically returned in the ID field by the analyses service. The username
+// should be the short username for the user that launched the job.
+func (j *JobKiller) killCondorJob(ctx context.Context, jobID, username string) error {
path: analysis_termination.go
user: login: johnworth
body: Should be fixed now, sorry about that.
created_at: 2025-04-18T21:56:30Z
author_association: CONTRIBUTOR

PR comments:
user: login: socket-security[bot]
created_at: 2025-04-18T17:14:08Z
body: **Review the following changes in direct dependencies.** Learn more about [Socket for GitHub](https://socket.dev?utm_medium=gh).

<table>
<thead>
<tr>
<th>Diff</th>
<th width="200px">Package</th>
<th align="left" width="100px">Supply Chain<br/>Security</th>
<th align="left" width="100px">Vulnerability</th>
<th align="left" width="100px">Quality</th>
<th align="left" width="100px">Maintenance</th>
<th align="left" width="100px">License</th>
</tr>
</thead>
<tbody>
<tr><td align="center"><a href="https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies&dependency_item_key=16005809544"><img src="https://github-app-statics.socket.dev/diff-added.svg" title="Added" alt="Added" width="20" height="20"></a></td><td><a href="https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies&dependency_item_key=16005809544">k8s.io/​api@​v0.32.3</a></td><td align="left"><a href="https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies&dependency_item_key=16005809544"><img src="https://github-app-statics.socket.dev/score-76.svg" title="Supply Chain Security" width="40" height="40" alt="76"></a></td><td align="left"><a href="https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies&dependency_item_key=16005809544"><img src="https://github-app-statics.socket.dev/score-100.svg" title="Vulnerability" width="40" height="40" alt="100"></a></td><td align="left"><a href="https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies&dependency_item_key=16005809544"><img src="https://github-app-statics.socket.dev/score-100.svg" title="Quality" width="40" height="40" alt="100"></a></td><td align="left"><a href="https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies&dependency_item_key=16005809544"><img src="https://github-app-statics.socket.dev/score-100.svg" title="Maintenance" width="40" height="40" alt="100"></a></td><td align="left"><a href="https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies&dependency_item_key=16005809544"><img src="https://github-app-statics.socket.dev/score-100.svg" title="License" width="40" height="40" alt="100"></a></td></tr>
</tbody>
</table>

[View full report](https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=dependencies)

<!-- overview-comment -->
author_association: NONE
user: login: socket-security[bot]
created_at: 2025-04-18T17:14:10Z
body: **All alerts resolved.** Learn more about [Socket for GitHub](https://socket.dev?utm_medium=gh).

This PR previously contained dependency changes with security issues that have been resolved, removed, or ignored.

[View full report](https://socket.dev/dashboard/org/cyverse-de/diff/abdd96f1-4ae7-43cf-935c-9722c5a64b98/4154ea19-6931-4a39-9edf-755e82321164?tab=alerts&action=error%2Cwarn)
author_association: NONE