Repository: fleetdm/fleet (Go)

Issue:
GitOps Labels: error when deleting a label not defined in yaml
**Fleet version**: Latest on dogfood
**Web browser and operating system**: Any

<hr/>

### 💥  Actual behavior

> [+] syncing 7 labels (0 new and 7 updated)
> [-] deleting label 'bwtest'
> Error: DELETE /api/latest/fleet/labels/bwtest received status 422 Validation Failed: the operation violates a foreign key constraint on labels: bwtest
> Error: Process completed with exit code 1.

### 🧑‍💻  Steps to reproduce
<!-- Provide step-by-step actions of how to recreate this bug in a clean install of Fleet. (This helps others understand and fix it more quickly.) -->
1. Manage labels via gitops/yaml
2. Create a manual label in the UI without adding it to your yaml
3. Run gitops and observe error

### 🕯️ More info _(optional)_
<!-- Add any additional details you think could be relevant to solving or reproducing the bug (e.g., "this does not reproduce when...") -->
N/A

<!-- If this is a performance issue, follow these steps to generate and attach a debug archive: https://fleetdm.com/docs/using-fleet/monitoring-fleet#debugging-performance-issues -->

### 🛠️ To fix
<!-- If this bug requires additional product design work, uncomment the heading above and add instructions to fix, Figma link, etc. here once design changes are settled. -->
<!-- Product designer: _________________________ --> <!-- Who is the product designer to contact if folks have questions about this fix? -->



Discussion: Hey team! Please [add your planning poker estimate](https://app.zenhub.com/workspaces/Drafting-6192dd66ea2562000faea25c/issues/gh/fleetdm/fleet/28440?planning-poker) with Zenhub @dantecatalfamo @jacobshandling @sgress454
@juan-fdz-hawa See https://fleetdm.slack.com/archives/C071NNMSP2R/p1745332100311179 (particularly [this comment](https://fleetdm.slack.com/archives/C071NNMSP2R/p1745339046993939?thread_ts=1745332100.311179&cid=C071NNMSP2R)).  I think we need to just re-order the logic so that we delete labels last.



Fix (patch):
diff --git a/changes/28440-deleting-non-managed-gitops-labels b/changes/28440-deleting-non-managed-gitops-labels
new file mode 100644
index 000000000000..7b463434fda6
--- /dev/null
+++ b/changes/28440-deleting-non-managed-gitops-labels
@@ -0,0 +1,1 @@
+* Fixed issue with the gitops command that prevented non-managed labels to be deleted if used by software installations.
\ No newline at end of file
diff --git a/cmd/fleetctl/fleetctl/gitops.go b/cmd/fleetctl/fleetctl/gitops.go
index 53966c3792f9..78d45fdcfa3e 100644
--- a/cmd/fleetctl/fleetctl/gitops.go
+++ b/cmd/fleetctl/fleetctl/gitops.go
@@ -133,6 +133,9 @@ func gitopsCommand() *cli.Command {
 			// We only want to have one global config loaded
 			globalConfigLoaded := false
 
+			// List of things we want to do at the end of this run
+			var allPostOps []func() error
+
 			for _, flFilename := range flFilenames.Value() {
 				baseDir := filepath.Dir(flFilename)
 				config, err := spec.GitOpsFromFile(flFilename, baseDir, appConfig, logf)
@@ -313,7 +316,7 @@ func gitopsCommand() *cli.Command {
 				if err != nil {
 					return err
 				}
-				assumptions, err := fleetClient.DoGitOps(c.Context, config, flFilename, logf, flDryRun, teamDryRunAssumptions, appConfig,
+				assumptions, postOps, err := fleetClient.DoGitOps(c.Context, config, flFilename, logf, flDryRun, teamDryRunAssumptions, appConfig,
 					teamsSoftwareInstallers, teamsVPPApps, teamsScripts)
 				if err != nil {
 					return err
@@ -323,6 +326,7 @@ func gitopsCommand() *cli.Command {
 				} else {
 					teamDryRunAssumptions = assumptions
 				}
+				allPostOps = append(allPostOps, postOps...)
 			}
 
 			// if there were assignments to tokens, and some of the teams were missing at that time, submit a separate patch request to set them now.
@@ -342,11 +346,12 @@ func gitopsCommand() *cli.Command {
 			for _, teamWithApps := range missingVPPTeamsWithApps {
 				_, _ = fmt.Fprintf(c.App.Writer, ReapplyingTeamForVPPAppsMsg, *teamWithApps.config.TeamName)
 				teamWithApps.config.Software.AppStoreApps = teamWithApps.vppApps
-				_, err := fleetClient.DoGitOps(c.Context, teamWithApps.config, teamWithApps.filename, logf, flDryRun, teamDryRunAssumptions, appConfig,
+				_, postOps, err := fleetClient.DoGitOps(c.Context, teamWithApps.config, teamWithApps.filename, logf, flDryRun, teamDryRunAssumptions, appConfig,
 					teamsSoftwareInstallers, teamsVPPApps, teamsScripts)
 				if err != nil {
 					return err
 				}
+				allPostOps = append(allPostOps, postOps...)
 			}
 
 			if flDeleteOtherTeams && appConfig.License.IsPremium() { // skip team deletion for non-premium users
@@ -383,11 +388,19 @@ func gitopsCommand() *cli.Command {
 			if globalConfigLoaded && !noTeamPresent {
 				defaultNoTeamConfig := new(spec.GitOps)
 				defaultNoTeamConfig.TeamName = ptr.String(fleet.TeamNameNoTeam)
-				_, err = fleetClient.DoGitOps(c.Context, defaultNoTeamConfig, "no-team.yml", logf, flDryRun, nil, appConfig,
+				_, postOps, err := fleetClient.DoGitOps(c.Context, defaultNoTeamConfig, "no-team.yml", logf, flDryRun, nil, appConfig,
 					map[string][]fleet.SoftwarePackageResponse{}, map[string][]fleet.VPPAppResponse{}, map[string][]fleet.ScriptResponse{})
 				if err != nil {
 					return err
 				}
+
+				allPostOps = append(allPostOps, postOps...)
+			}
+
+			for _, postOp := range allPostOps {
+				if err := postOp(); err != nil {
+					return err
+				}
 			}
 
 			if flDryRun {
@@ -395,6 +408,7 @@ func gitopsCommand() *cli.Command {
 			} else {
 				_, _ = fmt.Fprintf(c.App.Writer, "[!] gitops succeeded\n")
 			}
+
 			return nil
 		},
 	}
diff --git a/server/service/client.go b/server/service/client.go
index b6f5a96a5d0a..2e736c70b19e 100644
--- a/server/service/client.go
+++ b/server/service/client.go
@@ -1596,7 +1596,7 @@ func (c *Client) DoGitOps(
 	teamsSoftwareInstallers map[string][]fleet.SoftwarePackageResponse,
 	teamsVPPApps map[string][]fleet.VPPAppResponse,
 	teamsScripts map[string][]fleet.ScriptResponse,
-) (*fleet.TeamSpecsDryRunAssumptions, error) {
+) (*fleet.TeamSpecsDryRunAssumptions, []func() error, error) {
 	baseDir := filepath.Dir(fullFilename)
 	filename := filepath.Base(fullFilename)
 	var teamAssumptions *fleet.TeamSpecsDryRunAssumptions
@@ -1613,6 +1613,9 @@ func (c *Client) DoGitOps(
 	}
 	var mdmAppConfig map[string]interface{}
 	var team map[string]interface{}
+
+	var postOps []func() error
+
 	if config.TeamName == nil {
 		group.AppConfig = config.OrgSettings
 		group.EnrollSecret = &fleet.EnrollSecretSpec{Secrets: config.OrgSettings["secrets"].([]*fleet.EnrollSecret)}
@@ -1621,10 +1624,19 @@ func (c *Client) DoGitOps(
 
 		// Labels
 		if config.Labels == nil || len(config.Labels) > 0 {
-			err = c.doGitOpsLabels(config, logFn, dryRun)
+			labelsToDelete, err := c.doGitOpsLabels(config, logFn, dryRun)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
+			postOps = append(postOps, func() error {
+				for _, labelToDelete := range labelsToDelete {
+					logFn("[-] deleting label '%s'\n", labelToDelete)
+					if err := c.DeleteLabel(labelToDelete); err != nil {
+						return err
+					}
+				}
+				return nil
+			})
 		}
 
 		// Integrations
@@ -1636,7 +1648,7 @@ func (c *Client) DoGitOps(
 		}
 		integrations, ok = integrations.(map[string]interface{})
 		if !ok {
-			return nil, errors.New("org_settings.integrations config is not a map")
+			return nil, nil, errors.New("org_settings.integrations config is not a map")
 		}
 		if jira, ok := integrations.(map[string]interface{})["jira"]; !ok || jira == nil {
 			integrations.(map[string]interface{})["jira"] = []interface{}{}
@@ -1652,7 +1664,7 @@ func (c *Client) DoGitOps(
 			integrations.(map[string]interface{})["ndes_scep_proxy"] = nil
 		} else {
 			if _, ok = ndesSCEPProxy.(map[string]interface{}); !ok {
-				return nil, errors.New("org_settings.integrations.ndes_scep_proxy config is not a map")
+				return nil, nil, errors.New("org_settings.integrations.ndes_scep_proxy config is not a map")
 			}
 		}
 		if digicertIntegration, ok := integrations.(map[string]interface{})["digicert"]; !ok || digicertIntegration == nil {
@@ -1661,12 +1673,12 @@ func (c *Client) DoGitOps(
 			// We unmarshal DigiCert integration into its dedicated type for additional validation.
 			digicertJSON, err := json.Marshal(integrations.(map[string]interface{})["digicert"])
 			if err != nil {
-				return nil, fmt.Errorf("org_settings.integrations.digicert cannot be marshalled into JSON: %w", err)
+				return nil, nil, fmt.Errorf("org_settings.integrations.digicert cannot be marshalled into JSON: %w", err)
 			}
 			var digicertData optjson.Slice[fleet.DigiCertIntegration]
 			err = json.Unmarshal(digicertJSON, &digicertData)
 			if err != nil {
-				return nil, fmt.Errorf("org_settings.integrations.digicert cannot be parsed: %w", err)
+				return nil, nil, fmt.Errorf("org_settings.integrations.digicert cannot be parsed: %w", err)
 			}
 			integrations.(map[string]interface{})["digicert"] = digicertData
 		}
@@ -1676,12 +1688,12 @@ func (c *Client) DoGitOps(
 			// We unmarshal Custom SCEP integration into its dedicated type for additional validation
 			custonSCEPJSON, err := json.Marshal(integrations.(map[string]interface{})["custom_scep_proxy"])
 			if err != nil {
-				return nil, fmt.Errorf("org_settings.integrations.custom_scep_proxy cannot be marshalled into JSON: %w", err)
+				return nil, nil, fmt.Errorf("org_settings.integrations.custom_scep_proxy cannot be marshalled into JSON: %w", err)
 			}
 			var customSCEPData optjson.Slice[fleet.CustomSCEPProxyIntegration]
 			err = json.Unmarshal(custonSCEPJSON, &customSCEPData)
 			if err != nil {
-				return nil, fmt.Errorf("org_settings.integrations.custom_scep_proxy cannot be parsed: %w", err)
+				return nil, nil, fmt.Errorf("org_settings.integrations.custom_scep_proxy cannot be parsed: %w", err)
 			}
 			integrations.(map[string]interface{})["custom_scep_proxy"] = customSCEPData
 		}
@@ -1738,7 +1750,7 @@ func (c *Client) DoGitOps(
 		}
 		mdmAppConfig, ok = mdmConfig.(map[string]interface{})
 		if !ok {
-			return nil, errors.New("org_settings.mdm config is not a map")
+			return nil, nil, errors.New("org_settings.mdm config is not a map")
 		}
 
 		if _, ok := mdmAppConfig["apple_bm_default_team"]; !ok && appConfig.License.IsPremium() {
@@ -1831,14 +1843,14 @@ func (c *Client) DoGitOps(
 		team["integrations"] = integrations
 		_, ok = integrations.(map[string]interface{})
 		if !ok {
-			return nil, errors.New("team_settings.integrations config is not a map")
+			return nil, nil, errors.New("team_settings.integrations config is not a map")
 		}
 		if googleCal, ok := integrations.(map[string]interface{})["google_calendar"]; !ok || googleCal == nil {
 			integrations.(map[string]interface{})["google_calendar"] = map[string]interface{}{}
 		} else {
 			_, ok = googleCal.(map[string]interface{})
 			if !ok {
-				return nil, errors.New("team_settings.integrations.google_calendar config is not a map")
+				return nil, nil, errors.New("team_settings.integrations.google_calendar config is not a map")
 			}
 		}
 
@@ -1936,7 +1948,7 @@ func (c *Client) DoGitOps(
 			team["gitops_filename"] = filename
 			rawTeam, err := json.Marshal(team)
 			if err != nil {
-				return nil, fmt.Errorf("error marshalling team spec: %w", err)
+				return nil, nil, fmt.Errorf("error marshalling team spec: %w", err)
 			}
 			group.Teams = []json.RawMessage{rawTeam}
 			group.TeamsDryRunAssumptions = teamDryRunAssumptions
@@ -1951,7 +1963,7 @@ func (c *Client) DoGitOps(
 		ExpandEnvConfigProfiles: true,
 	}, teamsSoftwareInstallers, teamsVPPApps, teamsScripts)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	var teamSoftwareInstallers []fleet.SoftwarePackageResponse
@@ -1961,15 +1973,15 @@ func (c *Client) DoGitOps(
 	if config.TeamName != nil {
 		if !config.IsNoTeam() {
 			if len(teamIDsByName) != 1 {
-				return nil, fmt.Errorf("expected 1 team spec to be applied, got %d", len(teamIDsByName))
+				return nil, nil, fmt.Errorf("expected 1 team spec to be applied, got %d", len(teamIDsByName))
 			}
 			teamID, ok := teamIDsByName[*config.TeamName]
 			if ok && teamID == 0 {
 				if dryRun {
 					logFn("[+] would've added any policies/queries to new team %s\n", *config.TeamName)
-					return nil, nil
+					return nil, postOps, nil
 				}
-				return nil, fmt.Errorf("team %s not created", *config.TeamName)
+				return nil, nil, fmt.Errorf("team %s not created", *config.TeamName)
 			}
 			for _, teamID = range teamIDsByName {
 				config.TeamID = &teamID
@@ -1980,7 +1992,7 @@ func (c *Client) DoGitOps(
 		} else {
 			noTeamSoftwareInstallers, noTeamVPPApps, err := c.doGitOpsNoTeamSetupAndSoftware(config, baseDir, appConfig, logFn, dryRun)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			teamSoftwareInstallers = noTeamSoftwareInstallers
 			teamVPPApps = noTeamVPPApps
@@ -1990,7 +2002,7 @@ func (c *Client) DoGitOps(
 
 	err = c.doGitOpsPolicies(config, teamSoftwareInstallers, teamVPPApps, teamScripts, logFn, dryRun)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// We currently don't support queries for "No team" thus
@@ -1998,11 +2010,11 @@ func (c *Client) DoGitOps(
 	if !config.IsNoTeam() {
 		err = c.doGitOpsQueries(config, logFn, dryRun)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 
-	return teamAssumptions, nil
+	return teamAssumptions, postOps, nil
 }
 
 func (c *Client) doGitOpsNoTeamSetupAndSoftware(
@@ -2121,10 +2133,10 @@ func pluralize(count int, ifSingle string, ifPlural string) string {
 	return ifPlural
 }
 
-func (c *Client) doGitOpsLabels(config *spec.GitOps, logFn func(format string, args ...interface{}), dryRun bool) error {
+func (c *Client) doGitOpsLabels(config *spec.GitOps, logFn func(format string, args ...interface{}), dryRun bool) ([]string, error) {
 	persistedLabels, err := c.GetLabels()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	var numUpdates int
 	var labelsToDelete []string
@@ -2149,22 +2161,15 @@ func (c *Client) doGitOpsLabels(config *spec.GitOps, logFn func(format string, a
 		if numUpdates > 0 {
 			logFn("[+] would've updated %d label%s\n", numUpdates, pluralize(numUpdates, "", "s"))
 		}
-	} else {
-		logFn("[+] syncing %d label%s (%d new and %d updated)\n", len(config.Labels), pluralize(len(config.Labels), "", "s"), len(config.Labels)-numUpdates, numUpdates)
-		err = c.ApplyLabels(config.Labels)
-		if err != nil {
-			return err
-		}
+		return nil, nil
+	}
 
-		for _, labelToDelete := range labelsToDelete {
-			logFn("[-] deleting label '%s'\n", labelToDelete)
-			err = c.DeleteLabel(labelToDelete)
-			if err != nil {
-				return err
-			}
-		}
+	logFn("[+] syncing %d label%s (%d new and %d updated)\n", len(config.Labels), pluralize(len(config.Labels), "", "s"), len(config.Labels)-numUpdates, numUpdates)
+	err = c.ApplyLabels(config.Labels)
+	if err != nil {
+		return nil, err
 	}
-	return nil
+	return labelsToDelete, nil
 }
 
 func (c *Client) doGitOpsPolicies(config *spec.GitOps, teamSoftwareInstallers []fleet.SoftwarePackageResponse, teamVPPApps []fleet.VPPAppResponse, teamScripts []fleet.ScriptResponse, logFn func(format string, args ...interface{}), dryRun bool) error {