Repository: hashicorp/nomad
File: nomad/encrypter.go (Go)

Code before:
		if decryptTasks == 0 {

			e.decryptTasksLock.Lock()
			delete(e.decryptTasks, wrappedKeys.KeyID)
			e.decryptTasksLock.Unlock()

			logger.Error("root key cannot be decrypted", "error", err)
			return err
		}

		// If we have at least one task running, we can log a warning for the
		// operator but continue to wait for the other tasks to complete.
		logger.Warn("root key cannot be decrypted by some KMS providers",
			"error", err)
	}

	// The routine will now wait until the server tells us to stop or until we
	// successfully decrypt the key. The context's cancellation function has a
	// deferred call above, so we don't need to worry about it here unless we
	// want to preemptively cancel in-flight tasks.
	select {
	case <-completeCtx.Done():

		// In this event, the server is shutting down and the agent process will
		// exit. This means the decrypter state will be lost, so there is no
		// need to tidy the decryption task state.
		return completeCtx.Err()

	case generatedCipher := <-cipherSetCh:

		// By reaching this point, we have a decrypted cipher. No errors can
		// occur from here on, so start by telling any other running work to
		// stop.
		cancel()

		// Write the cipher to the encrypter keyring. We could just overwrite an
		// existing entry, but there is no harm in checking this first as we
		// need a write lock anyway.
		e.keyringLock.Lock()
		if _, ok := e.keyring[wrappedKeys.KeyID]; !ok {
			e.keyring[wrappedKeys.KeyID] = generatedCipher
		}
		e.keyringLock.Unlock()

		// We can now remove the decrypt task from the tracking map indicating
		// this key is ready for use.
		e.decryptTasksLock.Lock()
		delete(e.decryptTasks, wrappedKeys.KeyID)
		e.decryptTasksLock.Unlock()
	}


Review comment (bug):
I really like this warning because it tells the cluster admin "hey your KMS configuration is broken so you'd don't have HA here!" without breaking their whole world. :+1: 

Code after:
		}

		// fan-out decryption tasks for HA in Nomad Enterprise. we can use the
		// key whenever any one provider returns a successful decryption.
		go e.decryptWrappedKeyTask(completeCtx, wrapper, wrappedKeys.Meta(), wrappedKey, cipherSetCh)
		decryptTasks++
	}

	if err := mErr.ErrorOrNil(); err != nil {

		// If we have no tasks running, we can log an error for the operator and
		// exit.
		//
		// It is likely any decryption configuration for the key is incorrect
		// and follow-up attempts from other Raft/FMS calls for this key will
		// also fail. We should not, however, continue with the server startup
		// without this key, and therefore we do not delete any added tracking.
		if decryptTasks == 0 {
			logger.Error("root key cannot be decrypted", "error", err)
			return err
		}

		// If we have at least one task running, we can log a warning for the
		// operator but continue to wait for the other tasks to complete.
		logger.Warn("root key cannot be decrypted by some KMS providers",
			"error", err)
	}

	// The routine will now wait until the server tells us to stop or until we
	// successfully decrypt the key. The context's cancellation function has a
	// deferred call above, so we don't need to worry about it here unless we
	// want to preemptively cancel in-flight tasks.
	select {
	case <-completeCtx.Done():

		// In this event, the server is shutting down and the agent process will
		// exit. This means the decrypter state will be lost, so there is no
		// need to tidy the decryption task state.
		return completeCtx.Err()

	case generatedCipher := <-cipherSetCh:

		// By reaching this point, we have a decrypted cipher. No errors can
		// occur from here on, so start by telling any other running work to
		// stop.
		cancel()

		// Write the cipher to the encrypter keyring. We could just overwrite an
		// existing entry, but there is no harm in checking this first as we
		// need a write lock anyway.
		e.keyringLock.Lock()