Repository: nats-io/nats-server

Issue:
default_sentinel - auth callout - Non bearer JWT results in connection to eb assigned to $G
### Observed behavior

For a server in operator mode with auth callout enabled and default_sentinel set:. IFF the Sentinel is valid BUT not a bearer JWT (this is an error that is very easy to make)
Any incoming connection will be assigned to

### Expected behavior

Error on server startup (check if bearer token) OR fast fail and error message in server log.

Additional notes:
- When the sentinel is corrupted or cannot be validated by the server the authentication fails fast - but no warning is produced


### Server and client version

Server 2.11.6 
Any client

### Host environment

Windows/Linux

### Steps to reproduce

Run example conf below in debug. 
Connect with any non JWT credentials.
See Auth callout request go to $G

````
{
"default_sentinel": "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJDUEhWT1ZPWElJUFdKRDJZVkw1T0pIS0lLTFc2R05aWDdYQUNNN0hGWDNIVUZWVjRBNEFRIiwiaWF0IjoxNzUyNjMxODYzLCJpc3MiOiJBQjVYVklONEZaTllYTlYyWDZYN0ZWVlJFN0E0RUQ3M0JZNEVOUFJOQUdLS0pRN1JXUjJRNUFTUyIsIm5hbWUiOiJzZW50aW5lbCIsInN1YiI6IlVBMklZNlQ0Q1dOUlNFWVFZTkhQVzZIWEhCWkFOQ0ZaT0xMUVk3TlJWSlNYT0kzVUVZU1RaS0hGIiwibmF0cyI6eyJwdWIiOnsiZGVueSI6WyJcdTAwM2UiXX0sInN1YiI6eyJkZW55IjpbIlx1MDAzZSJdfSwic3VicyI6LTEsImRhdGEiOi0xLCJwYXlsb2FkIjotMSwidHlwZSI6InVzZXIiLCJ2ZXJzaW9uIjoyfX0.onyBWBv1a0g4HYS7nkYk59bsHgodtmUeoeWH72PVI76QjZzrGcR4iTeefjTc8pTqK0FibkLttpWhCN11IkktDg",

  "http_port": 8222,
  "jetstream": {
    "max_file_store": 1Gi,
    "max_memory_store": 1Gi,
    "store_dir": "/data/js"
  },
  "lame_duck_duration": "30s",
  "lame_duck_grace_period": "10s",
  "operator": "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiI3TENET0ZBRFdaSkZXTkpRVFMzM0VNU0tSWTRaVjZGNlRIR1dUN1NUS0hFTUNVNUhJRTdBIiwiaWF0IjoxNzUyNjMxNzc4LCJpc3MiOiJPQ0gzVFU0S0tTR0dXT0lXTTNWUzVBUk1JWUdVTFJZNjRGNVhVT1FBTUNUUllaS1VWRUhURzNYSiIsIm5hbWUiOiJsb2NhbC1wbGF0Zm9ybSIsInN1YiI6Ik9DSDNUVTRLS1NHR1dPSVdNM1ZTNUFSTUlZR1VMUlk2NEY1WFVPUUFNQ1RSWVpLVVZFSFRHM1hKIiwibmF0cyI6eyJzeXN0ZW1fYWNjb3VudCI6IkFCN0VZNUw2VU9JWkpQRVNDQ0VRSDQ1SE1QUlNCUEFRR0E3SE1aNEhPVDdSWEVYRUZYS0pTSUpTIiwidHlwZSI6Im9wZXJhdG9yIiwidmVyc2lvbiI6Mn19.tox7SqoNkVWa324WKmbnP6J3SpvFrgh0s66m6qeGiLFKmxP3L8UYbFnxa9tb55xGOTwnoXiaOO8qtazqrgawBQ",
  
  "port": 4222,
  "resolver": {
    "allow_delete": false,
    "dir": "./jwt",
    "interval": "2m",
    "timeout": "2.9s",
    "type": "full"
  },
  "resolver_preload": {
    "AB7EY5L6UOIZJPESCCEQH45HMPRSBPAQGA7HMZ4HOT7RXEXEFXKJSIJS": "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJQT1MyUUtENk5FTkpXWDJCWFRDUFFESVJYSTNZQTNGVEQzUVFNWk9QQkNJUzQ2SkZGNjVRIiwiaWF0IjoxNzUyNjMxNzc0LCJpc3MiOiJPQ0gzVFU0S0tTR0dXT0lXTTNWUzVBUk1JWUdVTFJZNjRGNVhVT1FBTUNUUllaS1VWRUhURzNYSiIsIm5hbWUiOiJTWVMiLCJzdWIiOiJBQjdFWTVMNlVPSVpKUEVTQ0NFUUg0NUhNUFJTQlBBUUdBN0hNWjRIT1Q3UlhFWEVGWEtKU0lKUyIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJhdXRob3JpemF0aW9uIjp7fSwidHlwZSI6ImFjY291bnQiLCJ2ZXJzaW9uIjoyfX0.bp66SbyKEkQDIB6MLlI5SrL020mt2mtjjA5vs9yv1_yB5DXnKqfa5JHiPQ6ScTukuEMaTIkeRwwvpjO9GuyCAQ"
    
    "AB5XVIN4FZNYXNV2X6X7FVVRE7A4ED73BY4ENPRNAGKKJQ7RWR2Q5ASS":  "eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJXS05IWUhONE01VDJLUVRJUk5QTExQQllBNFRZQjNUQzJBM0wzREZTWVhRQkE2T1E2SkpBIiwiaWF0IjoxNzUyNjMxODgwLCJpc3MiOiJPQ0gzVFU0S0tTR0dXT0lXTTNWUzVBUk1JWUdVTFJZNjRGNVhVT1FBTUNUUllaS1VWRUhURzNYSiIsIm5hbWUiOiJhdXRoIiwic3ViIjoiQUI1WFZJTjRGWk5ZWE5WMlg2WDdGVlZSRTdBNEVENzNCWTRFTlBSTkFHS0tKUTdSV1IyUTVBU1MiLCJuYXRzIjp7ImxpbWl0cyI6eyJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJpbXBvcnRzIjotMSwiZXhwb3J0cyI6LTEsIndpbGRjYXJkcyI6dHJ1ZSwiY29ubiI6LTEsImxlYWYiOi0xfSwiZGVmYXVsdF9wZXJtaXNzaW9ucyI6eyJwdWIiOnt9LCJzdWIiOnt9fSwiYXV0aG9yaXphdGlvbiI6eyJhdXRoX3VzZXJzIjpbIlVCSk9MNzQ1TElTVFZPUzVMTVVDVjRaVUNaNk02SFZaVVRFTUtHWU1PVkRTUVpGQ1VGR040VzNLIl0sImFsbG93ZWRfYWNjb3VudHMiOlsiKiJdfSwidHlwZSI6ImFjY291bnQiLCJ2ZXJzaW9uIjoyfX0.JV4usEF9dzAKEK9hRA9XlxWMgMskDyVQ39m_XugqySvW6unFOF5OWSfAqqp54QlU7WUeiSZ4-1NlwPjRApQdCw"
  },
  "server_name": SERVER_NAME,
  "system_account": "AB7EY5L6UOIZJPESCCEQH45HMPRSBPAQGA7HMZ4HOT7RXEXEFXKJSIJS",
  "websocket": {
    "no_tls": true,
    "port": 8080
  }
}
````



Hints from issue comments:

Solution patch:
diff --git a/server/configs/reload/defaultsentinel_1.conf b/server/configs/reload/defaultsentinel_1.conf
deleted file mode 100644
index 63549ebf..00000000
--- a/server/configs/reload/defaultsentinel_1.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-listen:   127.0.0.1:-1
-
-// Operator "O"
-operator: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJDUUsyV1c3SkhGVVhNR0g1TlFEWUlJT0UyWTRRWlZMVlFTS1ZKTVZITzZZRFBMWVg2Q1hRIiwiaWF0IjoxNzQ0NzI2MzA4LCJpc3MiOiJPRDQzNElRTVdGQTNTNzVBS1kyMlQ3UkdLRkdSSkRBNFFaU0xQRFFCRVZQVktONktBUTRTWVVMNiIsIm5hbWUiOiJPIiwic3ViIjoiT0Q0MzRJUU1XRkEzUzc1QUtZMjJUN1JHS0ZHUkpEQTRRWlNMUERRQkVWUFZLTjZLQVE0U1lVTDYiLCJuYXRzIjp7InR5cGUiOiJvcGVyYXRvciIsInZlcnNpb24iOjJ9fQ.DKuOHqWHgooC6x_vFaY66hLY5ZxoDKk2VVq71-gl_FDn2o_Amu_gQAxYvqLsNZvtlggWF-vKq944xcZmp0N1Dg
-
-resolver: MEMORY
-
-resolver_preload: {
-  // Account "SYS"
-  AB3GSO5OVHXQJYNNEMX4SGZCS3SPMNMDX4YA65M3SQX7XXHI4NFM2TF6: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJJSEtXN0RLTk9MRVY2T1hVUUZGQlRCN1pWM0NHNzc2M0RYVFdGNEsyU0NUVkNWUDNOSEFBIiwiaWF0IjoxNzQ0NzMwMDcyLCJpc3MiOiJPRDQzNElRTVdGQTNTNzVBS1kyMlQ3UkdLRkdSSkRBNFFaU0xQRFFCRVZQVktONktBUTRTWVVMNiIsIm5hbWUiOiJTWVMiLCJzdWIiOiJBQjNHU081T1ZIWFFKWU5ORU1YNFNHWkNTM1NQTU5NRFg0WUE2NU0zU1FYN1hYSEk0TkZNMlRGNiIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJhdXRob3JpemF0aW9uIjp7fSwidHlwZSI6ImFjY291bnQiLCJ2ZXJzaW9uIjoyfX0.cteeeg1_aw8FNGp6EMS1sGmggC165ZJ0QRynaHlg1j43ZrjJ5qZ43trW5BxbthY5EcplqmnvcCC7cmO7INLQDg
-
-  // Account "A"
-  ACX3MVLHNZ4SJZQFR72E2RQ7BUIDTQAAFOUAZLDQ7TK44RS55OSVSWU5: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiIyV1NPTDIzQzNISllLTTc2TkJBTzU0Tlc3QldNVU81QVo3Rk5ORE9XUVlRVVhORkxNTldBIiwiaWF0IjoxNzQ0NzI2MzEzLCJpc3MiOiJPRDQzNElRTVdGQTNTNzVBS1kyMlQ3UkdLRkdSSkRBNFFaU0xQRFFCRVZQVktONktBUTRTWVVMNiIsIm5hbWUiOiJBIiwic3ViIjoiQUNYM01WTEhOWjRTSlpRRlI3MkUyUlE3QlVJRFRRQUFGT1VBWkxEUTdUSzQ0UlM1NU9TVlNXVTUiLCJuYXRzIjp7ImxpbWl0cyI6eyJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJpbXBvcnRzIjotMSwiZXhwb3J0cyI6LTEsIndpbGRjYXJkcyI6dHJ1ZSwiY29ubiI6LTEsImxlYWYiOi0xfSwiZGVmYXVsdF9wZXJtaXNzaW9ucyI6eyJwdWIiOnt9LCJzdWIiOnt9fSwiYXV0aG9yaXphdGlvbiI6e30sInR5cGUiOiJhY2NvdW50IiwidmVyc2lvbiI6Mn19.AFbpHhn0eht07x-z_QdJOGF2mpmRyEm2MUZR5ZhTZ6m5uPAsOJh1coNqyLUDdrA7n1sLHKpwb26tzbZm4Ws7BA
-}
-
-default_sentinel one
\ No newline at end of file
diff --git a/server/configs/reload/defaultsentinel_2.conf b/server/configs/reload/defaultsentinel_2.conf
deleted file mode 100644
index 4f70f9e2..00000000
--- a/server/configs/reload/defaultsentinel_2.conf
+++ /dev/null
@@ -1,16 +0,0 @@
-listen:   127.0.0.1:-1
-
-// Operator "O"
-operator: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJDUUsyV1c3SkhGVVhNR0g1TlFEWUlJT0UyWTRRWlZMVlFTS1ZKTVZITzZZRFBMWVg2Q1hRIiwiaWF0IjoxNzQ0NzI2MzA4LCJpc3MiOiJPRDQzNElRTVdGQTNTNzVBS1kyMlQ3UkdLRkdSSkRBNFFaU0xQRFFCRVZQVktONktBUTRTWVVMNiIsIm5hbWUiOiJPIiwic3ViIjoiT0Q0MzRJUU1XRkEzUzc1QUtZMjJUN1JHS0ZHUkpEQTRRWlNMUERRQkVWUFZLTjZLQVE0U1lVTDYiLCJuYXRzIjp7InR5cGUiOiJvcGVyYXRvciIsInZlcnNpb24iOjJ9fQ.DKuOHqWHgooC6x_vFaY66hLY5ZxoDKk2VVq71-gl_FDn2o_Amu_gQAxYvqLsNZvtlggWF-vKq944xcZmp0N1Dg
-
-resolver: MEMORY
-
-resolver_preload: {
-  // Account "SYS"
-  AB3GSO5OVHXQJYNNEMX4SGZCS3SPMNMDX4YA65M3SQX7XXHI4NFM2TF6: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiJJSEtXN0RLTk9MRVY2T1hVUUZGQlRCN1pWM0NHNzc2M0RYVFdGNEsyU0NUVkNWUDNOSEFBIiwiaWF0IjoxNzQ0NzMwMDcyLCJpc3MiOiJPRDQzNElRTVdGQTNTNzVBS1kyMlQ3UkdLRkdSSkRBNFFaU0xQRFFCRVZQVktONktBUTRTWVVMNiIsIm5hbWUiOiJTWVMiLCJzdWIiOiJBQjNHU081T1ZIWFFKWU5ORU1YNFNHWkNTM1NQTU5NRFg0WUE2NU0zU1FYN1hYSEk0TkZNMlRGNiIsIm5hdHMiOnsibGltaXRzIjp7InN1YnMiOi0xLCJkYXRhIjotMSwicGF5bG9hZCI6LTEsImltcG9ydHMiOi0xLCJleHBvcnRzIjotMSwid2lsZGNhcmRzIjp0cnVlLCJjb25uIjotMSwibGVhZiI6LTF9LCJkZWZhdWx0X3Blcm1pc3Npb25zIjp7InB1YiI6e30sInN1YiI6e319LCJhdXRob3JpemF0aW9uIjp7fSwidHlwZSI6ImFjY291bnQiLCJ2ZXJzaW9uIjoyfX0.cteeeg1_aw8FNGp6EMS1sGmggC165ZJ0QRynaHlg1j43ZrjJ5qZ43trW5BxbthY5EcplqmnvcCC7cmO7INLQDg
-
-  // Account "A"
-  ACX3MVLHNZ4SJZQFR72E2RQ7BUIDTQAAFOUAZLDQ7TK44RS55OSVSWU5: eyJ0eXAiOiJKV1QiLCJhbGciOiJlZDI1NTE5LW5rZXkifQ.eyJqdGkiOiIyV1NPTDIzQzNISllLTTc2TkJBTzU0Tlc3QldNVU81QVo3Rk5ORE9XUVlRVVhORkxNTldBIiwiaWF0IjoxNzQ0NzI2MzEzLCJpc3MiOiJPRDQzNElRTVdGQTNTNzVBS1kyMlQ3UkdLRkdSSkRBNFFaU0xQRFFCRVZQVktONktBUTRTWVVMNiIsIm5hbWUiOiJBIiwic3ViIjoiQUNYM01WTEhOWjRTSlpRRlI3MkUyUlE3QlVJRFRRQUFGT1VBWkxEUTdUSzQ0UlM1NU9TVlNXVTUiLCJuYXRzIjp7ImxpbWl0cyI6eyJzdWJzIjotMSwiZGF0YSI6LTEsInBheWxvYWQiOi0xLCJpbXBvcnRzIjotMSwiZXhwb3J0cyI6LTEsIndpbGRjYXJkcyI6dHJ1ZSwiY29ubiI6LTEsImxlYWYiOi0xfSwiZGVmYXVsdF9wZXJtaXNzaW9ucyI6eyJwdWIiOnt9LCJzdWIiOnt9fSwiYXV0aG9yaXphdGlvbiI6e30sInR5cGUiOiJhY2NvdW50IiwidmVyc2lvbiI6Mn19.AFbpHhn0eht07x-z_QdJOGF2mpmRyEm2MUZR5ZhTZ6m5uPAsOJh1coNqyLUDdrA7n1sLHKpwb26tzbZm4Ws7BA
-}
-
-default_sentinel two
\ No newline at end of file
diff --git a/server/jwt.go b/server/jwt.go
index e8da5213..04d7dc60 100644
--- a/server/jwt.go
+++ b/server/jwt.go
@@ -70,11 +70,20 @@ func wipeSlice(buf []byte) {
 func validateTrustedOperators(o *Options) error {
 	if len(o.TrustedOperators) == 0 {
 		// if we have no operator, default sentinel shouldn't be set
-		if o.DefaultSentinel != "" {
+		if o.DefaultSentinel != _EMPTY_ {
 			return fmt.Errorf("default sentinel requires operators and accounts")
 		}
 		return nil
 	}
+	if o.DefaultSentinel != _EMPTY_ {
+		juc, err := jwt.DecodeUserClaims(o.DefaultSentinel)
+		if err != nil {
+			return fmt.Errorf("default sentinel JWT not valid")
+		}
+		if !juc.BearerToken {
+			return fmt.Errorf("default sentinel must be a bearer token")
+		}
+	}
 	if o.AccountResolver == nil {
 		return fmt.Errorf("operators require an account resolver to be configured")
 	}



Test patch:
diff --git a/server/jwt_test.go b/server/jwt_test.go
index 331e3117..043a113e 100644
--- a/server/jwt_test.go
+++ b/server/jwt_test.go
@@ -7155,7 +7155,7 @@ func TestDefaultSentinelUser(t *testing.T) {
 	uPub, err := uKP.PublicKey()
 	require_NoError(t, err)
 	uc := jwt.NewUserClaims(uPub)
-	uc.BearerToken = true
+	uc.BearerToken = false
 	uc.Name = "sentinel"
 	sentinelToken, err := uc.Encode(aKP)
 	require_NoError(t, err)
@@ -7168,6 +7168,25 @@ func TestDefaultSentinelUser(t *testing.T) {
 			default_sentinel: %s
 `, ojwt, sysPub, preloadConfig, sentinelToken)))
 
+	// test non-bearer sentinel is rejected
+	opts, err := ProcessConfigFile(conf)
+	require_NoError(t, err)
+	_, err = NewServer(opts)
+	require_Error(t, err, fmt.Errorf("default sentinel must be a bearer token"))
+
+	// correct and start server
+	uc.BearerToken = true
+	sentinelToken, err = uc.Encode(aKP)
+	require_NoError(t, err)
+	conf = createConfFile(t, []byte(fmt.Sprintf(`
+            listen: 127.0.0.1:4747
+            operator: %s
+            system_account: %s
+            resolver: MEM
+            resolver_preload: %s
+			default_sentinel: %s
+`, ojwt, sysPub, preloadConfig, sentinelToken)))
+
 	ns, _ = RunServerWithConfig(conf)
 	defer ns.Shutdown()
 	nc, err := nats.Connect(ns.ClientURL())
diff --git a/server/reload_test.go b/server/reload_test.go
index 3d267154..5af6548d 100644
--- a/server/reload_test.go
+++ b/server/reload_test.go
@@ -602,22 +602,53 @@ func TestConfigReloadRotateTLSMultiCert(t *testing.T) {
 }
 
 func TestConfigReloadDefaultSentinel(t *testing.T) {
-	server, opts, config := runReloadServerWithConfig(t, "./configs/reload/defaultsentinel_1.conf")
-	defer server.Shutdown()
+	var err error
+	preload := make(map[string]string)
 
-	if opts.DefaultSentinel != "one" {
-		t.Fatalf("Expected default sentinel to be 'one', got %s", opts.DefaultSentinel)
-	}
+	_, sysPub, sysAC := NewJwtAccountClaim("SYS")
+	preload[sysPub], err = sysAC.Encode(oKp)
+	require_NoError(t, err)
+
+	aKP, aPub, aAC := NewJwtAccountClaim("A")
+	preload[aPub], err = aAC.Encode(oKp)
+	require_NoError(t, err)
+
+	preloadConfig, err := json.MarshalIndent(preload, "", " ")
+	require_NoError(t, err)
 
-	changeCurrentConfigContent(t, config, "./configs/reload/defaultsentinel_2.conf")
+	uKP, err := nkeys.CreateUser()
+	require_NoError(t, err)
+	uPub, err := uKP.PublicKey()
+	require_NoError(t, err)
+	uc := jwt.NewUserClaims(uPub)
+	uc.BearerToken = true
+	uc.Name = "sentinel"
+	sentinelToken, err := uc.Encode(aKP)
+	require_NoError(t, err)
+	content := func() []byte {
+		return []byte(fmt.Sprintf(`
+            listen: 127.0.0.1:4747
+            operator: %s
+            system_account: %s
+            resolver: MEM
+            resolver_preload: %s
+			default_sentinel: %s
+`, ojwt, sysPub, preloadConfig, sentinelToken))
+	}
+
+	server, opts, config := runReloadServerWithContent(t, content())
+	defer server.Shutdown()
+	require_Equal(t, opts.DefaultSentinel, sentinelToken)
+
+	uc.Name = "sentinel-updated"
+	sentinelToken, err = uc.Encode(aKP)
+	require_NoError(t, err)
+	changeCurrentConfigContentWithNewContent(t, config, content())
 	if err := server.Reload(); err != nil {
 		t.Fatalf("Error reloading config: %v", err)
 	}
-
 	opts = server.getOpts()
-	if opts.DefaultSentinel != "two" {
-		t.Fatalf("Expected default sentinel to be 'two', got %s", opts.DefaultSentinel)
-	}
+	require_Equal(t, opts.DefaultSentinel, sentinelToken)
 }
 
 // Ensure Reload supports single user authentication config changes. Test this