Repository: nats-io/nats-server
Pull request #1771 (2020-12-12 00:23:50)

[FIXED] Config reload for gateways/leaf remote TLS configurations
Presence of TLS config in any remote gateway or leafnode would
cause the config reload to fail (because TLS config internal
content may change which fails the DeepEqual check).

This PR excludes the TLS configs in such case to check for
changes in gateways and leafnodes.

Although GW and LN config reload is technically supported, this
PR updates the internal remotes' TLS configuration so that
changes/updates to TLS certificates would take effect after
a configuration reload.

Signed-off-by: Ivan Kozlovic <ivan@synadia.com>



Patch:
diff --git a/docker/Dockerfile.nightly b/docker/Dockerfile.nightly
index 0bdb242d..7f68bcbe 100644
--- a/docker/Dockerfile.nightly
+++ b/docker/Dockerfile.nightly
@@ -4,7 +4,7 @@ RUN apk add --update git
 RUN mkdir -p src/github.com/nats-io && \
     cd src/github.com/nats-io/ && \
     git clone https://github.com/nats-io/natscli.git && \
-    cd natscli/nats && \
+    cd natscli && \
     go build -o /nats
 
 RUN go get github.com/nats-io/nsc
diff --git a/server/gateway.go b/server/gateway.go
index 5f7a2903..1629998c 100644
--- a/server/gateway.go
+++ b/server/gateway.go
@@ -370,6 +370,29 @@ func (s *Server) newGateway(opts *Options) error {
 	return nil
 }
 
+// Update remote gateways TLS configurations after a config reload.
+func (g *srvGateway) updateRemotesTLSConfig(opts *Options) {
+	g.Lock()
+	defer g.Unlock()
+
+	for _, ro := range opts.Gateway.Gateways {
+		if ro.Name == g.name {
+			continue
+		}
+		if cfg, ok := g.remotes[ro.Name]; ok {
+			cfg.Lock()
+			// If TLS config is in remote, use that one, otherwise,
+			// use the TLS config from the main block.
+			if ro.TLSConfig != nil {
+				cfg.TLSConfig = ro.TLSConfig.Clone()
+			} else if opts.Gateway.TLSConfig != nil {
+				cfg.TLSConfig = opts.Gateway.TLSConfig.Clone()
+			}
+			cfg.Unlock()
+		}
+	}
+}
+
 // Returns the Gateway's name of this server.
 func (g *srvGateway) getName() string {
 	g.RLock()
diff --git a/server/leafnode.go b/server/leafnode.go
index 61264f77..a448c331 100644
--- a/server/leafnode.go
+++ b/server/leafnode.go
@@ -105,7 +105,10 @@ func (c *client) isHubLeafNode() bool {
 // This will spin up go routines to solicit the remote leaf node connections.
 func (s *Server) solicitLeafNodeRemotes(remotes []*RemoteLeafOpts) {
 	for _, r := range remotes {
+		s.mu.Lock()
 		remote := newLeafNodeCfg(r)
+		s.leafRemoteCfgs = append(s.leafRemoteCfgs, remote)
+		s.mu.Unlock()
 		s.startGoRoutine(func() { s.connectToRemoteLeafNode(remote, true) })
 	}
 }
@@ -209,6 +212,32 @@ func validateLeafNodeAuthOptions(o *Options) error {
 	return nil
 }
 
+// Update remote LeafNode TLS configurations after a config reload.
+func (s *Server) updateRemoteLeafNodesTLSConfig(opts *Options) {
+	max := len(opts.LeafNode.Remotes)
+	if max == 0 {
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Changes in the list of remote leaf nodes is not supported.
+	// However, make sure that we don't go over the arrays.
+	if len(s.leafRemoteCfgs) < max {
+		max = len(s.leafRemoteCfgs)
+	}
+	for i := 0; i < max; i++ {
+		ro := opts.LeafNode.Remotes[i]
+		cfg := s.leafRemoteCfgs[i]
+		if ro.TLSConfig != nil {
+			cfg.Lock()
+			cfg.TLSConfig = ro.TLSConfig.Clone()
+			cfg.Unlock()
+		}
+	}
+}
+
 func (s *Server) reConnectToRemoteLeafNode(remote *leafNodeCfg) {
 	delay := s.getOpts().LeafNode.ReconnectInterval
 	select {
@@ -724,13 +753,16 @@ func (s *Server) createLeafNode(conn net.Conn, remote *leafNodeCfg) *client {
 		}
 
 		// Do TLS here as needed.
-		tlsRequired := remote.TLS || remote.TLSConfig != nil
+		remote.RLock()
+		remoteTLSConfig := remote.TLSConfig
+		tlsRequired := remote.TLS || remoteTLSConfig != nil
+		remote.RUnlock()
 		if tlsRequired {
 			c.Debugf("Starting TLS leafnode client handshake")
 			// Specify the ServerName we are expecting.
 			var tlsConfig *tls.Config
-			if remote.TLSConfig != nil {
-				tlsConfig = remote.TLSConfig.Clone()
+			if remoteTLSConfig != nil {
+				tlsConfig = remoteTLSConfig.Clone()
 			} else {
 				tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12}
 			}
diff --git a/server/mqtt.go b/server/mqtt.go
index 65fef7f7..b2470ff8 100644
--- a/server/mqtt.go
+++ b/server/mqtt.go
@@ -175,7 +175,6 @@ type mqttPersistedSession struct {
 }
 
 type mqttRetainedMsg struct {
-	Topic  []byte `json:"topic,omitempty"`
 	Msg    []byte `json:"msg,omitempty"`
 	Flags  byte   `json:"flags,omitempty"`
 	Source string `json:"source,omitempty"`
@@ -233,7 +232,6 @@ type mqttWriter struct {
 
 type mqttWill struct {
 	topic   []byte
-	subject []byte
 	message []byte
 	qos     byte
 	retain  bool
@@ -242,12 +240,9 @@ type mqttWill struct {
 type mqttFilter struct {
 	filter string
 	qos    byte
-	// Used only for tracing and should not be used after parsing of (un)sub protocols.
-	ttopic []byte
 }
 
 type mqttPublish struct {
-	topic   []byte
 	subject []byte
 	msg     []byte
 	sz      int
@@ -552,12 +547,12 @@ func (c *client) mqttParse(buf []byte) error {
 			var subs []*subscription
 			pi, filters, err = c.mqttParseSubs(r, b, pl)
 			if trace {
-				c.traceInOp("SUBSCRIBE", errOrTrace(err, mqttSubscribeTrace(pi, filters)))
+				c.traceInOp("SUBSCRIBE", errOrTrace(err, mqttSubscribeTrace(filters)))
 			}
 			if err == nil {
 				subs, err = c.mqttProcessSubs(filters)
 				if err == nil && trace {
-					c.traceOutOp("SUBACK", []byte(fmt.Sprintf("pi=%v", pi)))
+					c.traceOutOp("SUBACK", []byte(mqttSubscribeTrace(filters)))
 				}
 			}
 			if err == nil {
@@ -569,12 +564,12 @@ func (c *client) mqttParse(buf []byte) error {
 			var filters []*mqttFilter
 			pi, filters, err = c.mqttParseUnsubs(r, b, pl)
 			if trace {
-				c.traceInOp("UNSUBSCRIBE", errOrTrace(err, mqttUnsubscribeTrace(pi, filters)))
+				c.traceInOp("UNSUBSCRIBE", errOrTrace(err, mqttUnsubscribeTrace(filters)))
 			}
 			if err == nil {
 				err = c.mqttProcessUnsubs(filters)
 				if err == nil && trace {
-					c.traceOutOp("UNSUBACK", []byte(fmt.Sprintf("pi=%v", pi)))
+					c.traceOutOp("UNSUBACK", []byte(strconv.FormatInt(int64(pi), 10)))
 				}
 			}
 			if err == nil {
@@ -1054,13 +1049,13 @@ func (as *mqttAccountSessionManager) serializeRetainedMsgsForSub(sess *mqttSessi
 			pi := sess.getPubAckIdentifier(mqttGetQoS(rm.Flags), sub)
 			// Need to use the subject for the retained message, not the `sub` subject.
 			// We can find the published retained message in rm.sub.subject.
-			flags := mqttSerializePublishMsg(prm, pi, false, true, rm.Topic, rm.Msg)
+			flags := mqttSerializePublishMsg(prm, pi, false, true, string(rm.sub.subject), rm.Msg)
 			if trace {
 				pp := mqttPublish{
-					topic: rm.Topic,
-					flags: flags,
-					pi:    pi,
-					sz:    len(rm.Msg),
+					flags:   flags,
+					pi:      pi,
+					subject: rm.sub.subject,
+					sz:      len(rm.Msg),
 				}
 				c.traceOutOp("PUBLISH", []byte(mqttPubTrace(&pp)))
 			}
@@ -1391,9 +1386,7 @@ func (c *client) mqttParseConnect(r *mqttReader, pl int) (byte, *mqttConnectProt
 			retain: wretain,
 		}
 		var topic []byte
-		// Need to make a copy since we need to hold to this topic after the
-		// parsing of this protocol.
-		topic, err = r.readBytes("Will topic", true)
+		topic, err = r.readBytes("Will topic", false)
 		if err != nil {
 			return 0, nil, err
 		}
@@ -1403,17 +1396,24 @@ func (c *client) mqttParseConnect(r *mqttReader, pl int) (byte, *mqttConnectProt
 		if !utf8.Valid(topic) {
 			return 0, nil, fmt.Errorf("invalide utf8 for Will topic %q", topic)
 		}
-		cp.will.topic = topic
 		// Convert MQTT topic to NATS subject
-		if cp.will.subject, err = mqttTopicToNATSPubSubject(topic); err != nil {
+		var copied bool
+		copied, topic, err = mqttTopicToNATSPubSubject(topic)
+		if err != nil {
 			return 0, nil, err
 		}
-		// Now "will" message.
-		// Ask for a copy since we need to hold to this after parsing of this protocol.
-		cp.will.message, err = r.readBytes("Will message", true)
+		if !copied {
+			topic = copyBytes(topic)
+		}
+		cp.will.topic = topic
+		// Now will message
+		var msg []byte
+		msg, err = r.readBytes("Will message", false)
 		if err != nil {
 			return 0, nil, err
 		}
+		cp.will.message = make([]byte, 0, len(msg))
+		cp.will.message = append(cp.will.message, msg...)
 	}
 
 	if hasUser {
@@ -1596,8 +1596,7 @@ func (s *Server) mqttHandleWill(c *client) {
 		return
 	}
 	pp := c.mqtt.pp
-	pp.topic = will.topic
-	pp.subject = will.subject
+	pp.subject = will.topic
 	pp.msg = will.message
 	pp.sz = len(will.message)
 	pp.pi = 0
@@ -1628,19 +1627,19 @@ func (c *client) mqttParsePub(r *mqttReader, pl int, pp *mqttPublish) error {
 	start := r.pos
 
 	var err error
-	pp.topic, err = r.readBytes("topic", false)
+	pp.subject, err = r.readBytes("topic", false)
 	if err != nil {
 		return err
 	}
-	if len(pp.topic) == 0 {
+	if len(pp.subject) == 0 {
 		return fmt.Errorf("topic cannot be empty")
 	}
 	// Convert the topic to a NATS subject. This call will also check that
 	// there is no MQTT wildcards (Spec [MQTT-3.3.2-2] and [MQTT-4.7.1-1])
-	// Note that this may not result in a copy if there is no conversion.
-	// It is good because after the message is processed we won't have a
-	// reference to the buffer and we save a copy.
-	pp.subject, err = mqttTopicToNATSPubSubject(pp.topic)
+	// Note that this may not result in a copy if there is no special
+	// conversion. It is good because after the message is processed we
+	// won't have a reference to the buffer and we save a copy.
+	_, pp.subject, err = mqttTopicToNATSPubSubject(pp.subject)
 	if err != nil {
 		return err
 	}
@@ -1679,7 +1678,7 @@ func mqttPubTrace(pp *mqttPublish) string {
 		piStr = fmt.Sprintf(" pi=%v", pp.pi)
 	}
 	return fmt.Sprintf("%s dup=%v QoS=%v retain=%v size=%v%s",
-		pp.topic, dup, qos, retain, pp.sz, piStr)
+		pp.subject, dup, qos, retain, pp.sz, piStr)
 }
 
 func (s *Server) mqttProcessPub(c *client, pp *mqttPublish) {
@@ -1707,7 +1706,7 @@ func (s *Server) mqttProcessPub(c *client, pp *mqttPublish) {
 	if mqttGetQoS(pp.flags) > 0 {
 		// Since this is the fast path, we access the messages stream directly here
 		// without locking. All the fields mqtt.asm.mstream are immutable.
-		c.mqtt.asm.mstream.processInboundJetStreamMsg(nil, c, string(c.pa.subject), _EMPTY_, pp.msg)
+		c.mqtt.asm.mstream.processInboundJetStreamMsg(nil, c, string(c.pa.subject), "", pp.msg)
 	}
 	c.pa.subject, c.pa.hdr, c.pa.size, c.pa.szb = nil, -1, 0, nil
 }
@@ -1740,7 +1739,6 @@ func (c *client) mqttHandlePubRetain() {
 			// When coming from a publish protocol, `pp` is referencing a stack
 			// variable that itself possibly references the read buffer.
 			rm := &mqttRetainedMsg{
-				Topic:  copyBytes(pp.topic),
 				Msg:    copyBytes(pp.msg),
 				Flags:  pp.flags,
 				Source: c.opts.Username,
@@ -1986,10 +1984,9 @@ func (c *client) mqttParseSubsOrUnsubs(r *mqttReader, b byte, pl int, sub bool)
 			return 0, nil, fmt.Errorf("invalid utf8 for topic filter %q", filter)
 		}
 		var qos byte
-		topic := filter
 		// This won't return an error. We will find out if the subject
 		// is valid or not when trying to create the subscription.
-		filter, _ = mqttFilterToNATSSubject(filter)
+		_, filter, _ = mqttFilterToNATSSubject(filter)
 		if sub {
 			qos, err = r.readByte("QoS")
 			if err != nil {
@@ -2000,8 +1997,7 @@ func (c *client) mqttParseSubsOrUnsubs(r *mqttReader, b byte, pl int, sub bool)
 				return 0, nil, fmt.Errorf("subscribe QoS value must be 0, 1 or 2, got %v", qos)
 			}
 		}
-		f := &mqttFilter{ttopic: topic, filter: string(filter), qos: qos}
-		filters = append(filters, f)
+		filters = append(filters, &mqttFilter{string(filter), qos})
 	}
 	// Spec [MQTT-3.8.3-3], [MQTT-3.10.3-2]
 	if len(filters) == 0 {
@@ -2010,23 +2006,17 @@ func (c *client) mqttParseSubsOrUnsubs(r *mqttReader, b byte, pl int, sub bool)
 	return pi, filters, nil
 }
 
-func mqttSubscribeTrace(pi uint16, filters []*mqttFilter) string {
+func mqttSubscribeTrace(filters []*mqttFilter) string {
 	var sep string
-	sb := &strings.Builder{}
-	sb.WriteString("[")
+	trace := "["
 	for i, f := range filters {
-		sb.WriteString(sep)
-		sb.Write(f.ttopic)
-		sb.WriteString(" (")
-		sb.WriteString(f.filter)
-		sb.WriteString(") QoS=")
-		sb.WriteString(fmt.Sprintf("%v", f.qos))
+		trace += sep + fmt.Sprintf("%s QoS=%v", f.filter, f.qos)
 		if i == 0 {
 			sep = ", "
 		}
 	}
-	sb.WriteString(fmt.Sprintf("] pi=%v", pi))
-	return sb.String()
+	trace += "]"
+	return trace
 }
 
 func mqttDeliverMsgCb(sub *subscription, pc *client, subject, reply string, msg []byte) {
@@ -2035,7 +2025,6 @@ func mqttDeliverMsgCb(sub *subscription, pc *client, subject, reply string, msg
 	var pi uint16
 	var dup bool
 	var retained bool
-	var topic []byte
 
 	// This is the client associated with the subscription.
 	cc := sub.client
@@ -2072,13 +2061,10 @@ func mqttDeliverMsgCb(sub *subscription, pc *client, subject, reply string, msg
 			sess.mu.Unlock()
 			return
 		}
-		// TODO: We hae to conver the subject back to topic. One approach
-		// was the pass the parsed MQTT topic in jsPubMsg but it was decided
-		// to not do that for now.
-		topic = natsSubjectToMQTTTopic(string(pc.pa.deliver))
+		// In JS case, we need to use the pc.ca.deliver value as the subject.
+		subject = string(pc.pa.deliver)
 	} else if pc.isMqtt() {
 		// This is a MQTT publisher...
-		topic = pc.mqtt.pp.topic
 		ppFlags = pc.mqtt.pp.flags
 		pQoS = mqttGetQoS(ppFlags)
 		// If the QoS of published message and subscription is 1, then we return here to
@@ -2094,16 +2080,13 @@ func mqttDeliverMsgCb(sub *subscription, pc *client, subject, reply string, msg
 		if len(msg) > mqttMaxPayloadSize {
 			msg = msg[:mqttMaxPayloadSize]
 		}
-		// See TODO for JS producer. Right now, we need to do the conversion
-		// back to a subject everytime.
-		topic = natsSubjectToMQTTTopic(subject)
 	}
 	sess.mu.Unlock()
 
 	sw := mqttWriter{}
 	w := &sw
 
-	flags := mqttSerializePublishMsg(w, pi, dup, retained, topic, msg)
+	flags := mqttSerializePublishMsg(w, pi, dup, retained, subject, msg)
 
 	cc.mu.Lock()
 	if sub.mqtt.prm != nil {
@@ -2114,10 +2097,10 @@ func mqttDeliverMsgCb(sub *subscription, pc *client, subject, reply string, msg
 	pc.addToPCD(cc)
 	if cc.trace {
 		pp := mqttPublish{
-			topic: topic,
-			flags: flags,
-			pi:    pi,
-			sz:    len(msg),
+			flags:   flags,
+			pi:      pi,
+			subject: []byte(subject),
+			sz:      len(msg),
 		}
 		cc.traceOutOp("PUBLISH", []byte(mqttPubTrace(&pp)))
 	}
@@ -2125,7 +2108,8 @@ func mqttDeliverMsgCb(sub *subscription, pc *client, subject, reply string, msg
 }
 
 // Serializes to the given writer the message for the given subject.
-func mqttSerializePublishMsg(w *mqttWriter, pi uint16, dup, retained bool, topic, msg []byte) byte {
+func mqttSerializePublishMsg(w *mqttWriter, pi uint16, dup, retained bool, subject string, msg []byte) byte {
+	topic := natsSubjectToMQTTTopic(subject)
 
 	// Compute len (will have to add packet id if message is sent as QoS>=1)
 	pkLen := 2 + len(topic) + len(msg)
@@ -2384,22 +2368,17 @@ func (c *client) mqttEnqueueUnsubAck(pi uint16) {
 	c.mu.Unlock()
 }
 
-func mqttUnsubscribeTrace(pi uint16, filters []*mqttFilter) string {
+func mqttUnsubscribeTrace(filters []*mqttFilter) string {
 	var sep string
-	sb := strings.Builder{}
-	sb.WriteString("[")
+	trace := "["
 	for i, f := range filters {
-		sb.WriteString(sep)
-		sb.Write(f.ttopic)
-		sb.WriteString(" (")
-		sb.WriteString(f.filter)
-		sb.WriteString(")")
+		trace += sep + f.filter
 		if i == 0 {
 			sep = ", "
 		}
 	}
-	sb.WriteString(fmt.Sprintf("] pi=%v", pi))
-	return sb.String()
+	trace += "]"
+	return trace
 }
 
 //////////////////////////////////////////////////////////////////////////////
@@ -2435,17 +2414,17 @@ func errOrTrace(err error, trace string) []byte {
 
 // Converts an MQTT Topic Name to a NATS Subject (used by PUBLISH)
 // See mqttToNATSSubjectConversion() for details.
-func mqttTopicToNATSPubSubject(mt []byte) ([]byte, error) {
+func mqttTopicToNATSPubSubject(mt []byte) (bool, []byte, error) {
 	return mqttToNATSSubjectConversion(mt, false)
 }
 
 // Converts an MQTT Topic Filter to a NATS Subject (used by SUBSCRIBE)
 // See mqttToNATSSubjectConversion() for details.
-func mqttFilterToNATSSubject(filter []byte) ([]byte, error) {
+func mqttFilterToNATSSubject(filter []byte) (bool, []byte, error) {
 	return mqttToNATSSubjectConversion(filter, true)
 }
 
-// Converts an MQTT Topic Name or Filter to a NATS Subject.
+// Converts an MQTT Topic Name or Filter to a NATS Subject
 // In MQTT:
 // - a Topic Name does not have wildcard (PUBLISH uses only topic names).
 // - a Topic Filter can include wildcards (SUBSCRIBE uses those).
@@ -2459,72 +2438,67 @@ func mqttFilterToNATSSubject(filter []byte) ([]byte, error) {
 // - '/' is replaced with '.' for all other conditions (foo/bar -> foo.bar)
 // - '.' and ' ' cause an error to be returned.
 //
-// If there is no need to convert anything (say "foo" remains "foo"), then
-// the no memory is allocated and the returned slice is the original `mt`.
-func mqttToNATSSubjectConversion(mt []byte, wcOk bool) ([]byte, error) {
-	var cp bool
-	var j int
-	res := mt
+// If a copy occurred, the returned boolean will indicate this condition.
+func mqttToNATSSubjectConversion(mt []byte, wcOk bool) (bool, []byte, error) {
+	var res = mt
+	var newSlice bool
 
-	makeCopy := func(i int) {
-		cp = true
-		res = make([]byte, 0, len(mt)+10)
-		if i > 0 {
-			res = append(res, mt[:i]...)
+	copyTopic := func(pos int) []byte {
+		if newSlice && cap(res) > pos+2 {
+			return res
 		}
+		newSlice = true
+		b := make([]byte, len(res)+10)
+		copy(b, res[:pos])
+		res = b
+		return res
 	}
 
+	var j int
 	end := len(mt) - 1
 	for i := 0; i < len(mt); i++ {
 		switch mt[i] {
 		case mqttTopicLevelSep:
 			if i == 0 || res[j-1] == btsep {
-				if !cp {
-					makeCopy(0)
-				}
-				res = append(res, mqttTopicLevelSep, btsep)
+				res = copyTopic(0)
+				res[j] = mqttTopicLevelSep
 				j++
+				res[j] = btsep
 			} else if i == end || mt[i+1] == mqttTopicLevelSep {
-				if !cp {
-					makeCopy(i)
-				}
-				res = append(res, btsep, mqttTopicLevelSep)
+				res = copyTopic(j)
+				res[j] = btsep
 				j++
+				res[j] = mqttTopicLevelSep
 			} else {
-				if !cp {
-					makeCopy(i)
-				}
-				res = append(res, btsep)
+				res[j] = btsep
 			}
 		case btsep, ' ':
 			// As of now, we cannot support '.' or ' ' in the MQTT topic/filter.
-			return nil, fmt.Errorf("characters ' ' and '.' not supported for MQTT topics")
+			return false, nil, fmt.Errorf("characters ' ' and '.' not supported for MQTT topics")
 		case mqttSingleLevelWC, mqttMultiLevelWC:
 			if !wcOk {
 				// Spec [MQTT-3.3.2-2] and [MQTT-4.7.1-1]
 				// The wildcard characters can be used in Topic Filters, but MUST NOT be used within a Topic Name
-				return nil, fmt.Errorf("wildcards not allowed in publish's topic: %q", mt)
-			}
-			if !cp {
-				makeCopy(i)
+				return false, nil, fmt.Errorf("wildcards not allowed in publish's topic: %q", mt)
 			}
 			if mt[i] == mqttSingleLevelWC {
-				res = append(res, pwc)
+				res[j] = pwc
 			} else {
-				res = append(res, fwc)
+				res[j] = fwc
 			}
 		default:
-			if cp {
-				res = append(res, mt[i])
+			if newSlice {
+				res[j] = mt[i]
 			}
 		}
 		j++
 	}
-	if cp && res[j-1] == btsep {
-		res = append(res, mqttTopicLevelSep)
+	if newSlice && res[j-1] == btsep {
+		res = copyTopic(j)
+		res[j] = mqttTopicLevelSep
 		j++
 	}
-	return res[:j], nil
+	return newSlice, res[:j], nil
 }
 
 // Converts a NATS subject to MQTT topic. This is for publish
diff --git a/server/reload.go b/server/reload.go
index 790bb80a..a5339920 100644
--- a/server/reload.go
+++ b/server/reload.go
@@ -916,6 +916,13 @@ func (s *Server) diffOptions(newOpts *Options) ([]option, error) {
 			tmpNew := newValue.(GatewayOpts)
 			tmpOld.TLSConfig = nil
 			tmpNew.TLSConfig = nil
+
+			// Need to do the same for remote gateways' TLS configs.
+			// But we can't just set remotes' TLSConfig to nil otherwise this
+			// would lose the real TLS configuration.
+			tmpOld.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpOld.Gateways)
+			tmpNew.Gateways = copyRemoteGWConfigsWithoutTLSConfig(tmpNew.Gateways)
+
 			// If there is really a change prevents reload.
 			if !reflect.DeepEqual(tmpOld, tmpNew) {
 				// See TODO(ik) note below about printing old/new values.
@@ -929,6 +936,12 @@ func (s *Server) diffOptions(newOpts *Options) ([]option, error) {
 			tmpOld.TLSConfig = nil
 			tmpNew.TLSConfig = nil
 
+			// Need to do the same for remote leafnodes' TLS configs.
+			// But we can't just set remotes' TLSConfig to nil otherwise this
+			// would lose the real TLS configuration.
+			tmpOld.Remotes = copyRemoteLNConfigWithoutTLSConfig(tmpOld.Remotes)
+			tmpNew.Remotes = copyRemoteLNConfigWithoutTLSConfig(tmpNew.Remotes)
+
 			// Special check for leafnode remotes changes which are not supported right now.
 			leafRemotesChanged := func(a, b LeafNodeOpts) bool {
 				if len(tmpOld.Remotes) != len(tmpNew.Remotes) {
@@ -1045,6 +1058,37 @@ func (s *Server) diffOptions(newOpts *Options) ([]option, error) {
 	return diffOpts, nil
 }
 
+func copyRemoteGWConfigsWithoutTLSConfig(current []*RemoteGatewayOpts) []*RemoteGatewayOpts {
+	l := len(current)
+	if l == 0 {
+		return nil
+	}
+	rgws := make([]*RemoteGatewayOpts, 0, l)
+	for _, rcfg := range current {
+		cp := *rcfg
+		cp.TLSConfig = nil
+		rgws = append(rgws, &cp)
+	}
+	return rgws
+}
+
+func copyRemoteLNConfigWithoutTLSConfig(current []*RemoteLeafOpts) []*RemoteLeafOpts {
+	l := len(current)
+	if l == 0 {
+		return nil
+	}
+	rlns := make([]*RemoteLeafOpts, 0, l)
+	for _, rcfg := range current {
+		cp := *rcfg
+		cp.TLSConfig = nil
+		// This is set only when processing a CONNECT, so reset here so that we
+		// don't fail the DeepEqual comparison.
+		cp.TLS = false
+		rlns = append(rlns, &cp)
+	}
+	return rlns
+}
+
 func (s *Server) applyOptions(ctx *reloadContext, opts []option) {
 	var (
 		reloadLogging      = false
@@ -1080,6 +1124,16 @@ func (s *Server) applyOptions(ctx *reloadContext, opts []option) {
 	if reloadClusterPerms {
 		s.reloadClusterPermissions(ctx.oldClusterPerms)
 	}
+	// For remote gateways and leafnodes, make sure that their TLS configuration
+	// is updated (since the config is "captured" early and changes would otherwise
+	// not be visible).
+	newOpts := s.getOpts()
+	if s.gateway.enabled {
+		s.gateway.updateRemotesTLSConfig(newOpts)
+	}
+	if len(newOpts.LeafNode.Remotes) > 0 {
+		s.updateRemoteLeafNodesTLSConfig(newOpts)
+	}
 
 	s.Noticef("Reloaded server configuration")
 }
@@ -1209,7 +1263,9 @@ func (s *Server) reloadAuthorization() {
 			s.accounts.Store(s.sys.account.Name, s.sys.account)
 		}
 		// Double check any JetStream configs.
-		checkJetStream = true
+		// For now, seems like JS as a whole cannot be enabled/disabled with
+		// config reload, so check only if was started with JS enabled.
+		checkJetStream = s.js != nil
 	} else if s.opts.AccountResolver != nil {
 		s.configureResolver()
 		if _, ok := s.accResolver.(*MemAccResolver); ok {
diff --git a/server/server.go b/server/server.go
index d64cddd2..24b51150 100644
--- a/server/server.go
+++ b/server/server.go
@@ -151,6 +151,7 @@ type Server struct {
 		resolver    netResolver
 		dialTimeout time.Duration
 	}
+	leafRemoteCfgs []*leafNodeCfg
 
 	quitCh           chan struct{}
 	shutdownComplete chan struct{}


Test patch:
diff --git a/server/gateway_test.go b/server/gateway_test.go
index 44d4adc6..fd0eeeb8 100644
--- a/server/gateway_test.go
+++ b/server/gateway_test.go
@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"runtime"
 	"strconv"
 	"strings"
@@ -6248,3 +6249,86 @@ func TestGatewayPings(t *testing.T) {
 		}
 	}
 }
+
+func TestGatewayTLSConfigReload(t *testing.T) {
+	template := `
+		listen: 127.0.0.1:-1
+		gateway {
+			name: "A"
+			listen: "127.0.0.1:-1"
+			tls {
+				cert_file: "../test/configs/certs/server-cert.pem"
+				key_file:  "../test/configs/certs/server-key.pem"
+				%s
+				timeout: 2
+			}
+		}
+	`
+	confA := createConfFile(t, []byte(fmt.Sprintf(template, "")))
+	defer os.Remove(confA)
+
+	srvA, optsA := RunServerWithConfig(confA)
+	defer srvA.Shutdown()
+
+	optsB := testGatewayOptionsFromToWithTLS(t, "B", "A", []string{fmt.Sprintf("nats://127.0.0.1:%d", optsA.Gateway.Port)})
+	srvB := runGatewayServer(optsB)
+	defer srvB.Shutdown()
+
+	waitForGatewayFailedConnect(t, srvB, "A", true, time.Second)
+
+	reloadUpdateConfig(t, srvA, confA, fmt.Sprintf(template, `ca_file:   "../test/configs/certs/ca.pem"`))
+
+	waitForInboundGateways(t, srvA, 1, time.Second)
+	waitForOutboundGateways(t, srvA, 1, time.Second)
+	waitForInboundGateways(t, srvB, 1, time.Second)
+	waitForOutboundGateways(t, srvB, 1, time.Second)
+}
+
+func TestGatewayTLSConfigReloadForRemote(t *testing.T) {
+	SetGatewaysSolicitDelay(5 * time.Millisecond)
+	defer ResetGatewaysSolicitDelay()
+
+	optsA := testGatewayOptionsWithTLS(t, "A")
+	srvA := runGatewayServer(optsA)
+	defer srvA.Shutdown()
+
+	template := `
+		listen: 127.0.0.1:-1
+		gateway {
+			name: "B"
+			listen: "127.0.0.1:-1"
+			tls {
+				cert_file: "../test/configs/certs/server-cert.pem"
+				key_file:  "../test/configs/certs/server-key.pem"
+				ca_file:   "../test/configs/certs/ca.pem"
+				timeout: 2
+			}
+			gateways [
+				{
+					name: "A"
+					url: "nats://127.0.0.1:%d"
+					tls {
+						cert_file: "../test/configs/certs/server-cert.pem"
+						key_file:  "../test/configs/certs/server-key.pem"
+						%s
+						timeout: 2
+					}
+				}
+			]
+		}
+	`
+	confB := createConfFile(t, []byte(fmt.Sprintf(template, optsA.Gateway.Port, "")))
+	defer os.Remove(confB)
+
+	srvB, _ := RunServerWithConfig(confB)
+	defer srvB.Shutdown()
+
+	waitForGatewayFailedConnect(t, srvB, "A", true, time.Second)
+
+	reloadUpdateConfig(t, srvB, confB, fmt.Sprintf(template, optsA.Gateway.Port, `ca_file: "../test/configs/certs/ca.pem"`))
+
+	waitForInboundGateways(t, srvA, 1, time.Second)
+	waitForOutboundGateways(t, srvA, 1, time.Second)
+	waitForInboundGateways(t, srvB, 1, time.Second)
+	waitForOutboundGateways(t, srvB, 1, time.Second)
+}
diff --git a/server/leafnode_test.go b/server/leafnode_test.go
index 497f464d..7f93d4dd 100644
--- a/server/leafnode_test.go
+++ b/server/leafnode_test.go
@@ -2146,3 +2146,140 @@ func TestLeafNodeOperatorBadCfg(t *testing.T) {
 		})
 	}
 }
+
+func TestLeafNodeTLSConfigReload(t *testing.T) {
+	template := `
+		listen: 127.0.0.1:-1
+		leaf {
+			listen: "127.0.0.1:-1"
+			tls {
+				cert_file: "../test/configs/certs/server-cert.pem"
+				key_file:  "../test/configs/certs/server-key.pem"
+				%s
+				timeout: 2
+				verify: true
+			}
+		}
+	`
+	confA := createConfFile(t, []byte(fmt.Sprintf(template, "")))
+	defer os.Remove(confA)
+
+	srvA, optsA := RunServerWithConfig(confA)
+	defer srvA.Shutdown()
+
+	lg := &captureErrorLogger{errCh: make(chan string, 10)}
+	srvA.SetLogger(lg, false, false)
+
+	confB := createConfFile(t, []byte(fmt.Sprintf(`
+		listen: -1
+		leaf {
+			remotes [
+				{
+					url: "tls://127.0.0.1:%d"
+					tls {
+						cert_file: "../test/configs/certs/server-cert.pem"
+						key_file:  "../test/configs/certs/server-key.pem"
+						ca_file:   "../test/configs/certs/ca.pem"
+					}
+				}
+			]
+		}
+	`, optsA.LeafNode.Port)))
+	defer os.Remove(confB)
+
+	optsB, err := ProcessConfigFile(confB)
+	if err != nil {
+		t.Fatalf("Error processing config file: %v", err)
+	}
+	optsB.LeafNode.ReconnectInterval = 50 * time.Millisecond
+	optsB.NoLog, optsB.NoSigs = true, true
+
+	srvB := RunServer(optsB)
+	defer srvB.Shutdown()
+
+	// Wait for the error
+	select {
+	case err := <-lg.errCh:
+		if !strings.Contains(err, "unknown") {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("Did not get TLS error")
+	}
+
+	// Add the CA to srvA
+	reloadUpdateConfig(t, srvA, confA, fmt.Sprintf(template, `ca_file: "../test/configs/certs/ca.pem"`))
+
+	// Now make sure that srvB can create a LN connection.
+	checkFor(t, 3*time.Second, 10*time.Millisecond, func() error {
+		if nln := srvB.NumLeafNodes(); nln != 1 {
+			return fmt.Errorf("Number of leaf nodes is %d", nln)
+		}
+		return nil
+	})
+}
+
+func TestLeafNodeTLSConfigReloadForRemote(t *testing.T) {
+	confA := createConfFile(t, []byte(`
+		listen: 127.0.0.1:-1
+		leaf {
+			listen: "127.0.0.1:-1"
+			tls {
+				cert_file: "../test/configs/certs/server-cert.pem"
+				key_file:  "../test/configs/certs/server-key.pem"
+				ca_file: "../test/configs/certs/ca.pem"
+				timeout: 2
+				verify: true
+			}
+		}
+	`))
+	defer os.Remove(confA)
+
+	srvA, optsA := RunServerWithConfig(confA)
+	defer srvA.Shutdown()
+
+	lg := &captureErrorLogger{errCh: make(chan string, 10)}
+	srvA.SetLogger(lg, false, false)
+
+	template := `
+		listen: -1
+		leaf {
+			remotes [
+				{
+					url: "tls://127.0.0.1:%d"
+					tls {
+						cert_file: "../test/configs/certs/server-cert.pem"
+						key_file:  "../test/configs/certs/server-key.pem"
+						%s
+					}
+				}
+			]
+		}
+	`
+	confB := createConfFile(t, []byte(fmt.Sprintf(template, optsA.LeafNode.Port, "")))
+	defer os.Remove(confB)
+
+	srvB, _ := RunServerWithConfig(confB)
+	defer srvB.Shutdown()
+
+	// Wait for the error
+	select {
+	case err := <-lg.errCh:
+		if !strings.Contains(err, "bad certificate") {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("Did not get TLS error")
+	}
+
+	// Add the CA to srvB
+	reloadUpdateConfig(t, srvB, confB, fmt.Sprintf(template, optsA.LeafNode.Port, `ca_file: "../test/configs/certs/ca.pem"`))
+
+	// Now make sure that srvB can create a LN connection.
+	checkFor(t, 2*time.Second, 10*time.Millisecond, func() error {
+		if nln := srvB.NumLeafNodes(); nln != 1 {
+			return fmt.Errorf("Number of leaf nodes is %d", nln)
+		}
+		return nil
+	})
+}
diff --git a/server/mqtt_test.go b/server/mqtt_test.go
index 7ce5895a..bbf04ee4 100644
--- a/server/mqtt_test.go
+++ b/server/mqtt_test.go
@@ -1464,7 +1464,7 @@ func TestMQTTTopicAndSubjectConversion(t *testing.T) {
 		{"foo.bar", "foo.bar", "", "not supported"},
 	} {
 		t.Run(test.name, func(t *testing.T) {
-			res, err := mqttTopicToNATSPubSubject([]byte(test.mqttTopic))
+			_, res, err := mqttTopicToNATSPubSubject([]byte(test.mqttTopic))
 			if test.err != _EMPTY_ {
 				if err == nil || !strings.Contains(err.Error(), test.err) {
 					t.Fatalf("Expected error %q, got %q", test.err, err.Error())
@@ -1476,7 +1476,7 @@ func TestMQTTTopicAndSubjectConversion(t *testing.T) {
 				t.Fatalf("Expected subject %q got %q", test.natsSubject, toNATS)
 			}
 
-			res = natsSubjectToMQTTTopic(string(res))
+			res = natsSubjectToMQTTTopic(toNATS)
 			backToMQTT := string(res)
 			if backToMQTT != test.mqttTopic {
 				t.Fatalf("Expected topic %q got %q (NATS conversion was %q)", test.mqttTopic, backToMQTT, toNATS)
@@ -1503,22 +1503,15 @@ func TestMQTTFilterConversion(t *testing.T) {
 		{"single level wildcard", "foo/+/+", "foo.*.*"},
 		{"single level wildcard", "foo/+/+/", "foo.*.*./"},
 		{"single level wildcard", "foo/+/+/bar", "foo.*.*.bar"},
-		{"single level wildcard", "foo//+", "foo./.*"},
-		{"single level wildcard", "foo//+/", "foo./.*./"},
-		{"single level wildcard", "foo//+//", "foo./.*././"},
-		{"single level wildcard", "foo//+//bar", "foo./.*./.bar"},
-		{"single level wildcard", "foo///+///bar", "foo././.*././.bar"},
 
 		{"multi level wildcard", "#", ">"},
 		{"multi level wildcard", "/#", "/.>"},
 		{"multi level wildcard", "/foo/#", "/.foo.>"},
 		{"multi level wildcard", "foo/#", "foo.>"},
-		{"multi level wildcard", "foo//#", "foo./.>"},
-		{"multi level wildcard", "foo///#", "foo././.>"},
 		{"multi level wildcard", "foo/bar/#", "foo.bar.>"},
 	} {
 		t.Run(test.name, func(t *testing.T) {
-			res, err := mqttFilterToNATSSubject([]byte(test.mqttTopic))
+			_, res, err := mqttFilterToNATSSubject([]byte(test.mqttTopic))
 			if err != nil {
 				t.Fatalf("Error: %v", err)
 			}
diff --git a/test/gateway_test.go b/test/gateway_test.go
index 820f4b08..b4d97251 100644
--- a/test/gateway_test.go
+++ b/test/gateway_test.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"regexp"
 	"testing"
 	"time"
@@ -596,6 +597,7 @@ func TestGatewayTLSMixedIPAndDNS(t *testing.T) {
 			listen: "127.0.0.1:-1"
 		}
 	`))
+	defer os.Remove(confA1)
 	srvA1, optsA1 := RunServerWithConfig(confA1)
 	defer srvA1.Shutdown()
 
@@ -620,6 +622,7 @@ func TestGatewayTLSMixedIPAndDNS(t *testing.T) {
 	`
 	confA2 := createConfFile(t, []byte(fmt.Sprintf(confA2Template,
 		optsA1.Cluster.Host, optsA1.Cluster.Port)))
+	defer os.Remove(confA2)
 	srvA2, optsA2 := RunServerWithConfig(confA2)
 	defer srvA2.Shutdown()