Repository: lightningnetwork/lnd
Pull request #3705 (2019-11-11 11:29:02)

routing+htlcswitch: improve failed payment logging
This pr contains several changes to improve the information that is logged when a payment attempt fails, which will hopefully help us in debugging routing/mission control related issues.


Patch:
diff --git a/contractcourt/briefcase.go b/contractcourt/briefcase.go
index 22adcbdc1..989786f1a 100644
--- a/contractcourt/briefcase.go
+++ b/contractcourt/briefcase.go
@@ -440,7 +440,7 @@ func (b *boltArbitratorLog) CommitState(s ArbitratorState) error {
 //
 // NOTE: Part of the ContractResolver interface.
 func (b *boltArbitratorLog) FetchUnresolvedContracts() ([]ContractResolver, error) {
-	resolverCfg := ResolverConfig{
+	resKit := ResolverKit{
 		ChannelArbitratorConfig: b.cfg,
 		Checkpoint:              b.checkpointContract,
 	}
@@ -469,38 +469,56 @@ func (b *boltArbitratorLog) FetchUnresolvedContracts() ([]ContractResolver, erro
 
 			switch resType {
 			case resolverTimeout:
-				res, err = newTimeoutResolverFromReader(
-					resReader, resolverCfg,
-				)
+				timeoutRes := &htlcTimeoutResolver{}
+				if err := timeoutRes.Decode(resReader); err != nil {
+					return err
+				}
+				timeoutRes.AttachResolverKit(resKit)
+
+				res = timeoutRes
 
 			case resolverSuccess:
-				res, err = newSuccessResolverFromReader(
-					resReader, resolverCfg,
-				)
+				successRes := &htlcSuccessResolver{}
+				if err := successRes.Decode(resReader); err != nil {
+					return err
+				}
+
+				res = successRes
 
 			case resolverOutgoingContest:
-				res, err = newOutgoingContestResolverFromReader(
-					resReader, resolverCfg,
-				)
+				outContestRes := &htlcOutgoingContestResolver{
+					htlcTimeoutResolver: htlcTimeoutResolver{},
+				}
+				if err := outContestRes.Decode(resReader); err != nil {
+					return err
+				}
+
+				res = outContestRes
 
 			case resolverIncomingContest:
-				res, err = newIncomingContestResolverFromReader(
-					resReader, resolverCfg,
-				)
+				inContestRes := &htlcIncomingContestResolver{
+					htlcSuccessResolver: htlcSuccessResolver{},
+				}
+				if err := inContestRes.Decode(resReader); err != nil {
+					return err
+				}
+
+				res = inContestRes
 
 			case resolverUnilateralSweep:
-				res, err = newCommitSweepResolverFromReader(
-					resReader, resolverCfg,
-				)
+				sweepRes := &commitSweepResolver{}
+				if err := sweepRes.Decode(resReader); err != nil {
+					return err
+				}
+
+				res = sweepRes
 
 			default:
 				return fmt.Errorf("unknown resolver type: %v", resType)
 			}
 
-			if err != nil {
-				return err
-			}
-
+			resKit.Quit = make(chan struct{})
+			res.AttachResolverKit(resKit)
 			contracts = append(contracts, res)
 			return nil
 		})
diff --git a/contractcourt/channel_arbitrator.go b/contractcourt/channel_arbitrator.go
index 66730ed11..3bd669027 100644
--- a/contractcourt/channel_arbitrator.go
+++ b/contractcourt/channel_arbitrator.go
@@ -1685,7 +1685,7 @@ func (c *ChannelArbitrator) prepContractResolutions(
 
 	// We'll create the resolver kit that we'll be cloning for each
 	// resolver so they each can do their duty.
-	resolverCfg := ResolverConfig{
+	resKit := ResolverKit{
 		ChannelArbitratorConfig: c.cfg,
 		Checkpoint: func(res ContractResolver) error {
 			return c.log.InsertUnresolvedContracts(res)
@@ -1733,10 +1733,14 @@ func (c *ChannelArbitrator) prepContractResolutions(
 					continue
 				}
 
-				resolver := newSuccessResolver(
-					resolution, height,
-					htlc.RHash, htlc.Amt, resolverCfg,
-				)
+				resKit.Quit = make(chan struct{})
+				resolver := &htlcSuccessResolver{
+					htlcResolution:  resolution,
+					broadcastHeight: height,
+					payHash:         htlc.RHash,
+					htlcAmt:         htlc.Amt,
+					ResolverKit:     resKit,
+				}
 				htlcResolvers = append(htlcResolvers, resolver)
 			}
 
@@ -1757,10 +1761,14 @@ func (c *ChannelArbitrator) prepContractResolutions(
 					continue
 				}
 
-				resolver := newTimeoutResolver(
-					resolution, height, htlc.HtlcIndex,
-					htlc.Amt, resolverCfg,
-				)
+				resKit.Quit = make(chan struct{})
+				resolver := &htlcTimeoutResolver{
+					htlcResolution:  resolution,
+					broadcastHeight: height,
+					htlcIndex:       htlc.HtlcIndex,
+					htlcAmt:         htlc.Amt,
+					ResolverKit:     resKit,
+				}
 				htlcResolvers = append(htlcResolvers, resolver)
 			}
 
@@ -1790,11 +1798,18 @@ func (c *ChannelArbitrator) prepContractResolutions(
 					ChanID: c.cfg.ShortChanID,
 				}
 
-				resolver := newIncomingContestResolver(
-					htlc.RefundTimeout, circuitKey,
-					resolution, height, htlc.RHash,
-					htlc.Amt, resolverCfg,
-				)
+				resKit.Quit = make(chan struct{})
+				resolver := &htlcIncomingContestResolver{
+					htlcExpiry: htlc.RefundTimeout,
+					circuitKey: circuitKey,
+					htlcSuccessResolver: htlcSuccessResolver{
+						htlcResolution:  resolution,
+						broadcastHeight: height,
+						payHash:         htlc.RHash,
+						htlcAmt:         htlc.Amt,
+						ResolverKit:     resKit,
+					},
+				}
 				htlcResolvers = append(htlcResolvers, resolver)
 			}
 
@@ -1816,10 +1831,16 @@ func (c *ChannelArbitrator) prepContractResolutions(
 					continue
 				}
 
-				resolver := newOutgoingContestResolver(
-					resolution, height, htlc.HtlcIndex,
-					htlc.Amt, resolverCfg,
-				)
+				resKit.Quit = make(chan struct{})
+				resolver := &htlcOutgoingContestResolver{
+					htlcTimeoutResolver: htlcTimeoutResolver{
+						htlcResolution:  resolution,
+						broadcastHeight: height,
+						htlcIndex:       htlc.HtlcIndex,
+						htlcAmt:         htlc.Amt,
+						ResolverKit:     resKit,
+					},
+				}
 				htlcResolvers = append(htlcResolvers, resolver)
 			}
 		}
@@ -1829,10 +1850,14 @@ func (c *ChannelArbitrator) prepContractResolutions(
 	// a resolver to sweep our commitment output (but only if it wasn't
 	// trimmed).
 	if contractResolutions.CommitResolution != nil {
-		resolver := newCommitSweepResolver(
-			*contractResolutions.CommitResolution,
-			height, c.cfg.ChanPoint, resolverCfg,
-		)
+		resKit.Quit = make(chan struct{})
+		resolver := &commitSweepResolver{
+			commitResolution: *contractResolutions.CommitResolution,
+			broadcastHeight:  height,
+			chanPoint:        c.cfg.ChanPoint,
+			ResolverKit:      resKit,
+		}
+
 		htlcResolvers = append(htlcResolvers, resolver)
 	}
 
diff --git a/contractcourt/commit_sweep_resolver.go b/contractcourt/commit_sweep_resolver.go
index 3b7f3b855..4e5712728 100644
--- a/contractcourt/commit_sweep_resolver.go
+++ b/contractcourt/commit_sweep_resolver.go
@@ -36,20 +36,7 @@ type commitSweepResolver struct {
 	// chanPoint is the channel point of the original contract.
 	chanPoint wire.OutPoint
 
-	contractResolverKit
-}
-
-// newCommitSweepResolver instantiates a new direct commit output resolver.
-func newCommitSweepResolver(res lnwallet.CommitOutputResolution,
-	broadcastHeight uint32,
-	chanPoint wire.OutPoint, resCfg ResolverConfig) *commitSweepResolver {
-
-	return &commitSweepResolver{
-		contractResolverKit: *newContractResolverKit(resCfg),
-		commitResolution:    res,
-		broadcastHeight:     broadcastHeight,
-		chanPoint:           chanPoint,
-	}
+	ResolverKit
 }
 
 // ResolverKey returns an identifier which should be globally unique for this
@@ -93,7 +80,7 @@ func (c *commitSweepResolver) Resolve() (ContractResolver, error) {
 			return nil, errResolverShuttingDown
 		}
 
-	case <-c.quit:
+	case <-c.Quit:
 		return nil, errResolverShuttingDown
 	}
 
@@ -151,7 +138,7 @@ func (c *commitSweepResolver) Resolve() (ContractResolver, error) {
 
 			log.Infof("ChannelPoint(%v) commit tx is fully resolved by "+
 				"sweep tx: %v", c.chanPoint, sweepResult.Tx.TxHash())
-		case <-c.quit:
+		case <-c.Quit:
 			return nil, errResolverShuttingDown
 		}
 
@@ -193,7 +180,7 @@ func (c *commitSweepResolver) Resolve() (ContractResolver, error) {
 			log.Errorf("unable to Checkpoint: %v", err)
 			return nil, err
 		}
-	case <-c.quit:
+	case <-c.Quit:
 		return nil, errResolverShuttingDown
 	}
 
@@ -219,7 +206,7 @@ func (c *commitSweepResolver) Resolve() (ContractResolver, error) {
 		log.Infof("ChannelPoint(%v) commit tx is fully resolved, at height: %v",
 			c.chanPoint, confInfo.BlockHeight)
 
-	case <-c.quit:
+	case <-c.Quit:
 		return nil, errResolverShuttingDown
 	}
 
@@ -234,7 +221,7 @@ func (c *commitSweepResolver) Resolve() (ContractResolver, error) {
 //
 // NOTE: Part of the ContractResolver interface.
 func (c *commitSweepResolver) Stop() {
-	close(c.quit)
+	close(c.Quit)
 }
 
 // IsResolved returns true if the stored state in the resolve is fully
@@ -275,40 +262,44 @@ func (c *commitSweepResolver) Encode(w io.Writer) error {
 	return nil
 }
 
-// newCommitSweepResolverFromReader attempts to decode an encoded
-// ContractResolver from the passed Reader instance, returning an active
-// ContractResolver instance.
-func newCommitSweepResolverFromReader(r io.Reader, resCfg ResolverConfig) (
-	*commitSweepResolver, error) {
-
-	c := &commitSweepResolver{
-		contractResolverKit: *newContractResolverKit(resCfg),
-	}
-
+// Decode attempts to decode an encoded ContractResolver from the passed Reader
+// instance, returning an active ContractResolver instance.
+//
+// NOTE: Part of the ContractResolver interface.
+func (c *commitSweepResolver) Decode(r io.Reader) error {
 	if err := decodeCommitResolution(r, &c.commitResolution); err != nil {
-		return nil, err
+		return err
 	}
 
 	if err := binary.Read(r, endian, &c.resolved); err != nil {
-		return nil, err
+		return err
 	}
 	if err := binary.Read(r, endian, &c.broadcastHeight); err != nil {
-		return nil, err
+		return err
 	}
 	_, err := io.ReadFull(r, c.chanPoint.Hash[:])
 	if err != nil {
-		return nil, err
+		return err
 	}
 	err = binary.Read(r, endian, &c.chanPoint.Index)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	// Previously a sweep tx was deserialized at this point. Refactoring
 	// removed this, but keep in mind that this data may still be present in
 	// the database.
 
-	return c, nil
+	return nil
+}
+
+// AttachResolverKit should be called once a resolved is successfully decoded
+// from its stored format. This struct delivers a generic tool kit that
+// resolvers need to complete their duty.
+//
+// NOTE: Part of the ContractResolver interface.
+func (c *commitSweepResolver) AttachResolverKit(r ResolverKit) {
+	c.ResolverKit = r
 }
 
 // A compile time assertion to ensure commitSweepResolver meets the
diff --git a/contractcourt/contract_resolvers.go b/contractcourt/contract_resolvers.go
index 7248bba05..b39a04d4a 100644
--- a/contractcourt/contract_resolvers.go
+++ b/contractcourt/contract_resolvers.go
@@ -46,6 +46,16 @@ type ContractResolver interface {
 	// passed Writer.
 	Encode(w io.Writer) error
 
+	// Decode attempts to decode an encoded ContractResolver from the
+	// passed Reader instance, returning an active ContractResolver
+	// instance.
+	Decode(r io.Reader) error
+
+	// AttachResolverKit should be called once a resolved is successfully
+	// decoded from its stored format. This struct delivers a generic tool
+	// kit that resolvers need to complete their duty.
+	AttachResolverKit(ResolverKit)
+
 	// Stop signals the resolver to cancel any current resolution
 	// processes, and suspend.
 	Stop()
@@ -59,9 +69,10 @@ type reportingContractResolver interface {
 	report() *ContractReport
 }
 
-// ResolverConfig contains the externally supplied configuration items that are
-// required by a ContractResolver implementation.
-type ResolverConfig struct {
+// ResolverKit is meant to be used as a mix-in struct to be embedded within a
+// given ContractResolver implementation. It contains all the items that a
+// resolver requires to carry out its duties.
+type ResolverKit struct {
 	// ChannelArbitratorConfig contains all the interfaces and closures
 	// required for the resolver to interact with outside sub-systems.
 	ChannelArbitratorConfig
@@ -70,23 +81,8 @@ type ResolverConfig struct {
 	// should write the state of the resolver to persistent storage, and
 	// return a non-nil error upon success.
 	Checkpoint func(ContractResolver) error
-}
-
-// contractResolverKit is meant to be used as a mix-in struct to be embedded within a
-// given ContractResolver implementation. It contains all the common items that
-// a resolver requires to carry out its duties.
-type contractResolverKit struct {
-	ResolverConfig
-
-	quit chan struct{}
-}
 
-// newContractResolverKit instantiates the mix-in struct.
-func newContractResolverKit(cfg ResolverConfig) *contractResolverKit {
-	return &contractResolverKit{
-		ResolverConfig: cfg,
-		quit:           make(chan struct{}),
-	}
+	Quit chan struct{}
 }
 
 var (
diff --git a/contractcourt/htlc_success_resolver.go b/contractcourt/htlc_success_resolver.go
index 73bf5c5bc..d1c8716cb 100644
--- a/contractcourt/htlc_success_resolver.go
+++ b/contractcourt/htlc_success_resolver.go
@@ -52,22 +52,7 @@ type htlcSuccessResolver struct {
 	// account any fees that may have to be paid if it goes on chain.
 	htlcAmt lnwire.MilliSatoshi
 
-	contractResolverKit
-}
-
-// newSuccessResolver instanties a new htlc success resolver.
-func newSuccessResolver(res lnwallet.IncomingHtlcResolution,
-	broadcastHeight uint32, payHash lntypes.Hash,
-	htlcAmt lnwire.MilliSatoshi,
-	resCfg ResolverConfig) *htlcSuccessResolver {
-
-	return &htlcSuccessResolver{
-		contractResolverKit: *newContractResolverKit(resCfg),
-		htlcResolution:      res,
-		broadcastHeight:     broadcastHeight,
-		payHash:             payHash,
-		htlcAmt:             htlcAmt,
-	}
+	ResolverKit
 }
 
 // ResolverKey returns an identifier which should be globally unique for this
@@ -188,7 +173,7 @@ func (h *htlcSuccessResolver) Resolve() (ContractResolver, error) {
 				return nil, errResolverShuttingDown
 			}
 
-		case <-h.quit:
+		case <-h.Quit:
 			return nil, errResolverShuttingDown
 		}
 
@@ -253,7 +238,7 @@ func (h *htlcSuccessResolver) Resolve() (ContractResolver, error) {
 			return nil, errResolverShuttingDown
 		}
 
-	case <-h.quit:
+	case <-h.Quit:
 		return nil, errResolverShuttingDown
 	}
 
@@ -266,7 +251,7 @@ func (h *htlcSuccessResolver) Resolve() (ContractResolver, error) {
 //
 // NOTE: Part of the ContractResolver interface.
 func (h *htlcSuccessResolver) Stop() {
-	close(h.quit)
+	close(h.Quit)
 }
 
 // IsResolved returns true if the stored state in the resolve is fully
@@ -305,37 +290,41 @@ func (h *htlcSuccessResolver) Encode(w io.Writer) error {
 	return nil
 }
 
-// newSuccessResolverFromReader attempts to decode an encoded ContractResolver
-// from the passed Reader instance, returning an active ContractResolver
-// instance.
-func newSuccessResolverFromReader(r io.Reader, resCfg ResolverConfig) (
-	*htlcSuccessResolver, error) {
-
-	h := &htlcSuccessResolver{
-		contractResolverKit: *newContractResolverKit(resCfg),
-	}
-
+// Decode attempts to decode an encoded ContractResolver from the passed Reader
+// instance, returning an active ContractResolver instance.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcSuccessResolver) Decode(r io.Reader) error {
 	// First we'll decode our inner HTLC resolution.
 	if err := decodeIncomingResolution(r, &h.htlcResolution); err != nil {
-		return nil, err
+		return err
 	}
 
 	// Next, we'll read all the fields that are specified to the contract
 	// resolver.
 	if err := binary.Read(r, endian, &h.outputIncubating); err != nil {
-		return nil, err
+		return err
 	}
 	if err := binary.Read(r, endian, &h.resolved); err != nil {
-		return nil, err
+		return err
 	}
 	if err := binary.Read(r, endian, &h.broadcastHeight); err != nil {
-		return nil, err
+		return err
 	}
 	if _, err := io.ReadFull(r, h.payHash[:]); err != nil {
-		return nil, err
+		return err
 	}
 
-	return h, nil
+	return nil
+}
+
+// AttachResolverKit should be called once a resolved is successfully decoded
+// from its stored format. This struct delivers a generic tool kit that
+// resolvers need to complete their duty.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcSuccessResolver) AttachResolverKit(r ResolverKit) {
+	h.ResolverKit = r
 }
 
 // A compile time assertion to ensure htlcSuccessResolver meets the
diff --git a/contractcourt/htlc_timeout_resolver.go b/contractcourt/htlc_timeout_resolver.go
index 248cfa36b..9d1c92d72 100644
--- a/contractcourt/htlc_timeout_resolver.go
+++ b/contractcourt/htlc_timeout_resolver.go
@@ -48,22 +48,7 @@ type htlcTimeoutResolver struct {
 	// account any fees that may have to be paid if it goes on chain.
 	htlcAmt lnwire.MilliSatoshi
 
-	contractResolverKit
-}
-
-// newTimeoutResolver instantiates a new timeout htlc resolver.
-func newTimeoutResolver(res lnwallet.OutgoingHtlcResolution,
-	broadcastHeight uint32, htlcIndex uint64,
-	htlcAmt lnwire.MilliSatoshi,
-	resCfg ResolverConfig) *htlcTimeoutResolver {
-
-	return &htlcTimeoutResolver{
-		contractResolverKit: *newContractResolverKit(resCfg),
-		htlcResolution:      res,
-		broadcastHeight:     broadcastHeight,
-		htlcIndex:           htlcIndex,
-		htlcAmt:             htlcAmt,
-	}
+	ResolverKit
 }
 
 // ResolverKey returns an identifier which should be globally unique for this
@@ -289,7 +274,7 @@ func (h *htlcTimeoutResolver) Resolve() (ContractResolver, error) {
 				return errResolverShuttingDown
 			}
 
-		case <-h.quit:
+		case <-h.Quit:
 			return errResolverShuttingDown
 		}
 
@@ -327,7 +312,7 @@ func (h *htlcTimeoutResolver) Resolve() (ContractResolver, error) {
 			return nil, errResolverShuttingDown
 		}
 
-	case <-h.quit:
+	case <-h.Quit:
 		return nil, errResolverShuttingDown
 	}
 
@@ -380,7 +365,7 @@ func (h *htlcTimeoutResolver) Resolve() (ContractResolver, error) {
 //
 // NOTE: Part of the ContractResolver interface.
 func (h *htlcTimeoutResolver) Stop() {
-	close(h.quit)
+	close(h.Quit)
 }
 
 // IsResolved returns true if the stored state in the resolve is fully
@@ -421,39 +406,43 @@ func (h *htlcTimeoutResolver) Encode(w io.Writer) error {
 	return nil
 }
 
-// newTimeoutResolverFromReader attempts to decode an encoded ContractResolver
-// from the passed Reader instance, returning an active ContractResolver
-// instance.
-func newTimeoutResolverFromReader(r io.Reader, resCfg ResolverConfig) (
-	*htlcTimeoutResolver, error) {
-
-	h := &htlcTimeoutResolver{
-		contractResolverKit: *newContractResolverKit(resCfg),
-	}
-
+// Decode attempts to decode an encoded ContractResolver from the passed Reader
+// instance, returning an active ContractResolver instance.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcTimeoutResolver) Decode(r io.Reader) error {
 	// First, we'll read out all the mandatory fields of the
 	// OutgoingHtlcResolution that we store.
 	if err := decodeOutgoingResolution(r, &h.htlcResolution); err != nil {
-		return nil, err
+		return err
 	}
 
 	// With those fields read, we can now read back the fields that are
 	// specific to the resolver itself.
 	if err := binary.Read(r, endian, &h.outputIncubating); err != nil {
-		return nil, err
+		return err
 	}
 	if err := binary.Read(r, endian, &h.resolved); err != nil {
-		return nil, err
+		return err
 	}
 	if err := binary.Read(r, endian, &h.broadcastHeight); err != nil {
-		return nil, err
+		return err
 	}
 
 	if err := binary.Read(r, endian, &h.htlcIndex); err != nil {
-		return nil, err
+		return err
 	}
 
-	return h, nil
+	return nil
+}
+
+// AttachResolverKit should be called once a resolved is successfully decoded
+// from its stored format. This struct delivers a generic tool kit that
+// resolvers need to complete their duty.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcTimeoutResolver) AttachResolverKit(r ResolverKit) {
+	h.ResolverKit = r
 }
 
 // A compile time assertion to ensure htlcTimeoutResolver meets the
diff --git a/go.mod b/go.mod
index 1014fef38..7dd469714 100644
--- a/go.mod
+++ b/go.mod
@@ -33,7 +33,7 @@ require (
 	github.com/juju/utils v0.0.0-20180820210520-bf9cc5bdd62d // indirect
 	github.com/juju/version v0.0.0-20180108022336-b64dbd566305 // indirect
 	github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec
-	github.com/lightninglabs/neutrino v0.10.1-0.20191111074637-d9b036bf675c
+	github.com/lightninglabs/neutrino v0.10.0
 	github.com/lightningnetwork/lightning-onion v0.0.0-20190909101754-850081b08b6a
 	github.com/lightningnetwork/lnd/queue v1.0.1
 	github.com/lightningnetwork/lnd/ticker v1.0.0
diff --git a/go.sum b/go.sum
index 9003927df..98e1313a5 100644
--- a/go.sum
+++ b/go.sum
@@ -134,8 +134,6 @@ github.com/lightninglabs/gozmq v0.0.0-20190710231225-cea2a031735d h1:tt8hwvxl6fk
 github.com/lightninglabs/gozmq v0.0.0-20190710231225-cea2a031735d/go.mod h1:vxmQPeIQxPf6Jf9rM8R+B4rKBqLA2AjttNxkFBL2Plk=
 github.com/lightninglabs/neutrino v0.10.0 h1:yWVy2cOCCXbKFdpYCE9vD1fWRJDd9FtGXhUws4l9RkU=
 github.com/lightninglabs/neutrino v0.10.0/go.mod h1:C3KhCMk1Mcx3j8v0qRVWM1Ow6rIJSvSPnUAq00ZNAfk=
-github.com/lightninglabs/neutrino v0.10.1-0.20191111074637-d9b036bf675c h1:FDuM1xfX51eK7LLeW6WCndeViiqLEKLnRL4P2ak1J/Q=
-github.com/lightninglabs/neutrino v0.10.1-0.20191111074637-d9b036bf675c/go.mod h1:C3KhCMk1Mcx3j8v0qRVWM1Ow6rIJSvSPnUAq00ZNAfk=
 github.com/lightningnetwork/lightning-onion v0.0.0-20190909101754-850081b08b6a h1:GoWPN4i4jTKRxhVNh9a2vvBBO1Y2seiJB+SopUYoKyo=
 github.com/lightningnetwork/lightning-onion v0.0.0-20190909101754-850081b08b6a/go.mod h1:rigfi6Af/KqsF7Za0hOgcyq2PNH4AN70AaMRxcJkff4=
 github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796 h1:sjOGyegMIhvgfq5oaue6Td+hxZuf3tDC8lAPrFldqFw=
diff --git a/htlcswitch/failure.go b/htlcswitch/failure.go
index 4526e3560..0d817a07d 100644
--- a/htlcswitch/failure.go
+++ b/htlcswitch/failure.go
@@ -30,10 +30,14 @@ type ForwardingError struct {
 // returned.
 func (f *ForwardingError) Error() string {
 	if f.ExtraMsg == "" {
-		return fmt.Sprintf("%v", f.FailureMessage)
+		return fmt.Sprintf(
+			"%v@%v", f.FailureMessage, f.FailureSourceIdx,
+		)
 	}
 
-	return fmt.Sprintf("%v: %v", f.FailureMessage, f.ExtraMsg)
+	return fmt.Sprintf(
+		"%v@%v: %v", f.FailureMessage, f.FailureSourceIdx, f.ExtraMsg,
+	)
 }
 
 // ErrorDecrypter is an interface that is used to decrypt the onion encrypted
diff --git a/macaroons/store.go b/macaroons/store.go
index 6b80ef5a9..7a38dd92e 100644
--- a/macaroons/store.go
+++ b/macaroons/store.go
@@ -5,7 +5,6 @@ import (
 	"crypto/rand"
 	"fmt"
 	"io"
-	"sync"
 
 	"github.com/coreos/bbolt"
 
@@ -48,8 +47,7 @@ var (
 type RootKeyStorage struct {
 	*bbolt.DB
 
-	encKeyMtx sync.RWMutex
-	encKey    *snacl.SecretKey
+	encKey *snacl.SecretKey
 }
 
 // NewRootKeyStorage creates a RootKeyStorage instance.
@@ -65,15 +63,12 @@ func NewRootKeyStorage(db *bbolt.DB) (*RootKeyStorage, error) {
 	}
 
 	// Return the DB wrapped in a RootKeyStorage object.
-	return &RootKeyStorage{DB: db, encKey: nil}, nil
+	return &RootKeyStorage{db, nil}, nil
 }
 
 // CreateUnlock sets an encryption key if one is not already set, otherwise it
 // checks if the password is correct for the stored encryption key.
 func (r *RootKeyStorage) CreateUnlock(password *[]byte) error {
-	r.encKeyMtx.Lock()
-	defer r.encKeyMtx.Unlock()
-
 	// Check if we've already unlocked the store; return an error if so.
 	if r.encKey != nil {
 		return ErrAlreadyUnlocked
@@ -124,9 +119,6 @@ func (r *RootKeyStorage) CreateUnlock(password *[]byte) error {
 
 // Get implements the Get method for the bakery.RootKeyStorage interface.
 func (r *RootKeyStorage) Get(_ context.Context, id []byte) ([]byte, error) {
-	r.encKeyMtx.RLock()
-	defer r.encKeyMtx.RUnlock()
-
 	if r.encKey == nil {
 		return nil, ErrStoreLocked
 	}
@@ -158,9 +150,6 @@ func (r *RootKeyStorage) Get(_ context.Context, id []byte) ([]byte, error) {
 // interface.
 // TODO(aakselrod): Add support for key rotation.
 func (r *RootKeyStorage) RootKey(_ context.Context) ([]byte, []byte, error) {
-	r.encKeyMtx.RLock()
-	defer r.encKeyMtx.RUnlock()
-
 	if r.encKey == nil {
 		return nil, nil, ErrStoreLocked
 	}
@@ -206,9 +195,6 @@ func (r *RootKeyStorage) RootKey(_ context.Context) ([]byte, []byte, error) {
 // Close closes the underlying database and zeroes the encryption key stored
 // in memory.
 func (r *RootKeyStorage) Close() error {
-	r.encKeyMtx.Lock()
-	defer r.encKeyMtx.Unlock()
-
 	if r.encKey != nil {
 		r.encKey.Zero()
 	}
diff --git a/routing/payment_lifecycle.go b/routing/payment_lifecycle.go
index 695a0d498..a8017eaa3 100644
--- a/routing/payment_lifecycle.go
+++ b/routing/payment_lifecycle.go
@@ -341,8 +341,8 @@ func (p *paymentLifecycle) sendPaymentAttempt(firstHop lnwire.ShortChannelID,
 		return err
 	}
 
-	log.Debugf("Payment %x (pid=%v) successfully sent to switch",
-		p.payment.PaymentHash, p.attempt.PaymentID)
+	log.Debugf("Payment %x (pid=%v) successfully sent to switch, route: %v",
+		p.payment.PaymentHash, p.attempt.PaymentID, &p.attempt.Route)
 
 	return nil
 }
diff --git a/routing/route/route.go b/routing/route/route.go
index 62511c3f1..6858f3b8d 100644
--- a/routing/route/route.go
+++ b/routing/route/route.go
@@ -340,15 +340,19 @@ func (r *Route) ToSphinxPath() (*sphinx.PaymentPath, error) {
 func (r *Route) String() string {
 	var b strings.Builder
 
+	amt := r.TotalAmount
 	for i, hop := range r.Hops {
 		if i > 0 {
-			b.WriteString(",")
+			b.WriteString(" -> ")
 		}
-		b.WriteString(strconv.FormatUint(hop.ChannelID, 10))
+		b.WriteString(fmt.Sprintf("%v (%v)",
+			strconv.FormatUint(hop.ChannelID, 10),
+			amt,
+		))
+		amt = hop.AmtToForward
 	}
 
-	return fmt.Sprintf("amt=%v, fees=%v, tl=%v, chans=%v",
-		r.TotalAmount-r.TotalFees(), r.TotalFees(), r.TotalTimeLock,
-		b.String(),
+	return fmt.Sprintf("%v, cltv %v",
+		b.String(), r.TotalTimeLock,
 	)
 }


Test patch:
diff --git a/contractcourt/htlc_incoming_contest_resolver.go b/contractcourt/htlc_incoming_contest_resolver.go
index 5a40369cb..2f3348e6e 100644
--- a/contractcourt/htlc_incoming_contest_resolver.go
+++ b/contractcourt/htlc_incoming_contest_resolver.go
@@ -5,12 +5,11 @@ import (
 	"errors"
 	"io"
 
-	"github.com/btcsuite/btcutil"
 	"github.com/lightningnetwork/lnd/channeldb"
 	"github.com/lightningnetwork/lnd/invoices"
+
+	"github.com/btcsuite/btcutil"
 	"github.com/lightningnetwork/lnd/lntypes"
-	"github.com/lightningnetwork/lnd/lnwallet"
-	"github.com/lightningnetwork/lnd/lnwire"
 )
 
 // htlcIncomingContestResolver is a ContractResolver that's able to resolve an
@@ -35,24 +34,6 @@ type htlcIncomingContestResolver struct {
 	htlcSuccessResolver
 }
 
-// newIncomingContestResolver instantiates a new incoming htlc contest resolver.
-func newIncomingContestResolver(htlcExpiry uint32,
-	circuitKey channeldb.CircuitKey, res lnwallet.IncomingHtlcResolution,
-	broadcastHeight uint32, payHash lntypes.Hash,
-	htlcAmt lnwire.MilliSatoshi,
-	resCfg ResolverConfig) *htlcIncomingContestResolver {
-
-	success := newSuccessResolver(
-		res, broadcastHeight, payHash, htlcAmt, resCfg,
-	)
-
-	return &htlcIncomingContestResolver{
-		htlcExpiry:          htlcExpiry,
-		circuitKey:          circuitKey,
-		htlcSuccessResolver: *success,
-	}
-}
-
 // Resolve attempts to resolve this contract. As we don't yet know of the
 // preimage for the contract, we'll wait for one of two things to happen:
 //
@@ -87,7 +68,7 @@ func (h *htlcIncomingContestResolver) Resolve() (ContractResolver, error) {
 			return nil, errResolverShuttingDown
 		}
 		currentHeight = newBlock.Height
-	case <-h.quit:
+	case <-h.Quit:
 		return nil, errResolverShuttingDown
 	}
 
@@ -258,7 +239,7 @@ func (h *htlcIncomingContestResolver) Resolve() (ContractResolver, error) {
 				return nil, h.Checkpoint(h)
 			}
 
-		case <-h.quit:
+		case <-h.Quit:
 			return nil, errResolverShuttingDown
 		}
 	}
@@ -290,7 +271,7 @@ func (h *htlcIncomingContestResolver) report() *ContractReport {
 //
 // NOTE: Part of the ContractResolver interface.
 func (h *htlcIncomingContestResolver) Stop() {
-	close(h.quit)
+	close(h.Quit)
 }
 
 // IsResolved returns true if the stored state in the resolve is fully
@@ -315,27 +296,27 @@ func (h *htlcIncomingContestResolver) Encode(w io.Writer) error {
 	return h.htlcSuccessResolver.Encode(w)
 }
 
-// newIncomingContestResolverFromReader attempts to decode an encoded ContractResolver
-// from the passed Reader instance, returning an active ContractResolver
-// instance.
-func newIncomingContestResolverFromReader(r io.Reader, resCfg ResolverConfig) (
-	*htlcIncomingContestResolver, error) {
-
-	h := &htlcIncomingContestResolver{}
-
+// Decode attempts to decode an encoded ContractResolver from the passed Reader
+// instance, returning an active ContractResolver instance.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcIncomingContestResolver) Decode(r io.Reader) error {
 	// We'll first read the one field unique to this resolver.
 	if err := binary.Read(r, endian, &h.htlcExpiry); err != nil {
-		return nil, err
+		return err
 	}
 
 	// Then we'll decode our internal resolver.
-	successResolver, err := newSuccessResolverFromReader(r, resCfg)
-	if err != nil {
-		return nil, err
-	}
-	h.htlcSuccessResolver = *successResolver
+	return h.htlcSuccessResolver.Decode(r)
+}
 
-	return h, nil
+// AttachResolverKit should be called once a resolved is successfully decoded
+// from its stored format. This struct delivers a generic tool kit that
+// resolvers need to complete their duty.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcIncomingContestResolver) AttachResolverKit(r ResolverKit) {
+	h.ResolverKit = r
 }
 
 // A compile time assertion to ensure htlcIncomingContestResolver meets the
diff --git a/contractcourt/htlc_incoming_resolver_test.go b/contractcourt/htlc_incoming_resolver_test.go
index 1f28f0a50..43d01817b 100644
--- a/contractcourt/htlc_incoming_resolver_test.go
+++ b/contractcourt/htlc_incoming_resolver_test.go
@@ -199,18 +199,17 @@ func newIncomingResolverTestContext(t *testing.T) *incomingResolverTestContext {
 		},
 	}
 
-	cfg := ResolverConfig{
-		ChannelArbitratorConfig: chainCfg,
-		Checkpoint: func(_ ContractResolver) error {
-			checkPointChan <- struct{}{}
-			return nil
-		},
-	}
 	resolver := &htlcIncomingContestResolver{
 		htlcSuccessResolver: htlcSuccessResolver{
-			contractResolverKit: *newContractResolverKit(cfg),
-			htlcResolution:      lnwallet.IncomingHtlcResolution{},
-			payHash:             testResHash,
+			ResolverKit: ResolverKit{
+				ChannelArbitratorConfig: chainCfg,
+				Checkpoint: func(_ ContractResolver) error {
+					checkPointChan <- struct{}{}
+					return nil
+				},
+			},
+			htlcResolution: lnwallet.IncomingHtlcResolution{},
+			payHash:        testResHash,
 		},
 		htlcExpiry: testHtlcExpiry,
 	}
diff --git a/contractcourt/htlc_outgoing_contest_resolver.go b/contractcourt/htlc_outgoing_contest_resolver.go
index 7388b45a8..789558a67 100644
--- a/contractcourt/htlc_outgoing_contest_resolver.go
+++ b/contractcourt/htlc_outgoing_contest_resolver.go
@@ -5,8 +5,6 @@ import (
 	"io"
 
 	"github.com/btcsuite/btcutil"
-	"github.com/lightningnetwork/lnd/lnwallet"
-	"github.com/lightningnetwork/lnd/lnwire"
 )
 
 // htlcOutgoingContestResolver is a ContractResolver that's able to resolve an
@@ -20,21 +18,6 @@ type htlcOutgoingContestResolver struct {
 	htlcTimeoutResolver
 }
 
-// newOutgoingContestResolver instantiates a new outgoing contested htlc
-// resolver.
-func newOutgoingContestResolver(res lnwallet.OutgoingHtlcResolution,
-	broadcastHeight uint32, htlcIndex uint64, htlcAmt lnwire.MilliSatoshi,
-	resCfg ResolverConfig) *htlcOutgoingContestResolver {
-
-	timeout := newTimeoutResolver(
-		res, broadcastHeight, htlcIndex, htlcAmt, resCfg,
-	)
-
-	return &htlcOutgoingContestResolver{
-		htlcTimeoutResolver: *timeout,
-	}
-}
-
 // Resolve commences the resolution of this contract. As this contract hasn't
 // yet timed out, we'll wait for one of two things to happen
 //
@@ -147,7 +130,7 @@ func (h *htlcOutgoingContestResolver) Resolve() (ContractResolver, error) {
 			// claimed.
 			return h.claimCleanUp(commitSpend)
 
-		case <-h.quit:
+		case <-h.Quit:
 			return nil, fmt.Errorf("resolver canceled")
 		}
 	}
@@ -179,7 +162,7 @@ func (h *htlcOutgoingContestResolver) report() *ContractReport {
 //
 // NOTE: Part of the ContractResolver interface.
 func (h *htlcOutgoingContestResolver) Stop() {
-	close(h.quit)
+	close(h.Quit)
 }
 
 // IsResolved returns true if the stored state in the resolve is fully
@@ -198,19 +181,21 @@ func (h *htlcOutgoingContestResolver) Encode(w io.Writer) error {
 	return h.htlcTimeoutResolver.Encode(w)
 }
 
-// newOutgoingContestResolverFromReader attempts to decode an encoded ContractResolver
-// from the passed Reader instance, returning an active ContractResolver
-// instance.
-func newOutgoingContestResolverFromReader(r io.Reader, resCfg ResolverConfig) (
-	*htlcOutgoingContestResolver, error) {
+// Decode attempts to decode an encoded ContractResolver from the passed Reader
+// instance, returning an active ContractResolver instance.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcOutgoingContestResolver) Decode(r io.Reader) error {
+	return h.htlcTimeoutResolver.Decode(r)
+}
 
-	h := &htlcOutgoingContestResolver{}
-	timeoutResolver, err := newTimeoutResolverFromReader(r, resCfg)
-	if err != nil {
-		return nil, err
-	}
-	h.htlcTimeoutResolver = *timeoutResolver
-	return h, nil
+// AttachResolverKit should be called once a resolved is successfully decoded
+// from its stored format. This struct delivers a generic tool kit that
+// resolvers need to complete their duty.
+//
+// NOTE: Part of the ContractResolver interface.
+func (h *htlcOutgoingContestResolver) AttachResolverKit(r ResolverKit) {
+	h.ResolverKit = r
 }
 
 // A compile time assertion to ensure htlcOutgoingContestResolver meets the
diff --git a/contractcourt/htlc_outgoing_contest_resolver_test.go b/contractcourt/htlc_outgoing_contest_resolver_test.go
index 1f7dbed13..244b438d3 100644
--- a/contractcourt/htlc_outgoing_contest_resolver_test.go
+++ b/contractcourt/htlc_outgoing_contest_resolver_test.go
@@ -122,18 +122,16 @@ func newOutgoingResolverTestContext(t *testing.T) *outgoingResolverTestContext {
 		},
 	}
 
-	cfg := ResolverConfig{
-		ChannelArbitratorConfig: chainCfg,
-		Checkpoint: func(_ ContractResolver) error {
-			checkPointChan <- struct{}{}
-			return nil
-		},
-	}
-
 	resolver := &htlcOutgoingContestResolver{
 		htlcTimeoutResolver: htlcTimeoutResolver{
-			contractResolverKit: *newContractResolverKit(cfg),
-			htlcResolution:      outgoingRes,
+			ResolverKit: ResolverKit{
+				ChannelArbitratorConfig: chainCfg,
+				Checkpoint: func(_ ContractResolver) error {
+					checkPointChan <- struct{}{}
+					return nil
+				},
+			},
+			htlcResolution: outgoingRes,
 		},
 	}
 
diff --git a/contractcourt/htlc_timeout_resolver_test.go b/contractcourt/htlc_timeout_resolver_test.go
index 981cefc41..7dcf025ad 100644
--- a/contractcourt/htlc_timeout_resolver_test.go
+++ b/contractcourt/htlc_timeout_resolver_test.go
@@ -237,17 +237,14 @@ func TestHtlcTimeoutResolver(t *testing.T) {
 			},
 		}
 
-		cfg := ResolverConfig{
-			ChannelArbitratorConfig: chainCfg,
-			Checkpoint: func(_ ContractResolver) error {
-				checkPointChan <- struct{}{}
-				return nil
-			},
-		}
 		resolver := &htlcTimeoutResolver{
-			contractResolverKit: *newContractResolverKit(
-				cfg,
-			),
+			ResolverKit: ResolverKit{
+				ChannelArbitratorConfig: chainCfg,
+				Checkpoint: func(_ ContractResolver) error {
+					checkPointChan <- struct{}{}
+					return nil
+				},
+			},
 		}
 		resolver.htlcResolution.SweepSignDesc = *fakeSignDesc
 
diff --git a/htlcswitch/link_test.go b/htlcswitch/link_test.go
index 2ad917067..06613aa1a 100644
--- a/htlcswitch/link_test.go
+++ b/htlcswitch/link_test.go
@@ -1248,8 +1248,14 @@ func TestChannelLinkMultiHopUnknownNextHop(t *testing.T) {
 		totalTimelock).Wait(30 * time.Second)
 	if err == nil {
 		t.Fatal("error haven't been received")
-	} else if err.Error() != lnwire.CodeUnknownNextPeer.String() {
-		t.Fatalf("wrong error have been received: %v", err)
+	}
+	fErr, ok := err.(*ForwardingError)
+	if !ok {
+		t.Fatalf("expected ForwardingError")
+	}
+	if _, ok = fErr.FailureMessage.(*lnwire.FailUnknownNextPeer); !ok {
+		t.Fatalf("wrong error has been received: %T",
+			fErr.FailureMessage)
 	}
 
 	// Wait for Alice to receive the revocation.