From dfabdef8edba96d0ee7ae302e0cbe69759ea664e Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 27 Sep 2022 13:00:35 +0200 Subject: [PATCH 01/22] add migration script --- fix_initialized_payments/main.go | 65 ++++++++++++++++++++ integration_tests/lnd_mock.go | 4 ++ integration_tests/subscription_start_test.go | 4 ++ lib/service/checkpayments.go | 42 +++++++++++++ lnd/interface.go | 1 + lnd/lnd.go | 19 +++++- 6 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 fix_initialized_payments/main.go create mode 100644 lib/service/checkpayments.go diff --git a/fix_initialized_payments/main.go b/fix_initialized_payments/main.go new file mode 100644 index 00000000..dd642d76 --- /dev/null +++ b/fix_initialized_payments/main.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "log" + + "github.com/getAlby/lndhub.go/db" + "github.com/getAlby/lndhub.go/lib" + "github.com/getAlby/lndhub.go/lib/service" + "github.com/getAlby/lndhub.go/lnd" + "github.com/kelseyhightower/envconfig" +) + +func main() { + //hardcode hash and user id for this fix + //TODO: first test on testnet, then run with real user id and hash: + //hash := "5cbde6f7ea043470c1b05d1b9fc2fbe50e5a86ad9782c8991ef33aca4496829b" + //userId := 4285 + hash := "" + userId := 0 + + ctx := context.Background() + c := &service.Config{} + + err := envconfig.Process("", c) + if err != nil { + log.Fatalf("Error loading environment variables: %v", err) + } + + // Setup logging to STDOUT or a configrued log file + logger := lib.Logger(c.LogFilePath) + + // Open a DB connection based on the configured DATABASE_URI + dbConn, err := db.Open(c.DatabaseUri) + if err != nil { + logger.Fatalf("Error initializing db connection: %v", err) + } + // Init new LND client + lndClient, err := lnd.NewLNDclient(lnd.LNDoptions{ + Address: c.LNDAddress, + MacaroonFile: c.LNDMacaroonFile, + MacaroonHex: c.LNDMacaroonHex, + CertFile: c.LNDCertFile, + CertHex: c.LNDCertHex, + }) + if err != nil { + logger.Fatalf("Error initializing the LND connection: %v", err) + } + svc := &service.LndhubService{ + Config: c, + DB: dbConn, + LndClient: lndClient, + Logger: logger, + InvoicePubSub: service.NewPubsub(), + } + invoice, err := svc.FindInvoiceByPaymentHash(ctx, int64(userId), hash) + if err != nil { + logger.Fatal(err) + } + //call svc.TrackPayment + err = svc.TrackOutgoingPaymentstatus(ctx, invoice) + if err != nil { + logger.Error(err) + } +} diff --git a/integration_tests/lnd_mock.go b/integration_tests/lnd_mock.go index 77327ac5..a2a390c3 100644 --- a/integration_tests/lnd_mock.go +++ b/integration_tests/lnd_mock.go @@ -216,6 +216,10 @@ func (mlnd *MockLND) GetInfo(ctx context.Context, req *lnrpc.GetInfoRequest, opt }, nil } +func (mlnd *MockLND) TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) { + return nil, nil +} + func (mlnd *MockLND) DecodeBolt11(ctx context.Context, bolt11 string, options ...grpc.CallOption) (*lnrpc.PayReq, error) { inv, err := zpay32.Decode(bolt11, &chaincfg.RegressionNetParams) if err != nil { diff --git a/integration_tests/subscription_start_test.go b/integration_tests/subscription_start_test.go index 2cde79f4..3dd4f7d1 100644 --- a/integration_tests/subscription_start_test.go +++ b/integration_tests/subscription_start_test.go @@ -136,3 +136,7 @@ func (mock *lndSubscriptionStartMockClient) GetInfo(ctx context.Context, req *ln func (mock *lndSubscriptionStartMockClient) DecodeBolt11(ctx context.Context, bolt11 string, options ...grpc.CallOption) (*lnrpc.PayReq, error) { panic("not implemented") // TODO: Implement } + +func (mlnd *lndSubscriptionStartMockClient) TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) { + return nil, nil +} diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go new file mode 100644 index 00000000..a2182319 --- /dev/null +++ b/lib/service/checkpayments.go @@ -0,0 +1,42 @@ +package service + +import ( + "context" + "fmt" + + "github.com/getAlby/lndhub.go/db/models" + "github.com/lightningnetwork/lnd/lnrpc" +) + +func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) error { + + //fetch the tx entry for the invoice + entry := models.TransactionEntry{} + err := svc.DB.NewSelect().Model(&entry).Where("invoice_id = ?", invoice.ID).Limit(1).Scan(ctx) + if err != nil { + return err + } + if entry.UserID != invoice.UserID { + return fmt.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) + } + //ask lnd using TrackPaymentV2 by hash of payment + payment, err := svc.LndClient.TrackPayment(ctx, invoice.RHash) + if err != nil { + return err + } + //call HandleFailedPayment or HandleSuccesfulPayment + if payment.Status == lnrpc.Payment_FAILED { + return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) + } + if payment.Status == lnrpc.Payment_SUCCEEDED { + invoice.Fee = payment.FeeSat + invoice.Preimage = payment.PaymentPreimage + //is it needed to update the hash here? + return svc.HandleSuccessfulPayment(ctx, invoice, entry) + } + if payment.Status == lnrpc.Payment_IN_FLIGHT { + //todo, we need to keep calling Recv() in this case, in a seperate goroutine maybe? + return nil + } + return nil +} diff --git a/lnd/interface.go b/lnd/interface.go index 0c3733b4..a8bfe139 100644 --- a/lnd/interface.go +++ b/lnd/interface.go @@ -14,6 +14,7 @@ type LightningClientWrapper interface { SubscribeInvoices(ctx context.Context, req *lnrpc.InvoiceSubscription, options ...grpc.CallOption) (SubscribeInvoicesWrapper, error) GetInfo(ctx context.Context, req *lnrpc.GetInfoRequest, options ...grpc.CallOption) (*lnrpc.GetInfoResponse, error) DecodeBolt11(ctx context.Context, bolt11 string, options ...grpc.CallOption) (*lnrpc.PayReq, error) + TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) } type SubscribeInvoicesWrapper interface { diff --git a/lnd/lnd.go b/lnd/lnd.go index bdfe887d..0d01a1f5 100644 --- a/lnd/lnd.go +++ b/lnd/lnd.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" "github.com/lightningnetwork/lnd/macaroons" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -30,7 +31,8 @@ type LNDoptions struct { } type LNDWrapper struct { - client lnrpc.LightningClient + client lnrpc.LightningClient + routerClient routerrpc.RouterClient } func NewLNDclient(lndOptions LNDoptions) (result *LNDWrapper, err error) { @@ -92,7 +94,8 @@ func NewLNDclient(lndOptions LNDoptions) (result *LNDWrapper, err error) { } return &LNDWrapper{ - client: lnrpc.NewLightningClient(conn), + client: lnrpc.NewLightningClient(conn), + routerClient: routerrpc.NewRouterClient(conn), }, nil } @@ -117,7 +120,19 @@ func (wrapper *LNDWrapper) GetInfo(ctx context.Context, req *lnrpc.GetInfoReques } func (wrapper *LNDWrapper) DecodeBolt11(ctx context.Context, bolt11 string, options ...grpc.CallOption) (*lnrpc.PayReq, error) { + return wrapper.client.DecodePayReq(ctx, &lnrpc.PayReqString{ PayReq: bolt11, }) } + +func (wrapper *LNDWrapper) TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) { + client, err := wrapper.routerClient.TrackPaymentV2(ctx, &routerrpc.TrackPaymentRequest{ + PaymentHash: []byte(hash), + NoInflightUpdates: true, + }, options...) + if err != nil { + return nil, err + } + return client.Recv() +} From c23fd3b0b4802e2d2b0d74dab200d64929bb3463 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 27 Sep 2022 16:27:59 +0200 Subject: [PATCH 02/22] track payment: use byte representation of hash --- fix_initialized_payments/main.go | 65 -------------------- integration_tests/lnd_mock.go | 2 +- integration_tests/subscription_start_test.go | 2 +- lib/service/checkpayments.go | 12 +++- lnd/interface.go | 2 +- lnd/lnd.go | 2 +- 6 files changed, 13 insertions(+), 72 deletions(-) delete mode 100644 fix_initialized_payments/main.go diff --git a/fix_initialized_payments/main.go b/fix_initialized_payments/main.go deleted file mode 100644 index dd642d76..00000000 --- a/fix_initialized_payments/main.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import ( - "context" - "log" - - "github.com/getAlby/lndhub.go/db" - "github.com/getAlby/lndhub.go/lib" - "github.com/getAlby/lndhub.go/lib/service" - "github.com/getAlby/lndhub.go/lnd" - "github.com/kelseyhightower/envconfig" -) - -func main() { - //hardcode hash and user id for this fix - //TODO: first test on testnet, then run with real user id and hash: - //hash := "5cbde6f7ea043470c1b05d1b9fc2fbe50e5a86ad9782c8991ef33aca4496829b" - //userId := 4285 - hash := "" - userId := 0 - - ctx := context.Background() - c := &service.Config{} - - err := envconfig.Process("", c) - if err != nil { - log.Fatalf("Error loading environment variables: %v", err) - } - - // Setup logging to STDOUT or a configrued log file - logger := lib.Logger(c.LogFilePath) - - // Open a DB connection based on the configured DATABASE_URI - dbConn, err := db.Open(c.DatabaseUri) - if err != nil { - logger.Fatalf("Error initializing db connection: %v", err) - } - // Init new LND client - lndClient, err := lnd.NewLNDclient(lnd.LNDoptions{ - Address: c.LNDAddress, - MacaroonFile: c.LNDMacaroonFile, - MacaroonHex: c.LNDMacaroonHex, - CertFile: c.LNDCertFile, - CertHex: c.LNDCertHex, - }) - if err != nil { - logger.Fatalf("Error initializing the LND connection: %v", err) - } - svc := &service.LndhubService{ - Config: c, - DB: dbConn, - LndClient: lndClient, - Logger: logger, - InvoicePubSub: service.NewPubsub(), - } - invoice, err := svc.FindInvoiceByPaymentHash(ctx, int64(userId), hash) - if err != nil { - logger.Fatal(err) - } - //call svc.TrackPayment - err = svc.TrackOutgoingPaymentstatus(ctx, invoice) - if err != nil { - logger.Error(err) - } -} diff --git a/integration_tests/lnd_mock.go b/integration_tests/lnd_mock.go index a2a390c3..b2cf99d4 100644 --- a/integration_tests/lnd_mock.go +++ b/integration_tests/lnd_mock.go @@ -216,7 +216,7 @@ func (mlnd *MockLND) GetInfo(ctx context.Context, req *lnrpc.GetInfoRequest, opt }, nil } -func (mlnd *MockLND) TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) { +func (mlnd *MockLND) TrackPayment(ctx context.Context, hash []byte, options ...grpc.CallOption) (*lnrpc.Payment, error) { return nil, nil } diff --git a/integration_tests/subscription_start_test.go b/integration_tests/subscription_start_test.go index 3dd4f7d1..c900f8c7 100644 --- a/integration_tests/subscription_start_test.go +++ b/integration_tests/subscription_start_test.go @@ -137,6 +137,6 @@ func (mock *lndSubscriptionStartMockClient) DecodeBolt11(ctx context.Context, bo panic("not implemented") // TODO: Implement } -func (mlnd *lndSubscriptionStartMockClient) TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) { +func (mlnd *lndSubscriptionStartMockClient) TrackPayment(ctx context.Context, hash []byte, options ...grpc.CallOption) (*lnrpc.Payment, error) { return nil, nil } diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index a2182319..ed25f039 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/hex" "fmt" "github.com/getAlby/lndhub.go/db/models" @@ -20,22 +21,27 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic return fmt.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) } //ask lnd using TrackPaymentV2 by hash of payment - payment, err := svc.LndClient.TrackPayment(ctx, invoice.RHash) + rawHash, err := hex.DecodeString(invoice.RHash) + if err != nil { + return err + } + payment, err := svc.LndClient.TrackPayment(ctx, rawHash) if err != nil { return err } //call HandleFailedPayment or HandleSuccesfulPayment if payment.Status == lnrpc.Payment_FAILED { + svc.Logger.Infof("Updating failed payment %v", payment) return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) } if payment.Status == lnrpc.Payment_SUCCEEDED { invoice.Fee = payment.FeeSat invoice.Preimage = payment.PaymentPreimage - //is it needed to update the hash here? + svc.Logger.Infof("Updating completed payment %v", payment) return svc.HandleSuccessfulPayment(ctx, invoice, entry) } if payment.Status == lnrpc.Payment_IN_FLIGHT { - //todo, we need to keep calling Recv() in this case, in a seperate goroutine maybe? + //TODO, we need to keep calling Recv() in this case, in a seperate goroutine maybe? return nil } return nil diff --git a/lnd/interface.go b/lnd/interface.go index a8bfe139..f4b3da12 100644 --- a/lnd/interface.go +++ b/lnd/interface.go @@ -14,7 +14,7 @@ type LightningClientWrapper interface { SubscribeInvoices(ctx context.Context, req *lnrpc.InvoiceSubscription, options ...grpc.CallOption) (SubscribeInvoicesWrapper, error) GetInfo(ctx context.Context, req *lnrpc.GetInfoRequest, options ...grpc.CallOption) (*lnrpc.GetInfoResponse, error) DecodeBolt11(ctx context.Context, bolt11 string, options ...grpc.CallOption) (*lnrpc.PayReq, error) - TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) + TrackPayment(ctx context.Context, hash []byte, options ...grpc.CallOption) (*lnrpc.Payment, error) } type SubscribeInvoicesWrapper interface { diff --git a/lnd/lnd.go b/lnd/lnd.go index 0d01a1f5..2d4502e1 100644 --- a/lnd/lnd.go +++ b/lnd/lnd.go @@ -126,7 +126,7 @@ func (wrapper *LNDWrapper) DecodeBolt11(ctx context.Context, bolt11 string, opti }) } -func (wrapper *LNDWrapper) TrackPayment(ctx context.Context, hash string, options ...grpc.CallOption) (*lnrpc.Payment, error) { +func (wrapper *LNDWrapper) TrackPayment(ctx context.Context, hash []byte, options ...grpc.CallOption) (*lnrpc.Payment, error) { client, err := wrapper.routerClient.TrackPaymentV2(ctx, &routerrpc.TrackPaymentRequest{ PaymentHash: []byte(hash), NoInflightUpdates: true, From 209a0d0b8cc1bb1ca7b03b84fc8bea428fe16e64 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 4 Oct 2022 18:33:24 +0200 Subject: [PATCH 03/22] add more methods for checking all pending payments --- lib/service/checkpayments.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index ed25f039..24ed2c0f 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -9,6 +9,14 @@ import ( "github.com/lightningnetwork/lnd/lnrpc" ) +func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) (pendingPayments []models.Invoice, err error) { + //this should be called in a goroutine at startup + //todo + //check database for all pending payments + //call trackoutgoingpaymentstatus for each one + return nil, nil +} + func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) error { //fetch the tx entry for the invoice @@ -31,17 +39,22 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic } //call HandleFailedPayment or HandleSuccesfulPayment if payment.Status == lnrpc.Payment_FAILED { - svc.Logger.Infof("Updating failed payment %v", payment) - return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) + svc.Logger.Infof("Failed payment detected: %v", payment) + //todo handle failed payment + //return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) + return nil } if payment.Status == lnrpc.Payment_SUCCEEDED { invoice.Fee = payment.FeeSat invoice.Preimage = payment.PaymentPreimage - svc.Logger.Infof("Updating completed payment %v", payment) - return svc.HandleSuccessfulPayment(ctx, invoice, entry) + svc.Logger.Infof("Completed payment detected: %v", payment) + //todo handle succesful payment + //return svc.HandleSuccessfulPayment(ctx, invoice, entry) + return nil } if payment.Status == lnrpc.Payment_IN_FLIGHT { - //TODO, we need to keep calling Recv() in this case, in a seperate goroutine maybe? + //todo handle inflight payment + svc.Logger.Infof("In-flight payment detected: %v", payment) return nil } return nil From 0bcf83545b9d38975c9375f9a71e9b7963b51110 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 6 Oct 2022 11:11:03 +0200 Subject: [PATCH 04/22] check pending payments at startup --- lib/service/checkpayments.go | 18 ++++++++++++++---- main.go | 8 ++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 24ed2c0f..f9db4b4d 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -9,12 +9,22 @@ import ( "github.com/lightningnetwork/lnd/lnrpc" ) -func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) (pendingPayments []models.Invoice, err error) { - //this should be called in a goroutine at startup - //todo +func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) (err error) { //check database for all pending payments + pendingPayments := []models.Invoice{} + err = svc.DB.NewSelect().Model(&pendingPayments).Where("state = 'initialized'").Where("type = 'outgoing'").Scan(ctx) + if err != nil { + return err + } + svc.Logger.Infof("Found %d pending payments", len(pendingPayments)) //call trackoutgoingpaymentstatus for each one - return nil, nil + for _, inv := range pendingPayments { + err = svc.TrackOutgoingPaymentstatus(ctx, &inv) + if err != nil { + svc.Logger.Error(err) + } + } + return nil } func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) error { diff --git a/main.go b/main.go index 0a8d45ad..91895128 100644 --- a/main.go +++ b/main.go @@ -165,6 +165,14 @@ func main() { // Subscribe to LND invoice updates in the background go svc.InvoiceUpdateSubscription(context.Background()) + // Check the status of all pending outgoing payments + go func() { + err = svc.CheckAllPendingOutgoingPayments(context.Background()) + if err != nil { + svc.Logger.Error(err) + } + }() + //Start webhook subscription if svc.Config.WebhookUrl != "" { webhookCtx, cancelWebhook := context.WithCancel(context.Background()) From c437651cf87df56bc83608fc3bccb7de7655f641 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 6 Oct 2022 12:37:39 +0200 Subject: [PATCH 05/22] log invoice on error in track status --- lib/service/checkpayments.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index f9db4b4d..18324272 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -21,7 +21,7 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( for _, inv := range pendingPayments { err = svc.TrackOutgoingPaymentstatus(ctx, &inv) if err != nil { - svc.Logger.Error(err) + svc.Logger.Errorf("Error tracking payment %v: %s", inv, err.Error()) } } return nil From 10278c7de3af9d0af8a4f1f8220a6b69ac95af82 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 29 Nov 2022 13:34:54 +0100 Subject: [PATCH 06/22] track outgoing payments in goroutines --- integration_tests/lnd_mock.go | 4 ++ integration_tests/subscription_start_test.go | 4 ++ lib/service/checkpayments.go | 70 +++++++++++--------- lnd/interface.go | 6 +- lnd/lnd.go | 4 ++ main.go | 11 ++- 6 files changed, 62 insertions(+), 37 deletions(-) diff --git a/integration_tests/lnd_mock.go b/integration_tests/lnd_mock.go index b2cf99d4..86c51ff0 100644 --- a/integration_tests/lnd_mock.go +++ b/integration_tests/lnd_mock.go @@ -15,6 +15,7 @@ import ( "github.com/getAlby/lndhub.go/lnd" "github.com/labstack/gommon/random" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" "github.com/lightningnetwork/lnd/lnwire" "github.com/lightningnetwork/lnd/zpay32" "google.golang.org/grpc" @@ -58,6 +59,9 @@ func (mockSub *MockSubscribeInvoices) Recv() (*lnrpc.Invoice, error) { inv := <-mockSub.invoiceChan return inv, nil } +func (mlnd *MockLND) SubscribePayment(ctx context.Context, req *routerrpc.TrackPaymentRequest, options ...grpc.CallOption) (lnd.SubscribePaymentWrapper, error) { + return nil, nil +} func (mlnd *MockLND) ListChannels(ctx context.Context, req *lnrpc.ListChannelsRequest, options ...grpc.CallOption) (*lnrpc.ListChannelsResponse, error) { return &lnrpc.ListChannelsResponse{ diff --git a/integration_tests/subscription_start_test.go b/integration_tests/subscription_start_test.go index c900f8c7..25d3e6ec 100644 --- a/integration_tests/subscription_start_test.go +++ b/integration_tests/subscription_start_test.go @@ -17,6 +17,7 @@ import ( "github.com/go-playground/validator/v10" "github.com/labstack/echo/v4" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/uptrace/bun" @@ -128,6 +129,9 @@ func (mock *lndSubscriptionStartMockClient) SubscribeInvoices(ctx context.Contex func (mock *lndSubscriptionStartMockClient) Recv() (*lnrpc.Invoice, error) { select {} } +func (mock *lndSubscriptionStartMockClient) SubscribePayment(ctx context.Context, req *routerrpc.TrackPaymentRequest, options ...grpc.CallOption) (lnd.SubscribePaymentWrapper, error) { + return nil, nil +} func (mock *lndSubscriptionStartMockClient) GetInfo(ctx context.Context, req *lnrpc.GetInfoRequest, options ...grpc.CallOption) (*lnrpc.GetInfoResponse, error) { panic("not implemented") // TODO: Implement diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 18324272..32aa4e27 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -3,10 +3,10 @@ package service import ( "context" "encoding/hex" - "fmt" "github.com/getAlby/lndhub.go/db/models" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" ) func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) (err error) { @@ -19,53 +19,63 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( svc.Logger.Infof("Found %d pending payments", len(pendingPayments)) //call trackoutgoingpaymentstatus for each one for _, inv := range pendingPayments { - err = svc.TrackOutgoingPaymentstatus(ctx, &inv) - if err != nil { - svc.Logger.Errorf("Error tracking payment %v: %s", inv, err.Error()) - } + //spawn goroutines + go svc.TrackOutgoingPaymentstatus(ctx, &inv) } return nil } -func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) error { +//Should be called in a goroutine as the tracking can potentially take a long time +func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) { //fetch the tx entry for the invoice entry := models.TransactionEntry{} err := svc.DB.NewSelect().Model(&entry).Where("invoice_id = ?", invoice.ID).Limit(1).Scan(ctx) if err != nil { - return err + svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + return + } if entry.UserID != invoice.UserID { - return fmt.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) + svc.Logger.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) + return } //ask lnd using TrackPaymentV2 by hash of payment rawHash, err := hex.DecodeString(invoice.RHash) if err != nil { - return err + svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + return } - payment, err := svc.LndClient.TrackPayment(ctx, rawHash) + paymentTracker, err := svc.LndClient.SubscribePayment(ctx, &routerrpc.TrackPaymentRequest{ + PaymentHash: rawHash, + NoInflightUpdates: true, + }) if err != nil { - return err + svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + return } //call HandleFailedPayment or HandleSuccesfulPayment - if payment.Status == lnrpc.Payment_FAILED { - svc.Logger.Infof("Failed payment detected: %v", payment) - //todo handle failed payment - //return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) - return nil - } - if payment.Status == lnrpc.Payment_SUCCEEDED { - invoice.Fee = payment.FeeSat - invoice.Preimage = payment.PaymentPreimage - svc.Logger.Infof("Completed payment detected: %v", payment) - //todo handle succesful payment - //return svc.HandleSuccessfulPayment(ctx, invoice, entry) - return nil - } - if payment.Status == lnrpc.Payment_IN_FLIGHT { - //todo handle inflight payment - svc.Logger.Infof("In-flight payment detected: %v", payment) - return nil + for { + payment, err := paymentTracker.Recv() + if err != nil { + svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + return + } + if payment.Status == lnrpc.Payment_FAILED { + svc.Logger.Infof("Failed payment detected: %v", payment) + //todo handle failed payment + //return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) + return + } + if payment.Status == lnrpc.Payment_SUCCEEDED { + invoice.Fee = payment.FeeSat + invoice.Preimage = payment.PaymentPreimage + svc.Logger.Infof("Completed payment detected: %v", payment) + //todo handle succesful payment + //return svc.HandleSuccessfulPayment(ctx, invoice, entry) + return + } + //Since we shouldn't get in-flight updates we shouldn't get here + svc.Logger.Warnf("Got an unexpected in-flight update %v", payment) } - return nil } diff --git a/lnd/interface.go b/lnd/interface.go index f4b3da12..e7de0502 100644 --- a/lnd/interface.go +++ b/lnd/interface.go @@ -4,6 +4,7 @@ import ( "context" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" "google.golang.org/grpc" ) @@ -12,11 +13,14 @@ type LightningClientWrapper interface { SendPaymentSync(ctx context.Context, req *lnrpc.SendRequest, options ...grpc.CallOption) (*lnrpc.SendResponse, error) AddInvoice(ctx context.Context, req *lnrpc.Invoice, options ...grpc.CallOption) (*lnrpc.AddInvoiceResponse, error) SubscribeInvoices(ctx context.Context, req *lnrpc.InvoiceSubscription, options ...grpc.CallOption) (SubscribeInvoicesWrapper, error) + SubscribePayment(ctx context.Context, req *routerrpc.TrackPaymentRequest, options ...grpc.CallOption) (SubscribePaymentWrapper, error) GetInfo(ctx context.Context, req *lnrpc.GetInfoRequest, options ...grpc.CallOption) (*lnrpc.GetInfoResponse, error) DecodeBolt11(ctx context.Context, bolt11 string, options ...grpc.CallOption) (*lnrpc.PayReq, error) - TrackPayment(ctx context.Context, hash []byte, options ...grpc.CallOption) (*lnrpc.Payment, error) } type SubscribeInvoicesWrapper interface { Recv() (*lnrpc.Invoice, error) } +type SubscribePaymentWrapper interface { + Recv() (*lnrpc.Payment, error) +} diff --git a/lnd/lnd.go b/lnd/lnd.go index 2d4502e1..ec044bac 100644 --- a/lnd/lnd.go +++ b/lnd/lnd.go @@ -126,6 +126,10 @@ func (wrapper *LNDWrapper) DecodeBolt11(ctx context.Context, bolt11 string, opti }) } +func (wrapper *LNDWrapper) SubscribePayment(ctx context.Context, req *routerrpc.TrackPaymentRequest, options ...grpc.CallOption) (SubscribePaymentWrapper, error) { + return wrapper.routerClient.TrackPaymentV2(ctx, req, options...) +} + func (wrapper *LNDWrapper) TrackPayment(ctx context.Context, hash []byte, options ...grpc.CallOption) (*lnrpc.Payment, error) { client, err := wrapper.routerClient.TrackPaymentV2(ctx, &routerrpc.TrackPaymentRequest{ PaymentHash: []byte(hash), diff --git a/main.go b/main.go index 91895128..4032e679 100644 --- a/main.go +++ b/main.go @@ -166,12 +166,11 @@ func main() { go svc.InvoiceUpdateSubscription(context.Background()) // Check the status of all pending outgoing payments - go func() { - err = svc.CheckAllPendingOutgoingPayments(context.Background()) - if err != nil { - svc.Logger.Error(err) - } - }() + // A goroutine will be spawned for each one + err = svc.CheckAllPendingOutgoingPayments(context.Background()) + if err != nil { + svc.Logger.Error(err) + } //Start webhook subscription if svc.Config.WebhookUrl != "" { From 47894ad3c2a9f27af82261232480c382fd9894a8 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 29 Nov 2022 13:40:11 +0100 Subject: [PATCH 07/22] payment tracking logging --- lib/service/checkpayments.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 32aa4e27..ce7ba15b 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -16,7 +16,7 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( if err != nil { return err } - svc.Logger.Infof("Found %d pending payments", len(pendingPayments)) + svc.Logger.Infof("Found %d pending payments, spawning trackers", len(pendingPayments)) //call trackoutgoingpaymentstatus for each one for _, inv := range pendingPayments { //spawn goroutines @@ -25,7 +25,7 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( return nil } -//Should be called in a goroutine as the tracking can potentially take a long time +// Should be called in a goroutine as the tracking can potentially take a long time func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) { //fetch the tx entry for the invoice @@ -62,7 +62,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic return } if payment.Status == lnrpc.Payment_FAILED { - svc.Logger.Infof("Failed payment detected: %v", payment) + svc.Logger.Infof("Failed payment detected: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) //todo handle failed payment //return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) return @@ -70,7 +70,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic if payment.Status == lnrpc.Payment_SUCCEEDED { invoice.Fee = payment.FeeSat invoice.Preimage = payment.PaymentPreimage - svc.Logger.Infof("Completed payment detected: %v", payment) + svc.Logger.Infof("Completed payment detected: hash %s", payment.PaymentHash) //todo handle succesful payment //return svc.HandleSuccessfulPayment(ctx, invoice, entry) return From f2a4a4d4354cb2923654b903010b4b8bdcc2be5e Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 29 Nov 2022 13:41:07 +0100 Subject: [PATCH 08/22] payment tracking logging --- lib/service/checkpayments.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index ce7ba15b..62502f7d 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -3,6 +3,7 @@ package service import ( "context" "encoding/hex" + "fmt" "github.com/getAlby/lndhub.go/db/models" "github.com/lightningnetwork/lnd/lnrpc" @@ -64,16 +65,22 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic if payment.Status == lnrpc.Payment_FAILED { svc.Logger.Infof("Failed payment detected: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) //todo handle failed payment - //return svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) + err = svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) + if err != nil { + svc.Logger.Errorf("Error handling failed payment %v: %s", invoice, err.Error()) + return + } return } if payment.Status == lnrpc.Payment_SUCCEEDED { invoice.Fee = payment.FeeSat invoice.Preimage = payment.PaymentPreimage svc.Logger.Infof("Completed payment detected: hash %s", payment.PaymentHash) - //todo handle succesful payment - //return svc.HandleSuccessfulPayment(ctx, invoice, entry) - return + err = svc.HandleSuccessfulPayment(ctx, invoice, entry) + if err != nil { + svc.Logger.Errorf("Error handling successful payment %v: %s", invoice, err.Error()) + return + } } //Since we shouldn't get in-flight updates we shouldn't get here svc.Logger.Warnf("Got an unexpected in-flight update %v", payment) From 9c75a92adf6c09517454f6bb23863a5bbad9ca41 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 11:49:04 +0100 Subject: [PATCH 09/22] clean up code, remove bug --- lib/service/checkpayments.go | 39 +++++++++++++++++++----------------- lnd/lnd.go | 11 ---------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 62502f7d..8ae6bc01 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -13,34 +13,22 @@ import ( func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) (err error) { //check database for all pending payments pendingPayments := []models.Invoice{} + //since this part is synchronously executed before the main server starts, we should not get into race conditions err = svc.DB.NewSelect().Model(&pendingPayments).Where("state = 'initialized'").Where("type = 'outgoing'").Scan(ctx) if err != nil { return err } - svc.Logger.Infof("Found %d pending payments, spawning trackers", len(pendingPayments)) + svc.Logger.Infof("Found %d pending payments", len(pendingPayments)) //call trackoutgoingpaymentstatus for each one for _, inv := range pendingPayments { //spawn goroutines - go svc.TrackOutgoingPaymentstatus(ctx, &inv) + go svc.TrackOutgoingPaymentstatus(ctx, inv) } return nil } // Should be called in a goroutine as the tracking can potentially take a long time -func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) { - - //fetch the tx entry for the invoice - entry := models.TransactionEntry{} - err := svc.DB.NewSelect().Model(&entry).Where("invoice_id = ?", invoice.ID).Limit(1).Scan(ctx) - if err != nil { - svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) - return - - } - if entry.UserID != invoice.UserID { - svc.Logger.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) - return - } +func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice models.Invoice) { //ask lnd using TrackPaymentV2 by hash of payment rawHash, err := hex.DecodeString(invoice.RHash) if err != nil { @@ -55,6 +43,18 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) return } + //fetch the tx entry for the invoice + entry := models.TransactionEntry{} + err = svc.DB.NewSelect().Model(&entry).Where("invoice_id = ?", invoice.ID).Limit(1).Scan(ctx) + if err != nil { + svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + return + + } + if entry.UserID != invoice.UserID { + svc.Logger.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) + return + } //call HandleFailedPayment or HandleSuccesfulPayment for { payment, err := paymentTracker.Recv() @@ -65,22 +65,25 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic if payment.Status == lnrpc.Payment_FAILED { svc.Logger.Infof("Failed payment detected: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) //todo handle failed payment - err = svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) + err = svc.HandleFailedPayment(ctx, &invoice, entry, fmt.Errorf(payment.FailureReason.String())) if err != nil { svc.Logger.Errorf("Error handling failed payment %v: %s", invoice, err.Error()) return } + svc.Logger.Infof("Updated failed payment: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) return } if payment.Status == lnrpc.Payment_SUCCEEDED { invoice.Fee = payment.FeeSat invoice.Preimage = payment.PaymentPreimage svc.Logger.Infof("Completed payment detected: hash %s", payment.PaymentHash) - err = svc.HandleSuccessfulPayment(ctx, invoice, entry) + err = svc.HandleSuccessfulPayment(ctx, &invoice, entry) if err != nil { svc.Logger.Errorf("Error handling successful payment %v: %s", invoice, err.Error()) return } + svc.Logger.Infof("Updated completed payment: hash %s", payment.PaymentHash) + return } //Since we shouldn't get in-flight updates we shouldn't get here svc.Logger.Warnf("Got an unexpected in-flight update %v", payment) diff --git a/lnd/lnd.go b/lnd/lnd.go index ec044bac..d61c3faf 100644 --- a/lnd/lnd.go +++ b/lnd/lnd.go @@ -129,14 +129,3 @@ func (wrapper *LNDWrapper) DecodeBolt11(ctx context.Context, bolt11 string, opti func (wrapper *LNDWrapper) SubscribePayment(ctx context.Context, req *routerrpc.TrackPaymentRequest, options ...grpc.CallOption) (SubscribePaymentWrapper, error) { return wrapper.routerClient.TrackPaymentV2(ctx, req, options...) } - -func (wrapper *LNDWrapper) TrackPayment(ctx context.Context, hash []byte, options ...grpc.CallOption) (*lnrpc.Payment, error) { - client, err := wrapper.routerClient.TrackPaymentV2(ctx, &routerrpc.TrackPaymentRequest{ - PaymentHash: []byte(hash), - NoInflightUpdates: true, - }, options...) - if err != nil { - return nil, err - } - return client.Recv() -} From 3a5261388c3aa6e4d18e4c30074378e0b2fef004 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 11:53:46 +0100 Subject: [PATCH 10/22] rm unnecessary whitespace --- lnd/lnd.go | 1 - 1 file changed, 1 deletion(-) diff --git a/lnd/lnd.go b/lnd/lnd.go index d61c3faf..a95b4442 100644 --- a/lnd/lnd.go +++ b/lnd/lnd.go @@ -120,7 +120,6 @@ func (wrapper *LNDWrapper) GetInfo(ctx context.Context, req *lnrpc.GetInfoReques } func (wrapper *LNDWrapper) DecodeBolt11(ctx context.Context, bolt11 string, options ...grpc.CallOption) (*lnrpc.PayReq, error) { - return wrapper.client.DecodePayReq(ctx, &lnrpc.PayReqString{ PayReq: bolt11, }) From dbb8eb4709bc7965cc16a53fc54c8b56e301ec49 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 12:10:52 +0100 Subject: [PATCH 11/22] avoid goroutine bug --- lib/service/checkpayments.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 8ae6bc01..5f844d23 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -22,13 +22,16 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( //call trackoutgoingpaymentstatus for each one for _, inv := range pendingPayments { //spawn goroutines - go svc.TrackOutgoingPaymentstatus(ctx, inv) + //https://go.dev/doc/faq#closures_and_goroutines + inv := inv + go svc.TrackOutgoingPaymentstatus(ctx, &inv) } return nil } // Should be called in a goroutine as the tracking can potentially take a long time -func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice models.Invoice) { +func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) { + fmt.Println(invoice.RHash) //ask lnd using TrackPaymentV2 by hash of payment rawHash, err := hex.DecodeString(invoice.RHash) if err != nil { @@ -65,7 +68,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic if payment.Status == lnrpc.Payment_FAILED { svc.Logger.Infof("Failed payment detected: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) //todo handle failed payment - err = svc.HandleFailedPayment(ctx, &invoice, entry, fmt.Errorf(payment.FailureReason.String())) + err = svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) if err != nil { svc.Logger.Errorf("Error handling failed payment %v: %s", invoice, err.Error()) return @@ -77,7 +80,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic invoice.Fee = payment.FeeSat invoice.Preimage = payment.PaymentPreimage svc.Logger.Infof("Completed payment detected: hash %s", payment.PaymentHash) - err = svc.HandleSuccessfulPayment(ctx, &invoice, entry) + err = svc.HandleSuccessfulPayment(ctx, invoice, entry) if err != nil { svc.Logger.Errorf("Error handling successful payment %v: %s", invoice, err.Error()) return From a21d2eb1801771e8a9b2909acfaf702976f14bbc Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 12:38:52 +0100 Subject: [PATCH 12/22] add mock hodl lnd --- integration_tests/lnd_mock_hodl.go | 51 ++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 integration_tests/lnd_mock_hodl.go diff --git a/integration_tests/lnd_mock_hodl.go b/integration_tests/lnd_mock_hodl.go new file mode 100644 index 00000000..42ca2610 --- /dev/null +++ b/integration_tests/lnd_mock_hodl.go @@ -0,0 +1,51 @@ +package integration_tests + +import ( + "context" + "errors" + + "github.com/getAlby/lndhub.go/lnd" + "github.com/lightningnetwork/lnd/lnrpc" + "google.golang.org/grpc" +) + +type LNDMockHodlWrapper struct { + lnd.LightningClientWrapper +} + +func NewLNDMockHodlWrapper(lnd lnd.LightningClientWrapper) (result *LNDMockWrapper, err error) { + return &LNDMockWrapper{ + lnd, + }, nil +} + +func (wrapper *LNDMockHodlWrapper) SendPaymentSync(ctx context.Context, req *lnrpc.SendRequest, options ...grpc.CallOption) (*lnrpc.SendResponse, error) { + return nil, errors.New(SendPaymentMockError) +} + +// mock where send payment sync failure is controlled by channel +// even though send payment method is still sync, suffix "Async" here is used to show intention of using this mock +var paymentResultChannel = make(chan bool, 1) + +type LNDMockHodlWrapperAsync struct { + lnd.LightningClientWrapper +} + +func NewLNDMockHodlWrapperAsync(lnd lnd.LightningClientWrapper) (result *LNDMockWrapperAsync, err error) { + return &LNDMockWrapperAsync{ + lnd, + }, nil +} + +func (wrapper *LNDMockHodlWrapperAsync) SendPaymentSync(ctx context.Context, req *lnrpc.SendRequest, options ...grpc.CallOption) (*lnrpc.SendResponse, error) { + //block indefinetely + select {} +} + +func (wrapper *LNDMockHodlWrapperAsync) SettlePayment(success bool) { + paymentResultChannel <- success +} + +//TODO: payment tracker implemantation: read from channel, return to receive method +//write test that completes payment +//write test that fails payment From f2edae717db221e809bbc97ebce68f045d7410a3 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 13:52:44 +0100 Subject: [PATCH 13/22] start writing integration test --- integration_tests/hodl_invoice_test.go | 186 +++++++++++++++++++++++++ integration_tests/lnd_mock_hodl.go | 42 +++--- 2 files changed, 210 insertions(+), 18 deletions(-) create mode 100644 integration_tests/hodl_invoice_test.go diff --git a/integration_tests/hodl_invoice_test.go b/integration_tests/hodl_invoice_test.go new file mode 100644 index 00000000..b2a82e9e --- /dev/null +++ b/integration_tests/hodl_invoice_test.go @@ -0,0 +1,186 @@ +package integration_tests + +import ( + "context" + "fmt" + "log" + "testing" + "time" + + "github.com/getAlby/lndhub.go/common" + "github.com/getAlby/lndhub.go/controllers" + "github.com/getAlby/lndhub.go/lib" + "github.com/getAlby/lndhub.go/lib/responses" + "github.com/getAlby/lndhub.go/lib/service" + "github.com/getAlby/lndhub.go/lib/tokens" + "github.com/go-playground/validator/v10" + "github.com/labstack/echo/v4" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type HodlInvoiceSuite struct { + TestSuite + mlnd *MockLND + externalLND *MockLND + service *service.LndhubService + userLogin ExpectedCreateUserResponseBody + userToken string + invoiceUpdateSubCancelFn context.CancelFunc + hodlLND *LNDMockHodlWrapperAsync +} + +func (suite *HodlInvoiceSuite) SetupSuite() { + mlnd := newDefaultMockLND() + externalLND, err := NewMockLND("1234567890abcdefabcd", 0, make(chan (*lnrpc.Invoice))) + if err != nil { + log.Fatalf("Error initializing test service: %v", err) + } + suite.externalLND = externalLND + suite.mlnd = mlnd + // inject hodl lnd client + lndClient, err := NewLNDMockHodlWrapperAsync(mlnd) + suite.hodlLND = lndClient + if err != nil { + log.Fatalf("Error setting up test client: %v", err) + } + + svc, err := LndHubTestServiceInit(lndClient) + if err != nil { + log.Fatalf("Error initializing test service: %v", err) + } + users, userTokens, err := createUsers(svc, 1) + if err != nil { + log.Fatalf("Error creating test users: %v", err) + } + // Subscribe to LND invoice updates in the background + // store cancel func to be called in tear down suite + ctx, cancel := context.WithCancel(context.Background()) + suite.invoiceUpdateSubCancelFn = cancel + go svc.InvoiceUpdateSubscription(ctx) + suite.service = svc + e := echo.New() + + e.HTTPErrorHandler = responses.HTTPErrorHandler + e.Validator = &lib.CustomValidator{Validator: validator.New()} + suite.echo = e + assert.Equal(suite.T(), 1, len(users)) + assert.Equal(suite.T(), 1, len(userTokens)) + suite.userLogin = users[0] + suite.userToken = userTokens[0] + suite.echo.Use(tokens.Middleware([]byte(suite.service.Config.JWTSecret))) + suite.echo.GET("/balance", controllers.NewBalanceController(suite.service).Balance) + suite.echo.POST("/addinvoice", controllers.NewAddInvoiceController(suite.service).AddInvoice) + suite.echo.POST("/payinvoice", controllers.NewPayInvoiceController(suite.service).PayInvoice) +} + +func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { + userFundingSats := 1000 + externalSatRequested := 500 + // fund user account + invoiceResponse := suite.createAddInvoiceReq(userFundingSats, "integration test external payment user", suite.userToken) + err := suite.mlnd.mockPaidInvoice(invoiceResponse, 0, false, nil) + assert.NoError(suite.T(), err) + + // wait a bit for the callback event to hit + time.Sleep(10 * time.Millisecond) + + // create external invoice + externalInvoice := lnrpc.Invoice{ + Memo: "integration tests: external pay from user", + Value: int64(externalSatRequested), + } + invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice) + assert.NoError(suite.T(), err) + // pay external from user, req will be canceled after 2 sec + go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken) + + // wait for request to fail + time.Sleep(5 * time.Second) + + //TODO + //start payment tracking loop + //send settle invoice with lnrpc.payment + + // check to see that balance was reduced + userId := getUserIdFromToken(suite.userToken) + userBalance, err := suite.service.CurrentUserBalance(context.Background(), userId) + if err != nil { + fmt.Printf("Error when getting balance %v\n", err.Error()) + } + assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) + //todo: check that invoice was updated as completed + +} + +func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { + userFundingSats := 1000 + externalSatRequested := 500 + // fund user account + invoiceResponse := suite.createAddInvoiceReq(userFundingSats, "integration test external payment user", suite.userToken) + err := suite.mlnd.mockPaidInvoice(invoiceResponse, 0, false, nil) + assert.NoError(suite.T(), err) + + // wait a bit for the callback event to hit + time.Sleep(10 * time.Millisecond) + + // create external invoice + externalInvoice := lnrpc.Invoice{ + Memo: "integration tests: external pay from user", + Value: int64(externalSatRequested), + } + invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice) + assert.NoError(suite.T(), err) + // pay external from user, req will be canceled after 2 sec + go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken) + + // wait for request to fail + time.Sleep(5 * time.Second) + + // check to see that balance was reduced + userId := getUserIdFromToken(suite.userToken) + userBalance, err := suite.service.CurrentUserBalance(context.Background(), userId) + if err != nil { + fmt.Printf("Error when getting balance %v\n", err.Error()) + } + assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) + + //TODO + //start payment tracking loop + //send cancel invoice with lnrpc.payment + + // check that balance was reverted and invoice is in error state + userBalance, err = suite.service.CurrentUserBalance(context.Background(), userId) + if err != nil { + fmt.Printf("Error when getting balance %v\n", err.Error()) + } + assert.Equal(suite.T(), int64(userFundingSats), userBalance) + + invoices, err := suite.service.InvoicesFor(context.Background(), userId, common.InvoiceTypeOutgoing) + if err != nil { + fmt.Printf("Error when getting invoices %v\n", err.Error()) + } + assert.Equal(suite.T(), 1, len(invoices)) + assert.Equal(suite.T(), common.InvoiceStateError, invoices[0].State) + assert.Equal(suite.T(), SendPaymentMockError, invoices[0].ErrorMessage) + + transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId) + if err != nil { + fmt.Printf("Error when getting transaction entries %v\n", err.Error()) + } + // check if there are 3 transaction entries, with reversed credit and debit account ids + assert.Equal(suite.T(), 3, len(transactonEntries)) + assert.Equal(suite.T(), transactonEntries[1].CreditAccountID, transactonEntries[2].DebitAccountID) + assert.Equal(suite.T(), transactonEntries[1].DebitAccountID, transactonEntries[2].CreditAccountID) + assert.Equal(suite.T(), transactonEntries[1].Amount, int64(externalSatRequested)) + assert.Equal(suite.T(), transactonEntries[2].Amount, int64(externalSatRequested)) +} + +func (suite *HodlInvoiceSuite) TearDownSuite() { + suite.invoiceUpdateSubCancelFn() +} + +func TestHodlInvoiceSuite(t *testing.T) { + suite.Run(t, new(HodlInvoiceSuite)) +} diff --git a/integration_tests/lnd_mock_hodl.go b/integration_tests/lnd_mock_hodl.go index 42ca2610..c68313a9 100644 --- a/integration_tests/lnd_mock_hodl.go +++ b/integration_tests/lnd_mock_hodl.go @@ -2,50 +2,56 @@ package integration_tests import ( "context" - "errors" "github.com/getAlby/lndhub.go/lnd" "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" "google.golang.org/grpc" ) -type LNDMockHodlWrapper struct { - lnd.LightningClientWrapper -} - func NewLNDMockHodlWrapper(lnd lnd.LightningClientWrapper) (result *LNDMockWrapper, err error) { return &LNDMockWrapper{ lnd, }, nil } -func (wrapper *LNDMockHodlWrapper) SendPaymentSync(ctx context.Context, req *lnrpc.SendRequest, options ...grpc.CallOption) (*lnrpc.SendResponse, error) { - return nil, errors.New(SendPaymentMockError) +type LNDMockHodlWrapperAsync struct { + hps *HodlPaymentSubscriber + lnd.LightningClientWrapper } -// mock where send payment sync failure is controlled by channel -// even though send payment method is still sync, suffix "Async" here is used to show intention of using this mock -var paymentResultChannel = make(chan bool, 1) +type HodlPaymentSubscriber struct { + ch chan (lnrpc.Payment) +} -type LNDMockHodlWrapperAsync struct { - lnd.LightningClientWrapper +// wait for channel, then return +func (hps *HodlPaymentSubscriber) Recv() (lnrpc.Payment, error) { + return <-hps.ch, nil } -func NewLNDMockHodlWrapperAsync(lnd lnd.LightningClientWrapper) (result *LNDMockWrapperAsync, err error) { - return &LNDMockWrapperAsync{ - lnd, +func NewLNDMockHodlWrapperAsync(lnd lnd.LightningClientWrapper) (result *LNDMockHodlWrapperAsync, err error) { + return &LNDMockHodlWrapperAsync{ + hps: &HodlPaymentSubscriber{ + ch: make(chan lnrpc.Payment), + }, + LightningClientWrapper: lnd, }, nil } +func (wrapper *LNDMockHodlWrapperAsync) SubscribePayment(ctx context.Context, req *routerrpc.TrackPaymentRequest, options ...grpc.CallOption) (lnd.SubscribePaymentWrapper, error) { + return nil, nil +} + func (wrapper *LNDMockHodlWrapperAsync) SendPaymentSync(ctx context.Context, req *lnrpc.SendRequest, options ...grpc.CallOption) (*lnrpc.SendResponse, error) { //block indefinetely + //because we don't want this function to ever return something here + //the payments should be processed asynchronously by the payment tracker select {} } -func (wrapper *LNDMockHodlWrapperAsync) SettlePayment(success bool) { - paymentResultChannel <- success +func (wrapper *LNDMockHodlWrapperAsync) SettlePayment(payment lnrpc.Payment) { + wrapper.hps.ch <- payment } -//TODO: payment tracker implemantation: read from channel, return to receive method //write test that completes payment //write test that fails payment From a633837d259d5912b457526934d7b4f10baa0569 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 13:57:11 +0100 Subject: [PATCH 14/22] more integration test work --- integration_tests/hodl_invoice_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/integration_tests/hodl_invoice_test.go b/integration_tests/hodl_invoice_test.go index b2a82e9e..82c83f72 100644 --- a/integration_tests/hodl_invoice_test.go +++ b/integration_tests/hodl_invoice_test.go @@ -96,11 +96,12 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { // pay external from user, req will be canceled after 2 sec go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken) - // wait for request to fail - time.Sleep(5 * time.Second) + // wait for payment to be updated as pending in database + time.Sleep(3 * time.Second) - //TODO - //start payment tracking loop + //start payment checking loop + err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) + assert.NoError(suite.T(), err) //send settle invoice with lnrpc.payment // check to see that balance was reduced @@ -135,7 +136,7 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { // pay external from user, req will be canceled after 2 sec go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken) - // wait for request to fail + // wait for payment to be updated as pending in database time.Sleep(5 * time.Second) // check to see that balance was reduced @@ -146,9 +147,10 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { } assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) - //TODO - //start payment tracking loop - //send cancel invoice with lnrpc.payment + //start payment checking loop + err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) + assert.NoError(suite.T(), err) + //todo: send cancel invoice with lnrpc.payment // check that balance was reverted and invoice is in error state userBalance, err = suite.service.CurrentUserBalance(context.Background(), userId) From 2db239868aec0f408259bc2ab9c71dbeb78d1506 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 14:05:02 +0100 Subject: [PATCH 15/22] more integration test work --- integration_tests/hodl_invoice_test.go | 31 +++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/integration_tests/hodl_invoice_test.go b/integration_tests/hodl_invoice_test.go index 82c83f72..f9ab375e 100644 --- a/integration_tests/hodl_invoice_test.go +++ b/integration_tests/hodl_invoice_test.go @@ -2,6 +2,7 @@ package integration_tests import ( "context" + "encoding/hex" "fmt" "log" "testing" @@ -103,6 +104,19 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) assert.NoError(suite.T(), err) //send settle invoice with lnrpc.payment + suite.hodlLND.SettlePayment(lnrpc.Payment{ + PaymentHash: hex.EncodeToString(invoice.RHash), + Value: externalInvoice.Value, + CreationDate: 0, + Fee: 0, + PaymentPreimage: "", + ValueSat: 0, + ValueMsat: 0, + PaymentRequest: invoice.PaymentRequest, + Status: lnrpc.Payment_SUCCEEDED, + FailureReason: 0, + }) + // check payment is pending // check to see that balance was reduced userId := getUserIdFromToken(suite.userToken) @@ -111,8 +125,7 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { fmt.Printf("Error when getting balance %v\n", err.Error()) } assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) - //todo: check that invoice was updated as completed - + // check payment is paid } func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { @@ -150,7 +163,19 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { //start payment checking loop err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) assert.NoError(suite.T(), err) - //todo: send cancel invoice with lnrpc.payment + //send cancel invoice with lnrpc.payment + suite.hodlLND.SettlePayment(lnrpc.Payment{ + PaymentHash: hex.EncodeToString(invoice.RHash), + Value: externalInvoice.Value, + CreationDate: 0, + Fee: 0, + PaymentPreimage: "", + ValueSat: 0, + ValueMsat: 0, + PaymentRequest: invoice.PaymentRequest, + Status: lnrpc.Payment_FAILED, + FailureReason: lnrpc.PaymentFailureReason_FAILURE_REASON_INCORRECT_PAYMENT_DETAILS, + }) // check that balance was reverted and invoice is in error state userBalance, err = suite.service.CurrentUserBalance(context.Background(), userId) From 1edfe2b3bf4f6e4a4122c95a31c78ae70e8573fc Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 14:14:53 +0100 Subject: [PATCH 16/22] more integration test work --- integration_tests/hodl_invoice_test.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/integration_tests/hodl_invoice_test.go b/integration_tests/hodl_invoice_test.go index f9ab375e..ae852415 100644 --- a/integration_tests/hodl_invoice_test.go +++ b/integration_tests/hodl_invoice_test.go @@ -96,10 +96,15 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { assert.NoError(suite.T(), err) // pay external from user, req will be canceled after 2 sec go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken) - + // check to see that balance was reduced + userId := getUserIdFromToken(suite.userToken) + userBalance, err := suite.service.CurrentUserBalance(context.Background(), userId) // wait for payment to be updated as pending in database time.Sleep(3 * time.Second) - + // check payment is pending + inv, err := suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State) //start payment checking loop err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) assert.NoError(suite.T(), err) @@ -116,16 +121,15 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { Status: lnrpc.Payment_SUCCEEDED, FailureReason: 0, }) - // check payment is pending - // check to see that balance was reduced - userId := getUserIdFromToken(suite.userToken) - userBalance, err := suite.service.CurrentUserBalance(context.Background(), userId) if err != nil { fmt.Printf("Error when getting balance %v\n", err.Error()) } assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) - // check payment is paid + // check payment is updated as succesful + inv, err = suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State) } func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { @@ -160,6 +164,11 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { } assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) + // check payment is pending + inv, err := suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State) + //start payment checking loop err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) assert.NoError(suite.T(), err) From bccd73312a6bf9af94bc51a5c96faeb82eabc058 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 14:52:15 +0100 Subject: [PATCH 17/22] tests should be working? --- integration_tests/hodl_invoice_test.go | 20 ++++++++++++++------ integration_tests/lnd_mock.go | 2 +- integration_tests/lnd_mock_hodl.go | 7 ++++--- lib/service/checkpayments.go | 2 +- lib/service/invoices.go | 10 +++++----- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/integration_tests/hodl_invoice_test.go b/integration_tests/hodl_invoice_test.go index ae852415..0e6cba20 100644 --- a/integration_tests/hodl_invoice_test.go +++ b/integration_tests/hodl_invoice_test.go @@ -114,22 +114,27 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { Value: externalInvoice.Value, CreationDate: 0, Fee: 0, - PaymentPreimage: "", - ValueSat: 0, + PaymentPreimage: "123preimage", + ValueSat: externalInvoice.Value, ValueMsat: 0, PaymentRequest: invoice.PaymentRequest, Status: lnrpc.Payment_SUCCEEDED, FailureReason: 0, }) + //wait a bit for db update to happen + time.Sleep(time.Second) if err != nil { fmt.Printf("Error when getting balance %v\n", err.Error()) } + //fetch user balance again + userBalance, err = suite.service.CurrentUserBalance(context.Background(), userId) + assert.NoError(suite.T(), err) assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) // check payment is updated as succesful inv, err = suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) assert.NoError(suite.T(), err) - assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State) + assert.Equal(suite.T(), common.InvoiceStateSettled, inv.State) } func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { @@ -178,13 +183,15 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { Value: externalInvoice.Value, CreationDate: 0, Fee: 0, - PaymentPreimage: "", - ValueSat: 0, + PaymentPreimage: "123preimage", + ValueSat: externalInvoice.Value, ValueMsat: 0, PaymentRequest: invoice.PaymentRequest, Status: lnrpc.Payment_FAILED, FailureReason: lnrpc.PaymentFailureReason_FAILURE_REASON_INCORRECT_PAYMENT_DETAILS, }) + //wait a bit for db update to happen + time.Sleep(time.Second) // check that balance was reverted and invoice is in error state userBalance, err = suite.service.CurrentUserBalance(context.Background(), userId) @@ -199,7 +206,8 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { } assert.Equal(suite.T(), 1, len(invoices)) assert.Equal(suite.T(), common.InvoiceStateError, invoices[0].State) - assert.Equal(suite.T(), SendPaymentMockError, invoices[0].ErrorMessage) + errorString := "FAILURE_REASON_INCORRECT_PAYMENT_DETAILS" + assert.Equal(suite.T(), errorString, invoices[0].ErrorMessage) transactonEntries, err := suite.service.TransactionEntriesFor(context.Background(), userId) if err != nil { diff --git a/integration_tests/lnd_mock.go b/integration_tests/lnd_mock.go index 86c51ff0..9f01a35c 100644 --- a/integration_tests/lnd_mock.go +++ b/integration_tests/lnd_mock.go @@ -231,7 +231,7 @@ func (mlnd *MockLND) DecodeBolt11(ctx context.Context, bolt11 string, options .. } result := &lnrpc.PayReq{ Destination: hex.EncodeToString(inv.Destination.SerializeCompressed()), - PaymentHash: string(inv.PaymentHash[:]), + PaymentHash: hex.EncodeToString(inv.PaymentHash[:]), NumSatoshis: int64(*inv.MilliSat) / 1000, Timestamp: inv.Timestamp.Unix(), Expiry: int64(inv.Expiry()), diff --git a/integration_tests/lnd_mock_hodl.go b/integration_tests/lnd_mock_hodl.go index c68313a9..01f7c5a3 100644 --- a/integration_tests/lnd_mock_hodl.go +++ b/integration_tests/lnd_mock_hodl.go @@ -25,8 +25,9 @@ type HodlPaymentSubscriber struct { } // wait for channel, then return -func (hps *HodlPaymentSubscriber) Recv() (lnrpc.Payment, error) { - return <-hps.ch, nil +func (hps *HodlPaymentSubscriber) Recv() (*lnrpc.Payment, error) { + result := <-hps.ch + return &result, nil } func NewLNDMockHodlWrapperAsync(lnd lnd.LightningClientWrapper) (result *LNDMockHodlWrapperAsync, err error) { @@ -39,7 +40,7 @@ func NewLNDMockHodlWrapperAsync(lnd lnd.LightningClientWrapper) (result *LNDMock } func (wrapper *LNDMockHodlWrapperAsync) SubscribePayment(ctx context.Context, req *routerrpc.TrackPaymentRequest, options ...grpc.CallOption) (lnd.SubscribePaymentWrapper, error) { - return nil, nil + return wrapper.hps, nil } func (wrapper *LNDMockHodlWrapperAsync) SendPaymentSync(ctx context.Context, req *lnrpc.SendRequest, options ...grpc.CallOption) (*lnrpc.SendResponse, error) { diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 5f844d23..891951fb 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -24,6 +24,7 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( //spawn goroutines //https://go.dev/doc/faq#closures_and_goroutines inv := inv + svc.Logger.Infof("Spawning tracker for payment with hash %s", inv.RHash) go svc.TrackOutgoingPaymentstatus(ctx, &inv) } return nil @@ -31,7 +32,6 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( // Should be called in a goroutine as the tracking can potentially take a long time func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoice *models.Invoice) { - fmt.Println(invoice.RHash) //ask lnd using TrackPaymentV2 by hash of payment rawHash, err := hex.DecodeString(invoice.RHash) if err != nil { diff --git a/lib/service/invoices.go b/lib/service/invoices.go index 601fd235..f62acc45 100644 --- a/lib/service/invoices.go +++ b/lib/service/invoices.go @@ -251,7 +251,7 @@ func (svc *LndhubService) HandleFailedPayment(ctx context.Context, invoice *mode _, err := svc.DB.NewInsert().Model(&entry).Exec(ctx) if err != nil { sentry.CaptureException(err) - svc.Logger.Errorf("Could not insert transaction entry user_id:%v invoice_id:%v", invoice.UserID, invoice.ID) + svc.Logger.Errorf("Could not insert transaction entry user_id:%v invoice_id:%v error %s", invoice.UserID, invoice.ID, err.Error()) return err } @@ -263,7 +263,7 @@ func (svc *LndhubService) HandleFailedPayment(ctx context.Context, invoice *mode _, err = svc.DB.NewUpdate().Model(invoice).WherePK().Exec(ctx) if err != nil { sentry.CaptureException(err) - svc.Logger.Errorf("Could not update failed payment invoice user_id:%v invoice_id:%v", invoice.UserID, invoice.ID) + svc.Logger.Errorf("Could not update failed payment invoice user_id:%v invoice_id:%v error %s", invoice.UserID, invoice.ID, err.Error()) } return err } @@ -275,7 +275,7 @@ func (svc *LndhubService) HandleSuccessfulPayment(ctx context.Context, invoice * _, err := svc.DB.NewUpdate().Model(invoice).WherePK().Exec(ctx) if err != nil { sentry.CaptureException(err) - svc.Logger.Errorf("Could not update sucessful payment invoice user_id:%v invoice_id:%v", invoice.UserID, invoice.ID) + svc.Logger.Errorf("Could not update sucessful payment invoice user_id:%v invoice_id:%v, error %s", invoice.UserID, invoice.ID, err.Error()) } // Get the user's fee account for the transaction entry, current account is already there in parent entry @@ -297,14 +297,14 @@ func (svc *LndhubService) HandleSuccessfulPayment(ctx context.Context, invoice * _, err = svc.DB.NewInsert().Model(&entry).Exec(ctx) if err != nil { sentry.CaptureException(err) - svc.Logger.Errorf("Could not insert fee transaction entry user_id:%v invoice_id:%v", invoice.UserID, invoice.ID) + svc.Logger.Errorf("Could not insert fee transaction entry user_id:%v invoice_id:%v error %s", invoice.UserID, invoice.ID, err.Error()) return err } userBalance, err := svc.CurrentUserBalance(ctx, entry.UserID) if err != nil { sentry.CaptureException(err) - svc.Logger.Errorf("Could not fetch user balance user_id:%v invoice_id:%v", invoice.UserID, invoice.ID) + svc.Logger.Errorf("Could not fetch user balance user_id:%v invoice_id:%v error %s", invoice.UserID, invoice.ID, err.Error()) return err } From e93ea670bac69c59b326e54ee0a7ae30406d6cd1 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 2 Dec 2022 15:06:01 +0100 Subject: [PATCH 18/22] tests finally working --- integration_tests/hodl_invoice_test.go | 120 ++++++++++++------------- 1 file changed, 55 insertions(+), 65 deletions(-) diff --git a/integration_tests/hodl_invoice_test.go b/integration_tests/hodl_invoice_test.go index 0e6cba20..c731dd78 100644 --- a/integration_tests/hodl_invoice_test.go +++ b/integration_tests/hodl_invoice_test.go @@ -76,7 +76,7 @@ func (suite *HodlInvoiceSuite) SetupSuite() { suite.echo.POST("/payinvoice", controllers.NewPayInvoiceController(suite.service).PayInvoice) } -func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { +func (suite *HodlInvoiceSuite) TestHodlInvoice() { userFundingSats := 1000 externalSatRequested := 500 // fund user account @@ -89,69 +89,9 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceSuccess() { // create external invoice externalInvoice := lnrpc.Invoice{ - Memo: "integration tests: external pay from user", - Value: int64(externalSatRequested), - } - invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice) - assert.NoError(suite.T(), err) - // pay external from user, req will be canceled after 2 sec - go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken) - // check to see that balance was reduced - userId := getUserIdFromToken(suite.userToken) - userBalance, err := suite.service.CurrentUserBalance(context.Background(), userId) - // wait for payment to be updated as pending in database - time.Sleep(3 * time.Second) - // check payment is pending - inv, err := suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State) - //start payment checking loop - err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) - assert.NoError(suite.T(), err) - //send settle invoice with lnrpc.payment - suite.hodlLND.SettlePayment(lnrpc.Payment{ - PaymentHash: hex.EncodeToString(invoice.RHash), - Value: externalInvoice.Value, - CreationDate: 0, - Fee: 0, - PaymentPreimage: "123preimage", - ValueSat: externalInvoice.Value, - ValueMsat: 0, - PaymentRequest: invoice.PaymentRequest, - Status: lnrpc.Payment_SUCCEEDED, - FailureReason: 0, - }) - //wait a bit for db update to happen - time.Sleep(time.Second) - - if err != nil { - fmt.Printf("Error when getting balance %v\n", err.Error()) - } - //fetch user balance again - userBalance, err = suite.service.CurrentUserBalance(context.Background(), userId) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) - // check payment is updated as succesful - inv, err = suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), common.InvoiceStateSettled, inv.State) -} - -func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { - userFundingSats := 1000 - externalSatRequested := 500 - // fund user account - invoiceResponse := suite.createAddInvoiceReq(userFundingSats, "integration test external payment user", suite.userToken) - err := suite.mlnd.mockPaidInvoice(invoiceResponse, 0, false, nil) - assert.NoError(suite.T(), err) - - // wait a bit for the callback event to hit - time.Sleep(10 * time.Millisecond) - - // create external invoice - externalInvoice := lnrpc.Invoice{ - Memo: "integration tests: external pay from user", - Value: int64(externalSatRequested), + Memo: "integration tests: external pay from user", + Value: int64(externalSatRequested), + RPreimage: []byte("preimage1"), } invoice, err := suite.externalLND.AddInvoice(context.Background(), &externalInvoice) assert.NoError(suite.T(), err) @@ -183,7 +123,7 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { Value: externalInvoice.Value, CreationDate: 0, Fee: 0, - PaymentPreimage: "123preimage", + PaymentPreimage: "", ValueSat: externalInvoice.Value, ValueMsat: 0, PaymentRequest: invoice.PaymentRequest, @@ -219,6 +159,56 @@ func (suite *HodlInvoiceSuite) TestHodlInvoiceFailure() { assert.Equal(suite.T(), transactonEntries[1].DebitAccountID, transactonEntries[2].CreditAccountID) assert.Equal(suite.T(), transactonEntries[1].Amount, int64(externalSatRequested)) assert.Equal(suite.T(), transactonEntries[2].Amount, int64(externalSatRequested)) + + // create external invoice + externalInvoice = lnrpc.Invoice{ + Memo: "integration tests: external pay from user", + Value: int64(externalSatRequested), + RPreimage: []byte("preimage2"), + } + invoice, err = suite.externalLND.AddInvoice(context.Background(), &externalInvoice) + assert.NoError(suite.T(), err) + // pay external from user, req will be canceled after 2 sec + go suite.createPayInvoiceReqWithCancel(invoice.PaymentRequest, suite.userToken) + // wait for payment to be updated as pending in database + time.Sleep(3 * time.Second) + // check payment is pending + inv, err = suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), common.InvoiceStateInitialized, inv.State) + //start payment checking loop + err = suite.service.CheckAllPendingOutgoingPayments(context.Background()) + assert.NoError(suite.T(), err) + //send settle invoice with lnrpc.payment + suite.hodlLND.SettlePayment(lnrpc.Payment{ + PaymentHash: hex.EncodeToString(invoice.RHash), + Value: externalInvoice.Value, + CreationDate: 0, + Fee: 0, + PaymentPreimage: "preimage2", + ValueSat: externalInvoice.Value, + ValueMsat: 0, + PaymentRequest: invoice.PaymentRequest, + Status: lnrpc.Payment_SUCCEEDED, + FailureReason: 0, + }) + //wait a bit for db update to happen + time.Sleep(time.Second) + + if err != nil { + fmt.Printf("Error when getting balance %v\n", err.Error()) + } + //fetch user balance again + userBalance, err = suite.service.CurrentUserBalance(context.Background(), userId) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), int64(userFundingSats-externalSatRequested), userBalance) + // check payment is updated as succesful + inv, err = suite.service.FindInvoiceByPaymentHash(context.Background(), userId, hex.EncodeToString(invoice.RHash)) + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), common.InvoiceStateSettled, inv.State) + clearTable(suite.service, "invoices") + clearTable(suite.service, "transaction_entries") + clearTable(suite.service, "accounts") } func (suite *HodlInvoiceSuite) TearDownSuite() { From 5e0424941b8354d92a7651b4bc5c89bb6d42b2f4 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Mon, 5 Dec 2022 17:14:15 +0100 Subject: [PATCH 19/22] add time filter when fetching invoices --- lib/service/checkpayments.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 891951fb..8880ea45 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -14,7 +14,8 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( //check database for all pending payments pendingPayments := []models.Invoice{} //since this part is synchronously executed before the main server starts, we should not get into race conditions - err = svc.DB.NewSelect().Model(&pendingPayments).Where("state = 'initialized'").Where("type = 'outgoing'").Scan(ctx) + //only fetch invoices from the last 2 weeks which should be a safe timeframe for hodl invoices to avoid refetching old invoices again and again + err = svc.DB.NewSelect().Model(&pendingPayments).Where("state = 'initialized'").Where("type = 'outgoing'").Where("created_at >= (now() - interval '2 weeks') ").Scan(ctx) if err != nil { return err } From ba99c491240e252203afd57be9f166a8b79774de Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Mon, 5 Dec 2022 17:16:09 +0100 Subject: [PATCH 20/22] add some sentry captures --- lib/service/checkpayments.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 8880ea45..1eeedcbc 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/getAlby/lndhub.go/db/models" + "github.com/getsentry/sentry-go" "github.com/lightningnetwork/lnd/lnrpc" "github.com/lightningnetwork/lnd/lnrpc/routerrpc" ) @@ -71,6 +72,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic //todo handle failed payment err = svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) if err != nil { + sentry.CaptureException(err) svc.Logger.Errorf("Error handling failed payment %v: %s", invoice, err.Error()) return } @@ -83,6 +85,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic svc.Logger.Infof("Completed payment detected: hash %s", payment.PaymentHash) err = svc.HandleSuccessfulPayment(ctx, invoice, entry) if err != nil { + sentry.CaptureException(err) svc.Logger.Errorf("Error handling successful payment %v: %s", invoice, err.Error()) return } @@ -90,6 +93,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic return } //Since we shouldn't get in-flight updates we shouldn't get here + sentry.CaptureException(fmt.Errorf("Got an unexpected payment update %v", payment)) svc.Logger.Warnf("Got an unexpected in-flight update %v", payment) } } From e63917a91922073c140f0753b0043f58d4184523 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 6 Dec 2022 11:32:17 +0100 Subject: [PATCH 21/22] better logging + don't fetch invoices without hash --- .gitignore | 2 ++ lib/service/checkpayments.go | 9 ++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 477faf08..cbf20ef1 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ vendor/ # env, use .env_example for reference .env + +initialized_payments_dryrun/* \ No newline at end of file diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index 1eeedcbc..e8af8c6d 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -16,7 +16,7 @@ func (svc *LndhubService) CheckAllPendingOutgoingPayments(ctx context.Context) ( pendingPayments := []models.Invoice{} //since this part is synchronously executed before the main server starts, we should not get into race conditions //only fetch invoices from the last 2 weeks which should be a safe timeframe for hodl invoices to avoid refetching old invoices again and again - err = svc.DB.NewSelect().Model(&pendingPayments).Where("state = 'initialized'").Where("type = 'outgoing'").Where("created_at >= (now() - interval '2 weeks') ").Scan(ctx) + err = svc.DB.NewSelect().Model(&pendingPayments).Where("state = 'initialized'").Where("type = 'outgoing'").Where("r_hash != ''").Where("created_at >= (now() - interval '2 weeks') ").Scan(ctx) if err != nil { return err } @@ -64,16 +64,15 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic for { payment, err := paymentTracker.Recv() if err != nil { - svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + svc.Logger.Errorf("Error tracking payment with hash %s: %s", invoice.RHash, err.Error()) return } if payment.Status == lnrpc.Payment_FAILED { svc.Logger.Infof("Failed payment detected: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) - //todo handle failed payment err = svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String())) if err != nil { sentry.CaptureException(err) - svc.Logger.Errorf("Error handling failed payment %v: %s", invoice, err.Error()) + svc.Logger.Errorf("Error handling failed payment %s: %s", invoice.RHash, err.Error()) return } svc.Logger.Infof("Updated failed payment: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) @@ -86,7 +85,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic err = svc.HandleSuccessfulPayment(ctx, invoice, entry) if err != nil { sentry.CaptureException(err) - svc.Logger.Errorf("Error handling successful payment %v: %s", invoice, err.Error()) + svc.Logger.Errorf("Error handling successful payment %s: %s", invoice.RHash, err.Error()) return } svc.Logger.Infof("Updated completed payment: hash %s", payment.PaymentHash) From 47690dba7eabf0fc3c772673c5c65c2081e0234e Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Tue, 6 Dec 2022 11:40:21 +0100 Subject: [PATCH 22/22] avoid unnecessary db calls --- lib/service/checkpayments.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/service/checkpayments.go b/lib/service/checkpayments.go index e8af8c6d..76a7538d 100644 --- a/lib/service/checkpayments.go +++ b/lib/service/checkpayments.go @@ -37,7 +37,7 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic //ask lnd using TrackPaymentV2 by hash of payment rawHash, err := hex.DecodeString(invoice.RHash) if err != nil { - svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + svc.Logger.Errorf("Error tracking payment %s: %s", invoice.RHash, err.Error()) return } paymentTracker, err := svc.LndClient.SubscribePayment(ctx, &routerrpc.TrackPaymentRequest{ @@ -45,21 +45,11 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic NoInflightUpdates: true, }) if err != nil { - svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) + svc.Logger.Errorf("Error tracking payment %s: %s", invoice.RHash, err.Error()) return } //fetch the tx entry for the invoice entry := models.TransactionEntry{} - err = svc.DB.NewSelect().Model(&entry).Where("invoice_id = ?", invoice.ID).Limit(1).Scan(ctx) - if err != nil { - svc.Logger.Errorf("Error tracking payment %v: %s", invoice, err.Error()) - return - - } - if entry.UserID != invoice.UserID { - svc.Logger.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) - return - } //call HandleFailedPayment or HandleSuccesfulPayment for { payment, err := paymentTracker.Recv() @@ -67,6 +57,16 @@ func (svc *LndhubService) TrackOutgoingPaymentstatus(ctx context.Context, invoic svc.Logger.Errorf("Error tracking payment with hash %s: %s", invoice.RHash, err.Error()) return } + err = svc.DB.NewSelect().Model(&entry).Where("invoice_id = ?", invoice.ID).Limit(1).Scan(ctx) + if err != nil { + svc.Logger.Errorf("Error tracking payment %s: %s", invoice.RHash, err.Error()) + return + + } + if entry.UserID != invoice.UserID { + svc.Logger.Errorf("User ID's don't match : entry %v, invoice %v", entry, invoice) + return + } if payment.Status == lnrpc.Payment_FAILED { svc.Logger.Infof("Failed payment detected: hash %s, reason %s", payment.PaymentHash, payment.FailureReason) err = svc.HandleFailedPayment(ctx, invoice, entry, fmt.Errorf(payment.FailureReason.String()))