From 6739e8c9c760b3de2ba63bb954843ce4bc06c40e Mon Sep 17 00:00:00 2001 From: Michael Bumann Date: Fri, 17 Feb 2023 01:49:06 +0100 Subject: [PATCH 1/7] Notify sentry if the select for the last add index fails --- lib/service/invoicesubscription.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/service/invoicesubscription.go b/lib/service/invoicesubscription.go index 8545cc6a..544d6eb6 100644 --- a/lib/service/invoicesubscription.go +++ b/lib/service/invoicesubscription.go @@ -222,6 +222,8 @@ func (svc *LndhubService) ConnectInvoiceSubscription(ctx context.Context) (lnd.S // IF we found an invoice we use that index to start the subscription if err == nil { invoiceSubscriptionOptions = lnrpc.InvoiceSubscription{AddIndex: invoice.AddIndex - 1} // -1 because we want updates for that invoice already + } else { + sentry.CaptureException(err) } svc.Logger.Infof("Starting invoice subscription from index: %v", invoiceSubscriptionOptions.AddIndex) return svc.LndClient.SubscribeInvoices(ctx, &invoiceSubscriptionOptions) From 6417298677f3e17459ed0499c59c1b0b9b1e3a21 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 17 Feb 2023 08:32:17 +0100 Subject: [PATCH 2/7] add safety buffer --- lib/service/invoicesubscription.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/service/invoicesubscription.go b/lib/service/invoicesubscription.go index 544d6eb6..4b2ca7bd 100644 --- a/lib/service/invoicesubscription.go +++ b/lib/service/invoicesubscription.go @@ -217,12 +217,14 @@ func (svc *LndhubService) ConnectInvoiceSubscription(ctx context.Context) (lnd.S var invoice models.Invoice invoiceSubscriptionOptions := lnrpc.InvoiceSubscription{} // Find the oldest NOT settled AND NOT expired invoice with an add_index - // Note: expired invoices will not be settled anymore, so we don't care about those - err := svc.DB.NewSelect().Model(&invoice).Where("invoice.settled_at IS NULL AND invoice.add_index IS NOT NULL AND invoice.expires_at >= now()").OrderExpr("invoice.id ASC").Limit(1).Scan(ctx) + // Build in a safety buffer of 14h to account for lndhub downtime + // Note: expired invoices will not be settled anymore, so we don't care about those + err := svc.DB.NewSelect().Model(&invoice).Where("invoice.settled_at IS NULL AND invoice.add_index IS NOT NULL AND invoice.expires_at >= (now() - interval '14 hours')").OrderExpr("invoice.id ASC").Limit(1).Scan(ctx) // IF we found an invoice we use that index to start the subscription if err == nil { invoiceSubscriptionOptions = lnrpc.InvoiceSubscription{AddIndex: invoice.AddIndex - 1} // -1 because we want updates for that invoice already } else { + svc.Logger.Error(err) sentry.CaptureException(err) } svc.Logger.Infof("Starting invoice subscription from index: %v", invoiceSubscriptionOptions.AddIndex) From bf315cee02bf664d4dfdc73772bc04034c1675b8 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 17 Feb 2023 08:42:32 +0100 Subject: [PATCH 3/7] add db timeout defautl of 60 seconds --- db/db.go | 5 ++++- lib/service/config.go | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db/db.go b/db/db.go index 7f0a437c..cc4d8c3b 100644 --- a/db/db.go +++ b/db/db.go @@ -18,7 +18,10 @@ func Open(config *service.Config) (*bun.DB, error) { dsn := config.DatabaseUri switch { case strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") || strings.HasPrefix(dsn, "unix://"): - dbConn := sql.OpenDB(pgdriver.NewConnector(pgdriver.WithDSN(dsn))) + dbConn := sql.OpenDB( + pgdriver.NewConnector( + pgdriver.WithDSN(dsn), + pgdriver.WithTimeout(time.Duration(config.DatabaseTimeout)*time.Second))) db = bun.NewDB(dbConn, pgdialect.New()) db.SetMaxOpenConns(config.DatabaseMaxConns) db.SetMaxIdleConns(config.DatabaseMaxIdleConns) diff --git a/lib/service/config.go b/lib/service/config.go index 4f813237..2c29733b 100644 --- a/lib/service/config.go +++ b/lib/service/config.go @@ -10,6 +10,7 @@ type Config struct { DatabaseMaxConns int `envconfig:"DATABASE_MAX_CONNS" default:"10"` DatabaseMaxIdleConns int `envconfig:"DATABASE_MAX_IDLE_CONNS" default:"5"` DatabaseConnMaxLifetime int `envconfig:"DATABASE_CONN_MAX_LIFETIME" default:"1800"` // 30 minutes + DatabaseTimeout int `envconfig:"DATABASE_TIMEOUT" default:"60"` // 60 seconds SentryDSN string `envconfig:"SENTRY_DSN"` SentryTracesSampleRate float64 `envconfig:"SENTRY_TRACES_SAMPLE_RATE"` LogFilePath string `envconfig:"LOG_FILE_PATH"` From 2c73684b37629c0817a1dd06886b62fb22998466 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 17 Feb 2023 11:12:46 +0100 Subject: [PATCH 4/7] wip: fail on db error --- lib/service/invoicesubscription.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/service/invoicesubscription.go b/lib/service/invoicesubscription.go index 4b2ca7bd..6a2b2339 100644 --- a/lib/service/invoicesubscription.go +++ b/lib/service/invoicesubscription.go @@ -221,11 +221,11 @@ func (svc *LndhubService) ConnectInvoiceSubscription(ctx context.Context) (lnd.S // Note: expired invoices will not be settled anymore, so we don't care about those err := svc.DB.NewSelect().Model(&invoice).Where("invoice.settled_at IS NULL AND invoice.add_index IS NOT NULL AND invoice.expires_at >= (now() - interval '14 hours')").OrderExpr("invoice.id ASC").Limit(1).Scan(ctx) // IF we found an invoice we use that index to start the subscription - if err == nil { - invoiceSubscriptionOptions = lnrpc.InvoiceSubscription{AddIndex: invoice.AddIndex - 1} // -1 because we want updates for that invoice already - } else { - svc.Logger.Error(err) + // if we didn't find any there might be a serious issue and we want to crash + // TODO: exclude no rows in result set error (that's allright) + if err != nil { sentry.CaptureException(err) + svc.Logger.Fatal(err) } svc.Logger.Infof("Starting invoice subscription from index: %v", invoiceSubscriptionOptions.AddIndex) return svc.LndClient.SubscribeInvoices(ctx, &invoiceSubscriptionOptions) From c293825ab97f461247aba3ac87823033dd518286 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 17 Feb 2023 17:03:44 +0100 Subject: [PATCH 5/7] return error in invoice subscription --- lib/service/invoicesubscription.go | 16 +++++++++------- main.go | 5 +++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/service/invoicesubscription.go b/lib/service/invoicesubscription.go index 6a2b2339..1520e431 100644 --- a/lib/service/invoicesubscription.go +++ b/lib/service/invoicesubscription.go @@ -221,11 +221,11 @@ func (svc *LndhubService) ConnectInvoiceSubscription(ctx context.Context) (lnd.S // Note: expired invoices will not be settled anymore, so we don't care about those err := svc.DB.NewSelect().Model(&invoice).Where("invoice.settled_at IS NULL AND invoice.add_index IS NOT NULL AND invoice.expires_at >= (now() - interval '14 hours')").OrderExpr("invoice.id ASC").Limit(1).Scan(ctx) // IF we found an invoice we use that index to start the subscription - // if we didn't find any there might be a serious issue and we want to crash - // TODO: exclude no rows in result set error (that's allright) - if err != nil { - sentry.CaptureException(err) - svc.Logger.Fatal(err) + // if we get an error there might be a serious issue here + // and we are at risk of missing paid invoices, so we should not continue + // if we just didn't find any unsettled invoices that's allright though + if err != nil && err != sql.ErrNoRows { + return nil, err } svc.Logger.Infof("Starting invoice subscription from index: %v", invoiceSubscriptionOptions.AddIndex) return svc.LndClient.SubscribeInvoices(ctx, &invoiceSubscriptionOptions) @@ -240,14 +240,16 @@ func (svc *LndhubService) InvoiceUpdateSubscription(ctx context.Context) error { for { select { case <-ctx.Done(): - return fmt.Errorf("Context was canceled") + return context.Canceled default: // receive the next invoice update rawInvoice, err := invoiceSubscriptionStream.Recv() + // in case of an error, we want to return and restart LNDhub + // in order to try and reconnect the gRPC subscription if err != nil { svc.Logger.Errorf("Error processing invoice update subscription: %v", err) sentry.CaptureException(err) - continue + return err } // Ignore updates for open invoices diff --git a/main.go b/main.go index 1da692b2..2ea44b48 100644 --- a/main.go +++ b/main.go @@ -179,8 +179,9 @@ func main() { backgroundWg.Add(1) go func() { err = svc.InvoiceUpdateSubscription(backGroundCtx) - if err != nil { - svc.Logger.Error(err) + if err != nil && err != context.Canceled { + // in case of an error in this routine, we want to restart LNDhub + svc.Logger.Fatal(err) } svc.Logger.Info("Invoice routine done") backgroundWg.Done() From 973f80a9939298c888109cc80b2cd82033e2b679 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 17 Feb 2023 17:15:06 +0100 Subject: [PATCH 6/7] forgot to re-add subscription --- integration_tests/subscription_start_test.go | 2 +- lib/service/invoicesubscription.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/subscription_start_test.go b/integration_tests/subscription_start_test.go index 25d3e6ec..01683dd4 100644 --- a/integration_tests/subscription_start_test.go +++ b/integration_tests/subscription_start_test.go @@ -125,7 +125,7 @@ func (mock *lndSubscriptionStartMockClient) SubscribeInvoices(ctx context.Contex return mock, nil } -//wait forever +// wait forever func (mock *lndSubscriptionStartMockClient) Recv() (*lnrpc.Invoice, error) { select {} } diff --git a/lib/service/invoicesubscription.go b/lib/service/invoicesubscription.go index 1520e431..93a34290 100644 --- a/lib/service/invoicesubscription.go +++ b/lib/service/invoicesubscription.go @@ -227,6 +227,8 @@ func (svc *LndhubService) ConnectInvoiceSubscription(ctx context.Context) (lnd.S if err != nil && err != sql.ErrNoRows { return nil, err } + // subtract 1 (read invoiceSubscriptionOptions.Addindex docs) + invoiceSubscriptionOptions.AddIndex = invoice.AddIndex - 1 svc.Logger.Infof("Starting invoice subscription from index: %v", invoiceSubscriptionOptions.AddIndex) return svc.LndClient.SubscribeInvoices(ctx, &invoiceSubscriptionOptions) } From e36d46ff9c143be06907b4753bd0d737850048bb Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Fri, 17 Feb 2023 17:22:13 +0100 Subject: [PATCH 7/7] add an extra sentry notification --- lib/service/invoicesubscription.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/service/invoicesubscription.go b/lib/service/invoicesubscription.go index 93a34290..c069a9d0 100644 --- a/lib/service/invoicesubscription.go +++ b/lib/service/invoicesubscription.go @@ -225,6 +225,7 @@ func (svc *LndhubService) ConnectInvoiceSubscription(ctx context.Context) (lnd.S // and we are at risk of missing paid invoices, so we should not continue // if we just didn't find any unsettled invoices that's allright though if err != nil && err != sql.ErrNoRows { + sentry.CaptureException(err) return nil, err } // subtract 1 (read invoiceSubscriptionOptions.Addindex docs)