Merge pull request #906 from abyssparanoia/feature/impl-rdb-tx-retry

Feature/impl rdb tx retry

go.mod

@@ -7,6 +7,7 @@ require (
 	cloud.google.com/go/storage v1.36.0
 	firebase.google.com/go v3.13.0+incompatible
 	github.com/abyssparanoia/memeduck v1.0.3
+	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.49.9
 	github.com/blendle/zapdriver v1.3.1
 	github.com/bufbuild/buf v1.28.1

go.sum

@@ -76,8 +76,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
-cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
-cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
 cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8=
 cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
 connectrpc.com/connect v1.12.0 h1:HwKdOY0lGhhoHdsza+hW55aqHEC64pYpObRNoAgn70g=
@@ -171,8 +169,8 @@ github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8ger
 github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU=
 github.com/ashanbrown/makezero v1.1.1 h1:iCQ87C0V0vSyO+M9E/FZYbu65auqH0lnsOkf5FcB28s=
 github.com/ashanbrown/makezero v1.1.1/go.mod h1:i1bJLCRSCHOcOa9Y6MyF2FTfMZMFdHvxKHxgO5Z1axI=
-github.com/aws/aws-sdk-go v1.45.6 h1:Y2isQQBZsnO15dzUQo9YQRThtHgrV200XCH05BRHVJI=
-github.com/aws/aws-sdk-go v1.45.6/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.49.9 h1:4xoyi707rsifB1yMsd5vGbAH21aBzwpL3gNRMSmjIyc=
 github.com/aws/aws-sdk-go v1.49.9/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -243,8 +241,7 @@ github.com/cli/safeexec v1.0.0/go.mod h1:Z/D4tTN8Vs5gXYHDCbaM1S/anmEDnJb1iW0+EJ5
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudspannerecosystem/memefish v0.0.0-20231128072053-0a1141e8eb65 h1:Rg9tANZcL6gdZlY1W5TeWMD4pIQN63qkAhS+HhjhbPA=
 github.com/cloudspannerecosystem/memefish v0.0.0-20231128072053-0a1141e8eb65/go.mod h1:Q40NmYZbmaw9ay7p94ng9NNBuX1N+4OK3cSjN4q5tPM=
-github.com/cloudspannerecosystem/wrench v1.5.0 h1:FUuIF32f7Rc8LtSC5DOXQBxMX/cPhsvCJrhV/kqKLc8=
-github.com/cloudspannerecosystem/wrench v1.5.0/go.mod h1:xKD6Zip4SHjdww6z2Mu259eRQv0ssFUFtPmEq/jLnnQ=
+github.com/cloudspannerecosystem/wrench v1.6.0 h1:gp45vWNiwO8sbd2nZvBL90FCKJkqxm8+XcxsXaFGyXo=
 github.com/cloudspannerecosystem/wrench v1.6.0/go.mod h1:kx4F5+m83oBNs7/6vH1GVzbAJ+Dq6irxHwEN4nTN6M4=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -366,8 +363,6 @@ github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk=
 github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
 github.com/go-critic/go-critic v0.9.0 h1:Pmys9qvU3pSML/3GEQ2Xd9RZ/ip+aXHKILuxczKGV/U=
 github.com/go-critic/go-critic v0.9.0/go.mod h1:5P8tdXL7m/6qnyG6oRAlYLORvoXH0WDypYgAEmagT40=
-github.com/go-faker/faker/v4 v4.1.1 h1:zkxj/JH/aezB4R6cTEMKU7qcVScGhlB3qRtF3D7K+rI=
-github.com/go-faker/faker/v4 v4.1.1/go.mod h1:uuNc0PSRxF8nMgjGrrrU4Nw5cF30Jc6Kd0/FUTTYbhg=
 github.com/go-faker/faker/v4 v4.2.0 h1:dGebOupKwssrODV51E0zbMrv5e2gO9VWSLNC1WDCpWg=
 github.com/go-faker/faker/v4 v4.2.0/go.mod h1:F/bBy8GH9NxOxMInug5Gx4WYeG6fHJZ8Ol/dhcpRub4=
 github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
@@ -448,8 +443,6 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq
 github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE=
-github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
 github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
@@ -567,8 +560,6 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
-github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
 github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=

internal/infrastructure/database/transactable/helper.go

@@ -3,8 +3,12 @@ package transactable
 import (
 	"context"
 	"database/sql"
+	std_errors "errors"
+	"time"
 
 	"github.com/abyssparanoia/rapid-go/internal/domain/errors"
+	"github.com/avast/retry-go"
+	"github.com/go-sql-driver/mysql"
 	"github.com/volatiletech/sqlboiler/v4/boil"
 )
 
@@ -31,31 +35,67 @@ var RunTx = func(ctx context.Context, fn func(context.Context) error) error {
 
 // RunTxWithDB :.
 func runTxWithDB(ctx context.Context, db boil.ContextBeginner, fn func(context.Context) error) error {
-	tx, err := db.BeginTx(ctx, nil)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := recover(); err != nil {
-			if err := tx.Rollback(); err != nil {
+	txFn := func() error {
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			return errors.InternalErr.Wrap(err)
+		}
+		defer func() {
+			if err := recover(); err != nil {
+				if err := tx.Rollback(); err != nil {
+					panic(err)
+				}
 				panic(err)
 			}
-			panic(err)
-		}
-	}()
+		}()
 
-	ctxWithTx := context.WithValue(ctx, &ctxTxKey, tx)
-	if err := fn(ctxWithTx); err != nil {
-		if err := tx.Rollback(); err != nil {
+		ctxWithTx := context.WithValue(ctx, &ctxTxKey, tx)
+		if err := fn(ctxWithTx); err != nil {
+			if err := tx.Rollback(); err != nil {
+				return errors.InternalErr.Wrap(err)
+			}
+			return err
+		}
+		if err := tx.Commit(); err != nil {
+			if err := tx.Rollback(); err != nil {
+				return errors.InternalErr.Wrap(err)
+			}
 			return errors.InternalErr.Wrap(err)
 		}
+		return nil
+	}
+
+	if err := retry.Do(
+		txFn,
+		retry.RetryIf(func(err error) bool {
+			return isDeadLock(err)
+		}),
+		retry.DelayType(func(n uint, err error, config *retry.Config) time.Duration {
+			return time.Duration(n) * time.Second
+		}),
+		retry.Attempts(4),
+		retry.LastErrorOnly(true),
+	); err != nil {
 		return err
 	}
-	if err := tx.Commit(); err != nil {
-		if err := tx.Rollback(); err != nil {
-			return errors.InternalErr.Wrap(err)
+
+	return nil
+}
+
+func isDeadLock(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	deadlockErr := &mysql.MySQLError{Number: 1213}
+	if std_errors.As(err, &deadlockErr) {
+		for err != nil {
+			err = std_errors.Unwrap(err)
+			if std_errors.Is(err, deadlockErr) {
+				return true
+			}
 		}
-		return errors.InternalErr.Wrap(err)
 	}
-	return nil
+
+	return false
 }