feat: update x/gov module parsing (forbole#652)

## Description

Closes: #XXXX
[BDU-11](https://forbole.atlassian.net/browse/BDU-11)
[BDU-771](https://forbole.atlassian.net/browse/BDU-771)
[BDU-1077](https://forbole.atlassian.net/browse/BDU-1077)

Changes:
`gov` module:
- Added handler on every block to check and update proposal status if `active_proposal` event has been emitted inside the EndBlockEvents
- Added periodic operations to update `proposal staking pool snapshots` every 5 mins 
- Added periodic operations to update `proposal tally results` every 5 mins 
- Added handler to update `tally results` for given proposal on every `MsgVote` and `MsgVoteWeighted` msgs
- Added handler for rpc error `rpc error returning code = NotFound desc = proposal x doesn't exist` returned from node  sometimes when processing `MsgSubmitProposal` msg and querying the node for proposal details
- Added amount to `unique_deposit` constraint to allow to store different deposits from the same address for the same proposal 
- Added `weight` column to `proposal_vote` table and added `option` to `unique_vote` constraint
- Added handler for `MsgVoteWeighted` msgs to correctly store votes in db
- Removed `auth` module from expected modules

`staking` module:
- Added periodic operations to update validators statuses, voting power and proposal validators status snapshots every 5 mins 
- Added handler to update validators statuses, voting powers and proposals validators status snapshots when there is a VP change on `MsgDelegate`, `MsgBeginRedelegate` and `MsgUndelegate` msgs



---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] added `!` to the type prefix if API or client breaking change
- [x] targeted the correct branch
- [x] provided a link to the relevant issue or specification
- [x] added a changelog entry to `CHANGELOG.md`
- [x] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [x] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [x] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)

[BDU-11]: https://forbole.atlassian.net/browse/BDU-11?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
[BDU-771]: https://forbole.atlassian.net/browse/BDU-771?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ
[BDU-1077]: https://forbole.atlassian.net/browse/BDU-1077?atlOrigin=eyJpIjoiNWRkNTljNzYxNjVmNDY3MDlhMDU5Y2ZhYzA5YTRkZjUiLCJwIjoiZ2l0aHViLWNvbS1KU1cifQ

CHANGELOG.md

@@ -1,6 +1,6 @@
 ## Unreleased
 - ([\#610](https://github.com/forbole/bdjuno/pull/610)) Add support for gov `v1` proposals
-
+- ([\#652](https://github.com/forbole/bdjuno/pull/652)) Update gov module parsing 
 
 ## Version v4.0.0
 ## Notes

cmd/parse/gov/proposal.go

@@ -4,7 +4,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"strconv"
-	"time"
 
 	"github.com/rs/zerolog/log"
 
@@ -56,7 +55,7 @@ func proposalCmd(parseConfig *parsecmdtypes.Config) *cobra.Command {
 			stakingModule := staking.NewModule(sources.StakingSource, parseCtx.EncodingConfig.Codec, db)
 
 			// Build the gov module
-			govModule := gov.NewModule(sources.GovSource, nil, distrModule, mintModule, slashingModule, stakingModule, parseCtx.EncodingConfig.Codec, db)
+			govModule := gov.NewModule(sources.GovSource, distrModule, mintModule, slashingModule, stakingModule, parseCtx.EncodingConfig.Codec, db)
 
 			err = refreshProposalDetails(parseCtx, proposalID, govModule)
 			if err != nil {
@@ -79,7 +78,17 @@ func proposalCmd(parseConfig *parsecmdtypes.Config) *cobra.Command {
 				return fmt.Errorf("error while getting chain latest block height: %s", err)
 			}
 
-			err = govModule.UpdateProposal(height, time.Now(), proposalID)
+			err = govModule.UpdateProposalStatus(height, proposalID)
+			if err != nil {
+				return err
+			}
+
+			err = govModule.UpdateProposalTallyResult(proposalID, height)
+			if err != nil {
+				return err
+			}
+
+			err = govModule.UpdateProposalStakingPoolSnapshot(height, proposalID)
 			if err != nil {
 				return err
 			}
@@ -178,13 +187,23 @@ func refreshProposalVotes(parseCtx *parser.Context, proposalID uint64, govModule
 
 		// Handle the MsgVote messages
 		for index, msg := range junoTx.GetMsgs() {
-			if _, ok := msg.(*govtypesv1.MsgVote); !ok {
+			if msgVote, ok := msg.(*govtypesv1.MsgVote); !ok {
 				continue
-			}
-
-			err = govModule.HandleMsg(index, msg, junoTx)
-			if err != nil {
-				return fmt.Errorf("error while handling MsgVote: %s", err)
+			} else {
+				// check if requested proposal ID is the same as proposal ID returned
+				// from the msg as some txs may contain multiple MsgVote msgs
+				// for different proposals which can cause error if one of the proposals
+				// info is not stored in database
+				if proposalID == msgVote.ProposalId {
+					err = govModule.HandleMsg(index, msg, junoTx)
+					if err != nil {
+						return fmt.Errorf("error while handling MsgVote: %s", err)
+					}
+				} else {
+					// skip votes for proposals with IDs
+					// different than requested in the query
+					continue
+				}
 			}
 		}
 	}

database/auth.go

@@ -98,8 +98,14 @@ func (db *Db) storeVestingAccount(account exported.VestingAccount) (int, error)
 			start_time = excluded.start_time
 			RETURNING id `
 
+	// Store the vesting account
+	err := db.SaveAccounts([]types.Account{types.NewAccount(account.GetAddress().String())})
+	if err != nil {
+		return 0, fmt.Errorf("error while storing vesting account address: %s", err)
+	}
+
 	var vestingAccountRowID int
-	err := db.SQL.QueryRow(stmt,
+	err = db.SQL.QueryRow(stmt,
 		proto.MessageName(account),
 		account.GetAddress().String(),
 		pq.Array(dbtypes.NewDbCoins(account.GetOriginalVesting())),
@@ -124,7 +130,13 @@ func (db *Db) StoreBaseVestingAccountFromMsg(bva *vestingtypes.BaseVestingAccoun
 			start_time = excluded.start_time, 
 			end_time = excluded.end_time`
 
-	_, err := db.SQL.Exec(stmt,
+	// Store the vesting account
+	err := db.SaveAccounts([]types.Account{types.NewAccount(bva.GetAddress().String())})
+	if err != nil {
+		return fmt.Errorf("error while storing vesting account address: %s", err)
+	}
+
+	_, err = db.SQL.Exec(stmt,
 		proto.MessageName(bva),
 		bva.GetAddress().String(),
 		pq.Array(dbtypes.NewDbCoins(bva.OriginalVesting)),

database/consensus.go

@@ -25,20 +25,20 @@ func (db *Db) GetLastBlock() (*dbtypes.BlockRow, error) {
 	return &blocks[0], nil
 }
 
-// GetLastBlockHeight returns the last block height stored inside the database
-func (db *Db) GetLastBlockHeight() (int64, error) {
-	stmt := `SELECT height FROM block ORDER BY height DESC LIMIT 1`
+// GetLastBlockHeight returns the last block height and timestamp stored inside the database
+func (db *Db) GetLastBlockHeightAndTimestamp() (dbtypes.BlockHeightAndTimestamp, error) {
+	stmt := `SELECT height, timestamp FROM block ORDER BY height DESC LIMIT 1`
 
-	var heights []int64
-	if err := db.Sqlx.Select(&heights, stmt); err != nil {
-		return 0, err
+	var blockHeightAndTimestamp []dbtypes.BlockHeightAndTimestamp
+	if err := db.Sqlx.Select(&blockHeightAndTimestamp, stmt); err != nil {
+		return dbtypes.BlockHeightAndTimestamp{}, fmt.Errorf("cannot get last block height and timestamp from db: %s", err)
 	}
 
-	if len(heights) == 0 {
-		return 0, nil
+	if len(blockHeightAndTimestamp) == 0 {
+		return dbtypes.BlockHeightAndTimestamp{}, nil
 	}
 
-	return heights[0], nil
+	return blockHeightAndTimestamp[0], nil
 }
 
 // -------------------------------------------------------------------------------------------------------------------

database/gov.go

@@ -112,15 +112,25 @@ INSERT INTO proposal(
 		)
 	}
 
-	// Store the accounts
+	// Store the proposers accounts
 	err := db.SaveAccounts(accounts)
 	if err != nil {
 		return fmt.Errorf("error while storing proposers accounts: %s", err)
 	}
 
 	// Store the proposals
 	proposalsQuery = proposalsQuery[:len(proposalsQuery)-1] // Remove trailing ","
-	proposalsQuery += " ON CONFLICT DO NOTHING"
+	proposalsQuery += `
+ON CONFLICT (id) DO UPDATE 
+	SET title = excluded.title,
+  		description = excluded.description,
+		content = excluded.content,
+		proposer_address = excluded.proposer_address,
+		status = excluded.status,
+		submit_time = excluded.submit_time,
+		deposit_end_time = excluded.deposit_end_time,
+		voting_start_time = excluded.voting_start_time,
+		voting_end_time = excluded.voting_end_time`
 	_, err = db.SQL.Exec(proposalsQuery, proposalsParams...)
 	if err != nil {
 		return fmt.Errorf("error while storing proposals: %s", err)
@@ -134,7 +144,7 @@ func (db *Db) GetProposal(id uint64) (types.Proposal, error) {
 	var rows []*dbtypes.ProposalRow
 	err := db.SQL.Select(&rows, `SELECT * FROM proposal WHERE id = $1`, id)
 	if err != nil {
-		return types.Proposal{}, err
+		return types.Proposal{}, fmt.Errorf("error while getting proposal %d: %s", id, err)
 	}
 
 	if len(rows) == 0 {
@@ -203,7 +213,7 @@ func (db *Db) UpdateProposal(update types.ProposalUpdate) error {
 		update.ProposalID,
 	)
 	if err != nil {
-		return fmt.Errorf("error while updating proposal: %s", err)
+		return fmt.Errorf("error while updating proposal %d: %s", update.ProposalID, err)
 	}
 
 	return nil
@@ -215,25 +225,29 @@ func (db *Db) SaveDeposits(deposits []types.Deposit) error {
 		return nil
 	}
 
-	query := `INSERT INTO proposal_deposit (proposal_id, depositor_address, amount, timestamp, height) VALUES `
+	query := `INSERT INTO proposal_deposit (proposal_id, depositor_address, 
+		amount, timestamp, transaction_hash, height) VALUES `
 	var param []interface{}
 	var accounts []types.Account
 	for i, deposit := range deposits {
-		vi := i * 5
-		query += fmt.Sprintf("($%d,$%d,$%d,$%d,$%d),", vi+1, vi+2, vi+3, vi+4, vi+5)
+		vi := i * 6
+
+		accounts = append(accounts, types.NewAccount(deposit.Depositor))
+
+		query += fmt.Sprintf("($%d,$%d,$%d,$%d,$%d,$%d),", vi+1, vi+2, vi+3, vi+4, vi+5, vi+6)
 		param = append(param, deposit.ProposalID,
 			deposit.Depositor,
 			pq.Array(dbtypes.NewDbCoins(deposit.Amount)),
 			deposit.Timestamp,
+			deposit.TransactionHash,
 			deposit.Height,
 		)
-		accounts = append(accounts, types.NewAccount(deposit.Depositor))
 	}
 
-	// Store the depositor account
+	// Store depositors accounts
 	err := db.SaveAccounts(accounts)
 	if err != nil {
-		return fmt.Errorf("error while storing depositor account: %s", err)
+		return fmt.Errorf("error while storing depositors accounts: %s", err)
 	}
 
 	query = query[:len(query)-1] // Remove trailing ","
@@ -256,10 +270,10 @@ WHERE proposal_deposit.height <= excluded.height`
 // SaveVote allows to save for the given height and the message vote
 func (db *Db) SaveVote(vote types.Vote) error {
 	query := `
-INSERT INTO proposal_vote (proposal_id, voter_address, option, timestamp, height)
-VALUES ($1, $2, $3, $4, $5)
+INSERT INTO proposal_vote (proposal_id, voter_address, option, weight, timestamp, height)
+VALUES ($1, $2, $3, $4, $5, $6)
 ON CONFLICT ON CONSTRAINT unique_vote DO UPDATE
-	SET option = excluded.option,
+	SET weight = excluded.weight, 
 		timestamp = excluded.timestamp,
 		height = excluded.height
 WHERE proposal_vote.height <= excluded.height`
@@ -270,9 +284,9 @@ WHERE proposal_vote.height <= excluded.height`
 		return fmt.Errorf("error while storing voter account: %s", err)
 	}
 
-	_, err = db.SQL.Exec(query, vote.ProposalID, vote.Voter, vote.Option.String(), vote.Timestamp, vote.Height)
+	_, err = db.SQL.Exec(query, vote.ProposalID, vote.Voter, vote.Option.String(), vote.Weight, vote.Timestamp, vote.Height)
 	if err != nil {
-		return fmt.Errorf("error while storing vote: %s", err)
+		return fmt.Errorf("error while storing vote for proposal %d: %s", vote.ProposalID, err)
 	}
 
 	return nil
@@ -323,17 +337,19 @@ func (db *Db) SaveProposalStakingPoolSnapshot(snapshot types.ProposalStakingPool
 	stmt := `
 INSERT INTO proposal_staking_pool_snapshot (proposal_id, bonded_tokens, not_bonded_tokens, height)
 VALUES ($1, $2, $3, $4)
-ON CONFLICT ON CONSTRAINT unique_staking_pool_snapshot DO UPDATE SET
-	proposal_id = excluded.proposal_id,
-    bonded_tokens = excluded.bonded_tokens,
-	not_bonded_tokens = excluded.not_bonded_tokens,
+ON CONFLICT ON CONSTRAINT unique_staking_pool_snapshot DO UPDATE 
+	SET bonded_tokens = excluded.bonded_tokens,
+	not_bonded_tokens = excluded.not_bonded_tokens, 
 	height = excluded.height
 WHERE proposal_staking_pool_snapshot.height <= excluded.height`
 
 	_, err := db.SQL.Exec(stmt,
-		snapshot.ProposalID, snapshot.Pool.BondedTokens.String(), snapshot.Pool.NotBondedTokens.String(), snapshot.Pool.Height)
+		snapshot.ProposalID, snapshot.Pool.BondedTokens.String(),
+		snapshot.Pool.NotBondedTokens.String(), snapshot.Pool.Height)
+
 	if err != nil {
-		return fmt.Errorf("error while storing proposal staking pool snapshot: %s", err)
+		return fmt.Errorf("error while storing proposal staking pool snapshot for proposal %d: %s",
+			snapshot.ProposalID, err)
 	}
 
 	return nil
@@ -346,7 +362,7 @@ func (db *Db) SaveProposalValidatorsStatusesSnapshots(snapshots []types.Proposal
 	}
 
 	stmt := `
-INSERT INTO proposal_validator_status_snapshot(proposal_id, validator_address, voting_power, status, jailed, height)
+INSERT INTO proposal_validator_status_snapshot (proposal_id, validator_address, voting_power, status, jailed, height) 
 VALUES `
 
 	var args []interface{}
@@ -361,17 +377,15 @@ VALUES `
 
 	stmt = stmt[:len(stmt)-1]
 	stmt += `
-ON CONFLICT ON CONSTRAINT unique_validator_status_snapshot DO UPDATE
-	SET proposal_id = excluded.proposal_id,
-		validator_address = excluded.validator_address,
-		voting_power = excluded.voting_power,
-		status = excluded.status,
+ON CONFLICT ON CONSTRAINT unique_validator_status_snapshot DO UPDATE 
+	SET voting_power = excluded.voting_power, 
+		status = excluded.status, 
 		jailed = excluded.jailed,
 		height = excluded.height
 WHERE proposal_validator_status_snapshot.height <= excluded.height`
 	_, err := db.SQL.Exec(stmt, args...)
 	if err != nil {
-		return fmt.Errorf("error while storing proposal validator statuses snapshot: %s", err)
+		return fmt.Errorf("error while storing proposal validator statuses snapshots: %s", err)
 	}
 
 	return nil
@@ -393,7 +407,7 @@ WHERE software_upgrade_plan.height <= excluded.height`
 	_, err := db.SQL.Exec(stmt,
 		proposalID, plan.Name, plan.Height, plan.Info, height)
 	if err != nil {
-		return fmt.Errorf("error while storing software upgrade plan: %s", err)
+		return fmt.Errorf("error while storing software upgrade plan for proposal %d: %s", proposalID, err)
 	}
 
 	return nil
@@ -405,7 +419,7 @@ func (db *Db) DeleteSoftwareUpgradePlan(proposalID uint64) error {
 
 	_, err := db.SQL.Exec(stmt, proposalID)
 	if err != nil {
-		return fmt.Errorf("error while deleting software upgrade plan: %s", err)
+		return fmt.Errorf("error while deleting software upgrade plan for proposal %d: %s", proposalID, err)
 	}
 
 	return nil