diff --git a/politeiad/cmd/legacyimport/git.go b/politeiad/cmd/legacyimport/git.go new file mode 100644 index 0000000000..06ab7d0f9a --- /dev/null +++ b/politeiad/cmd/legacyimport/git.go @@ -0,0 +1 @@ +package main diff --git a/politeiad/cmd/legacyimport/legacyimport.go b/politeiad/cmd/legacyimport/legacyimport.go index f4dae76d83..d08866b7ed 100644 --- a/politeiad/cmd/legacyimport/legacyimport.go +++ b/politeiad/cmd/legacyimport/legacyimport.go @@ -28,6 +28,7 @@ import ( "github.com/decred/politeia/politeiad/backendv2/tstorebe/tstore" "github.com/decred/politeia/politeiad/plugins/pi" "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/politeiad/plugins/usermd" pusermd "github.com/decred/politeia/politeiad/plugins/usermd" "github.com/decred/politeia/util" ) @@ -105,6 +106,8 @@ func newLegacyImport() (*legacyImport, error) { }, nil } +// preParsePaths builds an optimized traversal path for the git record +// repository. func preParsePaths(path string) (map[string]string, error) { // Pre-parse git records folder and get the path for each record's // latest version. @@ -156,17 +159,18 @@ func preParsePaths(path string) (map[string]string, error) { // data for tlog. func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { var ( - files []backend.File - metadata []backend.MetadataStream - proposalMd pi.ProposalMetadata - voteMd ticketvote.VoteMetadata - recordMd *backend.RecordMetadata - startVoteMd *ticketvote.Start - authDetailsMd *ticketvote.AuthDetails - voteDetailsMd *ticketvote.VoteDetails - commentsPath string - ballotPath string - parentToken string + files []backend.File + metadata []backend.MetadataStream + proposalMd pi.ProposalMetadata + voteMd ticketvote.VoteMetadata + recordMd *backend.RecordMetadata + statusChangeMd *usermd.StatusChangeMetadata + startVoteMd *ticketvote.Start + authDetailsMd *ticketvote.AuthDetails + voteDetailsMd *ticketvote.VoteDetails + commentsPath string + ballotPath string + parentToken string ) err := filepath.Walk(rootpath, func(path string, info os.FileInfo, err error) error { @@ -176,7 +180,7 @@ func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { // Build user metadata and get proposal name. if info.Name() == "00.metadata.txt" { - usermd, name, err := l.convertProposalGeneral(path) + usermd, name, err := l.convertUserMetadata(path) if err != nil { return err } @@ -194,19 +198,10 @@ func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { // Build status change metadata. if info.Name() == "02.metadata.txt" { - statuschangemd, err := convertStatusChangeMetadata(path) + statusChangeMd, err = convertStatusChangeMetadata(path) if err != nil { return err } - b, err := json.Marshal(statuschangemd) - if err != nil { - return err - } - metadata = append(metadata, backend.MetadataStream{ - PluginID: pusermd.PluginID, - StreamID: pusermd.StreamIDStatusChanges, - Payload: string(b), - }) } // Build authorize vote metadata. @@ -353,20 +348,45 @@ func (l *legacyImport) parseRecordData(rootpath string) (*parsedData, error) { // record path, and returns all data needed by tlog on the parsedData // struct. return &parsedData{ - files: files, - metadata: metadata, - recordMd: recordMd, - authDetailsMd: authDetailsMd, - voteDetailsMd: voteDetailsMd, - commentsPath: commentsPath, - ballotPath: ballotPath, - legacyToken: proposalMd.LegacyToken, + files: files, + metadata: metadata, + recordMd: recordMd, + statusChangeMd: statusChangeMd, + authDetailsMd: authDetailsMd, + voteDetailsMd: voteDetailsMd, + commentsPath: commentsPath, + ballotPath: ballotPath, + legacyToken: proposalMd.LegacyToken, + parentToken: parentToken, }, nil } // saveRecordData saves the parsed data onto tstore. func (l *legacyImport) saveRecordData(data parsedData) ([]byte, error) { - newToken, err := l.recordSave(data.files, data.metadata, *data.recordMd) + // Create a new tlog tree for the legacy record. + newToken, err := l.tstore.RecordNew() + if err != nil { + return nil, err + } + + // Save new token to record metadata. + data.recordMd.Token = hex.EncodeToString(newToken) + // Save new token to status change metadata. + data.statusChangeMd.Token = hex.EncodeToString(newToken) + + // Add status change metadata to metadata stream. + b, err := json.Marshal(data.statusChangeMd) + if err != nil { + return nil, err + } + data.metadata = append(data.metadata, backend.MetadataStream{ + PluginID: pusermd.PluginID, + StreamID: pusermd.StreamIDStatusChanges, + Payload: string(b), + }) + + // Save record to tstore. + err = l.recordSave(newToken, data.files, data.metadata, *data.recordMd) if err != nil { return nil, err } @@ -381,6 +401,9 @@ func (l *legacyImport) saveRecordData(data parsedData) ([]byte, error) { // Save vote details blob, if any. if data.voteDetailsMd != nil { + if data.parentToken != "" { + data.voteDetailsMd.Params.Parent = data.parentToken + } err = l.blobSaveVoteDetails(*data.voteDetailsMd, newToken) if err != nil { return nil, err @@ -467,7 +490,7 @@ func _main() error { // Save legacy record on tstore. newToken, err := l.saveRecordData(data) if err != nil { - return err + panic(err) } fmt.Printf("legacyimport: Parsed record %v. new tlog token: %v\n", @@ -476,7 +499,6 @@ func _main() error { return nil }(*pData) } - wg.Wait() fmt.Println("legacyimport: Done parsing first batch!") diff --git a/politeiad/cmd/legacyimport/record.go b/politeiad/cmd/legacyimport/record.go index 5c0240e481..419c2a25f0 100644 --- a/politeiad/cmd/legacyimport/record.go +++ b/politeiad/cmd/legacyimport/record.go @@ -3,7 +3,6 @@ package main import ( "bytes" "encoding/base64" - "encoding/hex" "encoding/json" "fmt" "io/ioutil" @@ -21,24 +20,15 @@ import ( "github.com/subosito/gozaru" ) -func (l *legacyImport) recordSave(files []backend.File, metadata []backend.MetadataStream, recordmd backend.RecordMetadata) ([]byte, error) { - // Create a new tlog tree for the legacy record. - newToken, err := l.tstore.RecordNew() - if err != nil { - return nil, err - } - - // Insert new token on record metadata. - recordmd.Token = hex.EncodeToString(newToken) - +func (l *legacyImport) recordSave(newToken []byte, files []backend.File, metadata []backend.MetadataStream, recordmd backend.RecordMetadata) error { // Verify data - err = metadataStreamsVerify(metadata) + err := metadataStreamsVerify(metadata) if err != nil { - return nil, err + return err } err = filesVerify(files, nil) if err != nil { - return nil, err + return err } // Check if record status is public. If so, we need to first save it as @@ -53,7 +43,7 @@ func (l *legacyImport) recordSave(files []backend.File, metadata []backend.Metad // Save record. err = l.tstore.RecordSave(newToken, recordmd, metadata, files) if err != nil { - return nil, err + return err } // If it's public, we need another save to ensure the public status @@ -63,11 +53,11 @@ func (l *legacyImport) recordSave(files []backend.File, metadata []backend.Metad recordmd.Iteration = 2 err = l.tstore.RecordSave(newToken, recordmd, metadata, files) if err != nil { - return nil, err + return err } } - return newToken, nil + return nil } func (l *legacyImport) fetchUserByPubKey(pubkey string) (*user, error) { @@ -146,8 +136,10 @@ func convertStatusChangeMetadata(path string) (*usermd.StatusChangeMetadata, err if err != nil { return nil, err } - var rsc mdstream.RecordStatusChangeV2 - var streams []mdstream.RecordStatusChangeV2 + var ( + rsc mdstream.RecordStatusChangeV2 + streams []mdstream.RecordStatusChangeV2 + ) err = json.Unmarshal(b, &rsc) if err != nil { // Record has 02.metadata.txt containing nested status changes. @@ -156,7 +148,6 @@ func convertStatusChangeMetadata(path string) (*usermd.StatusChangeMetadata, err if v == "" { continue } - var rsc mdstream.RecordStatusChangeV2 err = json.Unmarshal([]byte(v+"}"), &rsc) if err != nil { return nil, err @@ -170,6 +161,7 @@ func convertStatusChangeMetadata(path string) (*usermd.StatusChangeMetadata, err // Return most recent status change md. latest := streams[len(streams)-1] return &pusermd.StatusChangeMetadata{ + // Token: Version: uint32(latest.Version), Status: uint32(latest.NewStatus), Reason: latest.StatusChangeMessage, @@ -179,9 +171,9 @@ func convertStatusChangeMetadata(path string) (*usermd.StatusChangeMetadata, err }, nil } -// convertProposalGeneral converts the ProposalGeneralV1 metadata structure -// previously used on legacy git records. -func (l *legacyImport) convertProposalGeneral(path string) (*usermd.UserMetadata, string, error) { +// convertUserMetadata converts the 00.metadata.txt file which contains the +// ProposalGeneralV1 metadata structure previously used on legacy git records. +func (l *legacyImport) convertUserMetadata(path string) (*usermd.UserMetadata, string, error) { b, err := ioutil.ReadFile(path) if err != nil { return nil, "", err diff --git a/politeiad/cmd/legacyimport/ticketvote.go b/politeiad/cmd/legacyimport/ticketvote.go index 1018c45a9b..bbd3fb27e2 100644 --- a/politeiad/cmd/legacyimport/ticketvote.go +++ b/politeiad/cmd/legacyimport/ticketvote.go @@ -7,9 +7,11 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/http" "os" "strconv" + dcrdata "github.com/decred/dcrdata/v6/api/types" "github.com/decred/politeia/politeiad/backend/gitbe" "github.com/decred/politeia/politeiad/backendv2/tstorebe/store" tv "github.com/decred/politeia/politeiad/plugins/ticketvote" @@ -29,11 +31,13 @@ func convertAuthDetailsMetadata(path string) (*tv.AuthDetails, error) { } return &tv.AuthDetails{ - Token: av.Token, // This token will be changed with the new tlog one + Token: av.Token, Version: uint32(av.Version), Action: av.Action, PublicKey: av.PublicKey, Signature: av.Signature, + Timestamp: av.Timestamp, + Receipt: av.Receipt, }, nil } @@ -72,7 +76,7 @@ func convertStartVoteMetadata(path string) (*tv.Start, error) { QuorumPercentage: av.Vote.QuorumPercentage, PassPercentage: av.Vote.PassPercentage, Options: opts, - Parent: "", // Will be set later on + Parent: "", // This is set at the end of parsing. }, PublicKey: av.PublicKey, Signature: av.Signature, @@ -83,6 +87,7 @@ func convertStartVoteMetadata(path string) (*tv.Start, error) { // convertVoteDetailsMetadata converts the 15.metadata.txt file to the vote // details structure from tlog backend. +// TODO: review. see how its done on ticketvote/cmds.go. how vote details is saved for RFP subs func convertVoteDetailsMetadata(path string, startDetails []tv.StartDetails) (*tv.VoteDetails, error) { b, err := ioutil.ReadFile(path) if err != nil { @@ -125,14 +130,17 @@ func (l *legacyImport) convertBallotJournal(path string, newToken []byte) error return err } - s := bufio.NewScanner(fh) - - fmt.Printf(" ticketvote: Parsing ballot journal for %v ...\n", hex.EncodeToString(newToken)) + fmt.Printf(" ticketvote: Pre parsing ballot journal for %v ...\n", + hex.EncodeToString(newToken)) + var ( + tickets []string // Used to fetch largest commitment address + castVoteDetails []*tv.CastVoteDetails + ) + s := bufio.NewScanner(fh) for i := 0; s.Scan(); i++ { ss := bytes.NewReader([]byte(s.Text())) d := json.NewDecoder(ss) - var action gitbe.JournalAction err := d.Decode(&action) if err != nil { @@ -147,23 +155,56 @@ func (l *legacyImport) convertBallotJournal(path string, newToken []byte) error return fmt.Errorf("ballot journal add: %v", err) } - cv := tv.CastVoteDetails{ + tickets = append(tickets, cvj.CastVote.Ticket) + castVoteDetails = append(castVoteDetails, &tv.CastVoteDetails{ Token: hex.EncodeToString(newToken), Ticket: cvj.CastVote.Ticket, VoteBit: cvj.CastVote.VoteBit, Signature: cvj.CastVote.Signature, Receipt: cvj.Receipt, - } + // Add commitment address + // Add timestamp + }) - err = l.blobSaveCastVoteDetails(cv, newToken) - if err != nil { - return err - } default: return fmt.Errorf("invalid ballot journal action") } } + fmt.Println(" ticketvote: Fetching largest commitment addresses from dcrdata...") + + lcr, err := largestCommitmentAddresses(tickets) + if err != nil { + panic(err) + } + + fmt.Println(" ticketvote: Saving ticketvote blobs to tstore...") + + for k := range castVoteDetails { + // Save cast vote details blob to tstore. + cv := castVoteDetails[k] // vote details + cv.Address = lcr[k].bestAddr // largest commitment address + err = l.blobSaveCastVoteDetails(*cv, newToken) + if err != nil { + return err + } + + fmt.Printf(" ticketvote: Saved cast vote details blob: %v\n", cv) + + // Save vote collider blob to tstore. + vc := voteCollider{ + Token: hex.EncodeToString(newToken), + Ticket: cv.Ticket, + } + err = l.blobSaveVoteCollider(vc, newToken) + if err != nil { + return err + } + + fmt.Printf(" ticketvote: Saved vote collider blob: %v\n", vc) + + } + fmt.Printf(" ticketvote: Done for %v!\n", hex.EncodeToString(newToken)) return nil @@ -211,8 +252,8 @@ func (l *legacyImport) blobSaveAuthDetails(authDetails tv.AuthDetails, newToken if err != nil { return err } - be := store.NewBlobEntry(hint, data) + be := store.NewBlobEntry(hint, data) err = l.tstore.BlobSave(newToken, be) if err != nil { return err @@ -237,8 +278,31 @@ func (l *legacyImport) blobSaveVoteDetails(voteDetails tv.VoteDetails, newToken if err != nil { return err } + be := store.NewBlobEntry(hint, data) + err = l.tstore.BlobSave(newToken, be) + if err != nil { + return err + } + + return nil +} + +func (l *legacyImport) blobSaveVoteCollider(vc voteCollider, newToken []byte) error { + data, err := json.Marshal(vc) + if err != nil { + return err + } + hint, err := json.Marshal( + store.DataDescriptor{ + Type: store.DataTypeStructure, + Descriptor: tv.PluginID + "-vcollider-v1", + }) + if err != nil { + return err + } + be := store.NewBlobEntry(hint, data) err = l.tstore.BlobSave(newToken, be) if err != nil { return err @@ -246,3 +310,87 @@ func (l *legacyImport) blobSaveVoteDetails(voteDetails tv.VoteDetails, newToken return nil } + +// Get largest commitment address from dcrdata +func batchTransactions(hashes []string) ([]dcrdata.TrimmedTx, error) { + // Request body is dcrdataapi.Txns marshalled to JSON + reqBody, err := json.Marshal(dcrdata.Txns{ + Transactions: hashes, + }) + if err != nil { + return nil, err + } + + // Make the POST request + url := "https://dcrdata.decred.org/api/txs/trimmed" + r, err := http.Post(url, "application/json; charset=utf-8", + bytes.NewReader(reqBody)) + if err != nil { + return nil, err + } + defer r.Body.Close() + + if r.StatusCode != http.StatusOK { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, fmt.Errorf("dcrdata error: %v %v %v", + r.StatusCode, url, err) + } + return nil, fmt.Errorf("dcrdata error: %v %v %s", + r.StatusCode, url, body) + } + + // Unmarshal the response + var ttx []dcrdata.TrimmedTx + decoder := json.NewDecoder(r.Body) + if err := decoder.Decode(&ttx); err != nil { + return nil, err + } + return ttx, nil +} + +// largestCommitmentResult returns the largest commitment address or an error. +type largestCommitmentResult struct { + bestAddr string + err error +} + +func largestCommitmentAddresses(hashes []string) ([]largestCommitmentResult, error) { + // Batch request all of the transaction info from dcrdata. + ttxs, err := batchTransactions(hashes) + if err != nil { + return nil, err + } + + // Find largest commitment address for each transaction. + r := make([]largestCommitmentResult, len(hashes)) + for i := range ttxs { + // Best is address with largest commit amount. + var bestAddr string + var bestAmount float64 + for _, v := range ttxs[i].Vout { + if v.ScriptPubKeyDecoded.CommitAmt == nil { + continue + } + if *v.ScriptPubKeyDecoded.CommitAmt > bestAmount { + if len(v.ScriptPubKeyDecoded.Addresses) == 0 { + // jrick, does this need to be printed? + fmt.Errorf("unexpected addresses "+ + "length: %v", ttxs[i].TxID) + continue + } + bestAddr = v.ScriptPubKeyDecoded.Addresses[0] + bestAmount = *v.ScriptPubKeyDecoded.CommitAmt + } + } + + if bestAddr == "" || bestAmount == 0.0 { + r[i].err = fmt.Errorf("no best commitment address found: %v", + ttxs[i].TxID) + continue + } + r[i].bestAddr = bestAddr + } + + return r, nil +} diff --git a/politeiad/cmd/legacyimport/types.go b/politeiad/cmd/legacyimport/types.go index 5908026d2f..a21e4f7f8e 100644 --- a/politeiad/cmd/legacyimport/types.go +++ b/politeiad/cmd/legacyimport/types.go @@ -4,30 +4,37 @@ import ( "github.com/decred/dcrd/chaincfg/v3" backend "github.com/decred/politeia/politeiad/backendv2" "github.com/decred/politeia/politeiad/plugins/ticketvote" + "github.com/decred/politeia/politeiad/plugins/usermd" ) // parsedData holds the data needed by tlog to insert the legacy // records on tstore. type parsedData struct { - files []backend.File - metadata []backend.MetadataStream - recordMd *backend.RecordMetadata - authDetailsMd *ticketvote.AuthDetails - voteDetailsMd *ticketvote.VoteDetails - commentsPath string - ballotPath string - legacyToken string + files []backend.File + metadata []backend.MetadataStream + recordMd *backend.RecordMetadata + statusChangeMd *usermd.StatusChangeMetadata + authDetailsMd *ticketvote.AuthDetails + voteDetailsMd *ticketvote.VoteDetails + commentsPath string + ballotPath string + legacyToken string + parentToken string +} + +type voteCollider struct { + Token string `json:"token"` + Ticket string `json:"ticket"` } // likeCommentV1 unmarshals the like action data from the gitbe's comments // journal. type likeCommentV1 struct { - Token string `json:"token"` // Censorship token - CommentID string `json:"commentid"` // Comment ID - Action string `json:"action"` // Up or downvote (1, -1) - Signature string `json:"signature"` // Client Signature of Token+CommentID+Action - PublicKey string `json:"publickey"` // Pubkey used for Signature - + Token string `json:"token"` // Censorship token + CommentID string `json:"commentid"` // Comment ID + Action string `json:"action"` // Up or downvote (1, -1) + Signature string `json:"signature"` // Client Signature of Token+CommentID+Action + PublicKey string `json:"publickey"` // Pubkey used for Signature Receipt string `json:"receipt,omitempty"` // Signature of Signature Timestamp int64 `json:"timestamp,omitempty"` // Received UNIX timestamp } @@ -66,17 +73,17 @@ type authorizeVoteV1 struct { } type voteV1 struct { - Token string `json:"token"` // Token that identifies vote - ProposalVersion uint32 `json:"proposalversion"` // Proposal version being voted on - Type int `json:"type"` // Type of vote - Mask uint64 `json:"mask"` // Valid votebits - Duration uint32 `json:"duration"` // Duration in blocks - QuorumPercentage uint32 `json:"quorumpercentage"` // Percent of eligible votes required for quorum - PassPercentage uint32 `json:"passpercentage"` // Percent of total votes required to pass - Options []voteOption `json:"options"` // Vote option + Token string `json:"token"` // Token that identifies vote + ProposalVersion uint32 `json:"proposalversion"` // Proposal version being voted on + Type int `json:"type"` // Type of vote + Mask uint64 `json:"mask"` // Valid votebits + Duration uint32 `json:"duration"` // Duration in blocks + QuorumPercentage uint32 `json:"quorumpercentage"` // Percent of eligible votes required for quorum + PassPercentage uint32 `json:"passpercentage"` // Percent of total votes required to pass + Options []voteOptionV1 `json:"options"` // Vote option } -type voteOption struct { +type voteOptionV1 struct { Id string `json:"id"` // Single unique word identifying vote (e.g. yes) Description string `json:"description"` // Longer description of the vote Bits uint64 `json:"bits"` // Bits used for this option