Skip to content

Commit

Permalink
add tools
Browse files Browse the repository at this point in the history
  • Loading branch information
Ubuntu committed Mar 16, 2024
1 parent 1a59c94 commit 69e0eaa
Show file tree
Hide file tree
Showing 15 changed files with 1,252 additions and 0 deletions.
41 changes: 41 additions & 0 deletions tools/srs-utils/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# srs-verification

This project is written for EigenDA SRS related works. It includes
1. Extract G1 and G2 points used by EigenDA from ptau challenge file, created from the perpetual power of tau MPC instance run by EF
2. Verify the extracted points are indeed correct based on approaches used by EF KZG ceremony
3. Provide utility tools for miscellaneous tasks

## To download the ptau file

`wget https://pse-trusted-setup-ppot.s3.eu-central-1.amazonaws.com/challenge_0085`

See more informations from
1. https://docs.axiom.xyz/docs/transparency-and-security/kzg-trusted-setup
2. https://github.com/privacy-scaling-explorations/perpetualpowersoftau/tree/master

The challenge file has 103079215232 Bytes.

## How to use

`go run main.go help`

### How to parse G1, G2 points from the challenge file.

`go run main.go parse --ptau-path <Path to challenge file>`

It produces two files, g1.point and g2.point. g1.point contains 8589934592 Bytes and g2.point 17179869184 Bytes

This procedure takes roughly 10 minutes.

Note: The challenge files contains 2^29 G1 points and 2^28 G2 points with secret tau. We use only the first 2^28 G1 points for EigenDA.

### How to verify the parsed G1, G2 points

`go run main.go parse --g1-path <Path to g1.point> --g2-path <Path to g2.point>`

The verification is based on method listed here (https://github.com/ethereum/kzg-ceremony-specs/blob/master/docs/sequencer/sequencer.md#pairing-checks)

This procedures takes roughly 27 Hours on a 8 thread machines.

The program periodically prints out the time spent and its progress of validating 2^28 G1 and G2 points. If no error message is shown and program terminates with "Done. Everything is correct". Then SRS is deemed as correct.

24 changes: 24 additions & 0 deletions tools/srs-utils/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
module github.com/bxue-l2/srs-verification

go 1.21.6

require (
github.com/consensys/gnark-crypto v0.12.1
github.com/stretchr/testify v1.8.4
github.com/urfave/cli v1.22.14
)

require (
github.com/bits-and-blooms/bitset v1.10.0 // indirect
github.com/consensys/bavard v0.1.13 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
golang.org/x/sys v0.16.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
)
49 changes: 49 additions & 0 deletions tools/srs-utils/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=
github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ=
github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
45 changes: 45 additions & 0 deletions tools/srs-utils/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package main

import (
"fmt"
"log"
"os"

"github.com/bxue-l2/srs-verification/parser"
"github.com/bxue-l2/srs-verification/verifier"
"github.com/urfave/cli"
)

func main() {
app := &cli.App{
Commands: []cli.Command{
{
Name: "verify",
Aliases: []string{"v"},
Usage: "verify if the parsed SRS are consistent",
Action: func(cCtx *cli.Context) error {
config := verifier.ReadCLIConfig(cCtx)
verifier.VerifySRS(config)
return nil
},
Flags: verifier.Flags,
},
{
Name: "parse",
Aliases: []string{"p"},
Usage: "parse data from ptau challenge file into EigenDA SRS format",
Flags: parser.Flags,
Action: func(cCtx *cli.Context) error {
config := parser.ReadCLIConfig(cCtx)
fmt.Printf("config %v\n", config.PtauPath)
parser.ParsePtauChallenge(config)
return nil
},
},
},
}

if err := app.Run(os.Args); err != nil {
log.Fatal(err)
}
}
67 changes: 67 additions & 0 deletions tools/srs-utils/parser/flags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package parser

import (
"runtime"

"github.com/urfave/cli"
)

var (
/* Required Flags */
PtauPathFlag = cli.StringFlag{
Name: "ptau-path",
Usage: "File path to the ptau challenge file",
Required: true,
EnvVar: "PTAU_PATH",
}

/* Optional Flags */
ParserNumBatchFlag = cli.Uint64Flag{
Name: "parser-num-batch",
Usage: "Set total number batch size for parallel parser to work on",
Required: false,
EnvVar: "PARSER_NUM_BATCH",
Value: uint64(50),
}
NumPointToParseFlag = cli.Uint64Flag{
Name: "parser-num-points",
Usage: "Set total number of points (g1 and g2) to parse",
Required: false,
EnvVar: "PARSER_NUM_POINT",
Value: uint64(268435456),
}
NumWorkerFlag = cli.IntFlag{
Name: "verifier-num-worker",
Usage: "Set total number of worker thread",
Required: false,
EnvVar: "NUM_WORKER",
Value: runtime.GOMAXPROCS(0),
}
)

var requiredFlags = []cli.Flag{
PtauPathFlag,
}

var optionalFlags = []cli.Flag{
ParserNumBatchFlag,
NumPointToParseFlag,
NumWorkerFlag,
}

func ReadCLIConfig(ctx *cli.Context) Config {
cfg := Config{}
cfg.PtauPath = ctx.String(PtauPathFlag.Name)
cfg.NumBatch = ctx.Uint64(ParserNumBatchFlag.Name)
cfg.NumPoint = ctx.Uint64(NumPointToParseFlag.Name)
cfg.NumWorker = ctx.Int(NumWorkerFlag.Name)

return cfg
}

func init() {
Flags = append(requiredFlags, optionalFlags...)
}

// Flags contains the list of configuration options available to the binary.
var Flags []cli.Flag
158 changes: 158 additions & 0 deletions tools/srs-utils/parser/g1FileIO.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
package parser

import (
"bufio"
"fmt"
"log"
"os"
"sync"
"time"

"github.com/consensys/gnark-crypto/ecc/bn254"
"github.com/consensys/gnark-crypto/ecc/bn254/fp"
)

func ParseG1PointSection(filepath string, params Params, numWorker uint64) ([]bn254.G1Affine, error) {
fmt.Printf("Start to read %v points from Byte pos at %v to at %v",
params.NumPoint,
params.G1StartByte,
params.GetG1EndBytePos(),
)

g1f, err := os.Open(filepath)
if err != nil {
log.Println("ReadG1PointSection.ERR.0", err)
return nil, err
}

defer func() {
if err := g1f.Close(); err != nil {
panic(err)
}
}()

n := params.NumPoint
startTimer := time.Now()
g1r := bufio.NewReaderSize(g1f, int(params.NumPoint*params.G1Size))

_, err = g1f.Seek(int64(params.G1StartByte), 0)
if err != nil {
return nil, err
}

if n < numWorker {
numWorker = n
}

numToRead := params.NumPoint * params.G1Size
buf := make([]byte, numToRead)
numBytes, err := g1r.Read(buf)

if err != nil {
return nil, err
}

if uint64(numBytes) != numToRead {
log.Printf("Error. Insufficient G1 points. Only contains %v. Requesting %v, NumByte %v\n", len(buf)/64, params.NumPoint, numBytes)
log.Println("numBytes", numBytes, "numToRead", numToRead)
log.Println("ReadG1PointSection.ERR.1", err)
return nil, err
}

// measure reading time
t := time.Now()
elapsed := t.Sub(startTimer)
log.Printf(" Reading G1 points (%v bytes) takes %v\n", (n * 64), elapsed)
startTimer = time.Now()

s1Outs := make([]bn254.G1Affine, n)

var wg sync.WaitGroup
wg.Add(int(numWorker))

start := uint64(0)
end := uint64(0)
size := n / numWorker

for i := uint64(0); i < numWorker; i++ {
start = i * size

if i == numWorker-1 {
end = n
} else {
end = (i + 1) * size
}
//todo: handle error?
go readG1Worker(buf, s1Outs, start, end, 64, &wg)
}
wg.Wait()

t = time.Now()
elapsed = t.Sub(startTimer)
fmt.Println("Finish Parsing takes", elapsed)
return s1Outs, nil
}

func WriteG1PointsForEigenDA(points []bn254.G1Affine, from uint64, to uint64) error {
n := to - from
g1f, err := os.OpenFile("g1.point", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Printf("Canot write G1 from %v to %v . Error %v\n", from, to, err)
return err
}

g1w := bufio.NewWriter(g1f)

for i := uint64(0); i < n; i++ {
pointInBytes := points[i].Bytes()
numWritten, err := g1w.Write(pointInBytes[:])
if numWritten != 32 || err != nil {
fmt.Printf("Cannot write point %v . Error %v\n", from+i, err)
return err
}
}

if err = g1w.Flush(); err != nil {
log.Println("Cannot flush points", err)
return err
}
g1f.Close()
return nil

}

func readG1Worker(
buf []byte,
outs []bn254.G1Affine,
start uint64, // in element, not in byte
end uint64,
step uint64,
wg *sync.WaitGroup,
) {
for i := start; i < end; i++ {
fieldSize := step / uint64(2)
g1x := buf[i*step : (i)*step+fieldSize]
g1y := buf[i*step+fieldSize : (i+1)*step]

point := parseG1Point(g1x, g1y)
outs[i] = *point
}
wg.Done()
}

func parseG1Point(xBytes, yBytes []byte) *bn254.G1Affine {
var x fp.Element
var y fp.Element

x.SetBytes(xBytes[:])
y.SetBytes(yBytes[:])

g1Aff := bn254.G1Affine{}
g1Aff.X = x
g1Aff.Y = y

if !g1Aff.IsOnCurve() {
panic("g1Affine is not on curve")
}
return &g1Aff
}
Loading

0 comments on commit 69e0eaa

Please sign in to comment.