From bd5d16be08844ee28b548a7c8d482b3d0c96ad91 Mon Sep 17 00:00:00 2001 From: Olaf Flebbe Date: Sat, 23 Sep 2023 18:59:49 +0200 Subject: [PATCH 1/5] replace czlib with go-libdeflate --- README.md | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- osmpbf/README.md | 48 +++++++++++++++++++++++++++------------------- osmpbf/decode.go | 23 +--------------------- osmpbf/zlib_cgo.go | 21 +++++++++++++++----- osmpbf/zlib_go.go | 27 +++++++++++++++++++++++--- 7 files changed, 75 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index b8e29f6..541a525 100644 --- a/README.md +++ b/README.md @@ -142,9 +142,9 @@ BenchmarkChange_UnmarshalJSON-12 287707 317723 +10.43% ## CGO and zlib OSM PBF data comes in blocks, each block is zlib compressed. Decompressing this -data takes about 33% of the total read time. [DataDog/czlib](https://github.com/DataDog/czlib) is -used to speed this process. -See [osmpbf/README.md](osmpbf#using-cgoczlib-for-decompression) for more details. +data takes about 33% of the total read time. [4kills/go-libdeflate](https://github.com/4kills/go-libdeflate) is +used to speed up decompressing. +See [osmpbf/README.md](osmpbf#using-libdeflate-for-decompression) for more details. As a result, a C compiler is necessary to install this module. On macOS this may require installing pkg-config using something like `brew install pkg-config` diff --git a/go.mod b/go.mod index 5ea8c27..d6d940b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/paulmach/osm go 1.13 require ( - github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 + github.com/4kills/go-libdeflate/v2 v2.0.3 github.com/paulmach/orb v0.1.3 github.com/paulmach/protoscan v0.2.1 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 diff --git a/go.sum b/go.sum index f527be7..f1983fb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:ISaMhBq2dagaoptFGUyywT5SzpysCbHofX3sCNw1djo= -github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:2yDaWzisHKoQoxm+EU4YgKBaD7g1M0pxy7THWG44Lro= +github.com/4kills/go-libdeflate/v2 v2.0.3 h1:Y13oRUvtAXFJkcW4F0MnaQQB753a71sTutGrVbEAubQ= +github.com/4kills/go-libdeflate/v2 v2.0.3/go.mod h1:hyouZv4OAhHaaMpYuejstUN0xOg8mA+yy75WE3Ty6SM= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= diff --git a/osmpbf/README.md b/osmpbf/README.md index cdcdf61..fe171b1 100644 --- a/osmpbf/README.md +++ b/osmpbf/README.md @@ -80,31 +80,39 @@ This package supports reading OSM PBF files where the ways have been annotated w Coordinates are stored in the `Lat` and `Lon` fields of each `WayNode`. There is no need to specify an explicit option; when the node locations are present on the ways, they are loaded automatically. For more info about the OSM PBF format extension, see [the original blog post](https://blog.jochentopf.com/2016-04-20-node-locations-on-ways.html). -## Using cgo/czlib for decompression +## Using cgo libdeflate for decompression OSM PBF files are a set of blocks that are zlib compressed. When using the pure golang implementation this can account for about 1/3 of the read time. When cgo is enabled -the package will used [czlib](https://github.com/DataDog/czlib). +the package [go-libdeflate](https://github.com/4kills/libdeflate) will used. + +Previous versions used the lib czlib based on zlib. libdeflate is more performant +and more memory efficient for uncompressing. ``` $ CGO_ENABLED=0 go test -bench . > disabled.txt $ CGO_ENABLED=1 go test -bench . > enabled.txt $ benchcmp disabled.txt enabled.txt -benchmark old ns/op new ns/op delta -BenchmarkLondon-12 312294630 229927205 -26.37% -BenchmarkLondon_nodes-12 246562457 160021768 -35.10% -BenchmarkLondon_ways-12 216803544 134747327 -37.85% -BenchmarkLondon_relations-12 158722633 80560144 -49.24% - -benchmark old allocs new allocs delta -BenchmarkLondon-12 2469128 2416804 -2.12% -BenchmarkLondon_nodes-12 1056166 1003850 -4.95% -BenchmarkLondon_ways-12 1845032 1792716 -2.84% -BenchmarkLondon_relations-12 509090 456772 -10.28% - -benchmark old bytes new bytes delta -BenchmarkLondon-12 963734544 954877896 -0.92% -BenchmarkLondon_nodes-12 658337435 649482060 -1.35% -BenchmarkLondon_ways-12 441674734 432819378 -2.00% -BenchmarkLondon_relations-12 187941609 179086389 -4.71% -``` +benchmark old ns/op new ns/op delta +BenchmarkLondon-8 361519289 275254714 -23.86% +BenchmarkLondon_withFiltersTrue-8 392469042 263935960 -32.75% +BenchmarkLondon_withFiltersFalse-8 310824940 200477972 -35.50% +BenchmarkLondon_nodes-8 295277528 180614979 -38.83% +BenchmarkLondon_ways-8 257494509 140700970 -45.36% +BenchmarkLondon_relations-8 189490128 75263200 -60.28% + +benchmark old allocs new allocs delta +BenchmarkLondon-8 4863784 4808526 -1.14% +BenchmarkLondon_withFiltersTrue-8 4863786 4808515 -1.14% +BenchmarkLondon_withFiltersFalse-8 1419995 1364724 -3.89% +BenchmarkLondon_nodes-8 3450825 3395559 -1.60% +BenchmarkLondon_ways-8 1851359 1796099 -2.98% +BenchmarkLondon_relations-8 515422 460152 -10.72% + +benchmark old bytes new bytes delta +BenchmarkLondon-8 947061317 924789892 -2.35% +BenchmarkLondon_withFiltersTrue-8 947061146 924787588 -2.35% +BenchmarkLondon_withFiltersFalse-8 388725836 366452840 -5.73% +BenchmarkLondon_nodes-8 641663624 619391213 -3.47% +BenchmarkLondon_ways-8 460631859 438360054 -4.84% +BenchmarkLondon_relations-8 206899749 184626277 -10.77% \ No newline at end of file diff --git a/osmpbf/decode.go b/osmpbf/decode.go index eec3892..fa56d67 100644 --- a/osmpbf/decode.go +++ b/osmpbf/decode.go @@ -1,7 +1,6 @@ package osmpbf import ( - "bytes" "context" "encoding/binary" "errors" @@ -344,28 +343,8 @@ func getData(blob *osmpbf.Blob, data []byte) ([]byte, error) { return blob.GetRaw(), nil case blob.ZlibData != nil: - r, err := zlibReader(blob.GetZlibData()) - if err != nil { - return nil, err - } - - // using the bytes.Buffer allows for the preallocation of the necessary space. - l := blob.GetRawSize() + bytes.MinRead - if cap(data) < int(l) { - data = make([]byte, 0, l+l/10) - } else { - data = data[:0] - } - buf := bytes.NewBuffer(data) - if _, err = buf.ReadFrom(r); err != nil { - return nil, err - } - - if buf.Len() != int(blob.GetRawSize()) { - return nil, fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), blob.GetRawSize()) - } + return decompress(blob.GetZlibData(), (int)(blob.GetRawSize()), data) - return buf.Bytes(), nil default: return nil, errors.New("unknown blob data") } diff --git a/osmpbf/zlib_cgo.go b/osmpbf/zlib_cgo.go index fbcf3f6..e8e25ea 100644 --- a/osmpbf/zlib_cgo.go +++ b/osmpbf/zlib_cgo.go @@ -1,14 +1,25 @@ +//go:build cgo // +build cgo package osmpbf import ( - "bytes" - "io" + "fmt" - "github.com/datadog/czlib" + deflate "github.com/4kills/go-libdeflate/v2" ) -func zlibReader(data []byte) (io.ReadCloser, error) { - return czlib.NewReader(bytes.NewReader(data)) +func decompress(in []byte, size int, data []byte) ([]byte, error) { + if cap(data) > (int)(size) { + data = data[0:size] + } else { + data = nil + } + + _, buf, err := deflate.DecompressZlib(in, data) + if len(buf) != int(size) { + return nil, fmt.Errorf("raw blob data size %d but expected %d", len(buf), size) + } + + return buf, err } diff --git a/osmpbf/zlib_go.go b/osmpbf/zlib_go.go index ddae9b8..16ec1b8 100644 --- a/osmpbf/zlib_go.go +++ b/osmpbf/zlib_go.go @@ -1,3 +1,4 @@ +//go:build !cgo // +build !cgo package osmpbf @@ -5,9 +6,29 @@ package osmpbf import ( "bytes" "compress/zlib" - "io" + "fmt" ) -func zlibReader(data []byte) (io.ReadCloser, error) { - return zlib.NewReader(bytes.NewReader(data)) +func decompress(in []byte, size int, data []byte) ([]byte, error) { + r, err := zlib.NewReader(bytes.NewReader(in)) + if err != nil { + return nil, err + } + + // using the bytes.Buffer allows for the preallocation of the necessary space. + l := size + bytes.MinRead + if cap(data) < int(l) { + data = make([]byte, 0, l+l/10) + } else { + data = data[:0] + } + buf := bytes.NewBuffer(data) + if _, err = buf.ReadFrom(r); err != nil { + return nil, err + } + + if buf.Len() != int(size) { + return nil, fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), size) + } + return buf.Bytes(), nil } From b35665f79c5420e96653d9c95aac2705a1f8a9ef Mon Sep 17 00:00:00 2001 From: Olaf Flebbe Date: Sat, 23 Sep 2023 19:20:42 +0200 Subject: [PATCH 2/5] update go version --- .github/workflows/main.yml | 2 +- go.mod | 7 ++-- go.sum | 81 +++++++++++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 70c7be3..f611ee1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: '1.15.3' + go-version: '1.19.13' - name: Install dependencies run: | diff --git a/go.mod b/go.mod index d6d940b..00f25bc 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,13 @@ module github.com/paulmach/osm -go 1.13 +go 1.19 require ( github.com/4kills/go-libdeflate/v2 v2.0.3 - github.com/paulmach/orb v0.1.3 + github.com/paulmach/orb v0.10.0 github.com/paulmach/protoscan v0.2.1 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.27.1 ) + +require go.mongodb.org/mongo-driver v1.12.1 // indirect diff --git a/go.sum b/go.sum index f1983fb..c58781e 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,94 @@ github.com/4kills/go-libdeflate/v2 v2.0.3 h1:Y13oRUvtAXFJkcW4F0MnaQQB753a71sTutGrVbEAubQ= github.com/4kills/go-libdeflate/v2 v2.0.3/go.mod h1:hyouZv4OAhHaaMpYuejstUN0xOg8mA+yy75WE3Ty6SM= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM= -github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s= +github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= +go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 14df2f73ab18d34837d237ecd42cc1e6eb53d8c4 Mon Sep 17 00:00:00 2001 From: Olaf Flebbe Date: Sat, 23 Sep 2023 19:31:49 +0200 Subject: [PATCH 3/5] renovate workflow --- .github/workflows/main.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f611ee1..0ae97b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,28 +10,27 @@ jobs: build-and-test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: go-version: '1.19.13' + cache: false - name: Install dependencies run: | go version - go get -u golang.org/x/lint/golint - name: Run build run: go build . + - name: Run tests + run: go test -v -coverprofile=profile.cov ./... + - name: Run vet & lint run: | go vet . - golint . - - - name: Run tests - run: go test -v -coverprofile=profile.cov ./... - name: codecov uses: codecov/codecov-action@v1 From 73efb86a0faa3094c889b7c59784986bff959118 Mon Sep 17 00:00:00 2001 From: Paul Mach Date: Wed, 27 Dec 2023 22:47:42 -0800 Subject: [PATCH 4/5] osmpbf: move shared decompress code back into decode.go --- osmpbf/README.md | 14 ++++++++++---- osmpbf/decode.go | 19 ++++++++++++++++++- osmpbf/zlib_cgo.go | 16 ++-------------- osmpbf/zlib_go.go | 15 ++------------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/osmpbf/README.md b/osmpbf/README.md index fe171b1..6bf4a3c 100644 --- a/osmpbf/README.md +++ b/osmpbf/README.md @@ -76,9 +76,15 @@ type Scanner struct { ## OSM PBF files with node locations on ways -This package supports reading OSM PBF files where the ways have been annotated with the coordinates of each node. Such files can be generated using [osmium](https://osmcode.org/osmium-tool), with the [add-locations-to-ways](https://docs.osmcode.org/osmium/latest/osmium-add-locations-to-ways.html) subcommand. This feature makes it possible to work with the ways and their geometries without having to keep all node locations in some index (which takes work and memory resources). +This package supports reading OSM PBF files where the ways have been annotated with the coordinates of each node. +Such files can be generated using [osmium](https://osmcode.org/osmium-tool), +with the [add-locations-to-ways](https://docs.osmcode.org/osmium/latest/osmium-add-locations-to-ways.html) subcommand. +This feature makes it possible to work with the ways and their geometries without having to keep all node locations +in some index (which takes work and memory resources). -Coordinates are stored in the `Lat` and `Lon` fields of each `WayNode`. There is no need to specify an explicit option; when the node locations are present on the ways, they are loaded automatically. For more info about the OSM PBF format extension, see [the original blog post](https://blog.jochentopf.com/2016-04-20-node-locations-on-ways.html). +Coordinates are stored in the `Lat` and `Lon` fields of each `WayNode`. There is no need to specify an explicit option; +when the node locations are present on the ways, they are loaded automatically. For more info about the OSM PBF format extension, +see [the original blog post](https://blog.jochentopf.com/2016-04-20-node-locations-on-ways.html). ## Using cgo libdeflate for decompression @@ -87,7 +93,7 @@ implementation this can account for about 1/3 of the read time. When cgo is enab the package [go-libdeflate](https://github.com/4kills/libdeflate) will used. Previous versions used the lib czlib based on zlib. libdeflate is more performant -and more memory efficient for uncompressing. +and more memory efficient for decompression. ``` $ CGO_ENABLED=0 go test -bench . > disabled.txt @@ -115,4 +121,4 @@ BenchmarkLondon_withFiltersTrue-8 947061146 924787588 -2.35% BenchmarkLondon_withFiltersFalse-8 388725836 366452840 -5.73% BenchmarkLondon_nodes-8 641663624 619391213 -3.47% BenchmarkLondon_ways-8 460631859 438360054 -4.84% -BenchmarkLondon_relations-8 206899749 184626277 -10.77% \ No newline at end of file +BenchmarkLondon_relations-8 206899749 184626277 -10.77% diff --git a/osmpbf/decode.go b/osmpbf/decode.go index fa56d67..f8f0ec9 100644 --- a/osmpbf/decode.go +++ b/osmpbf/decode.go @@ -1,6 +1,7 @@ package osmpbf import ( + "bytes" "context" "encoding/binary" "errors" @@ -343,8 +344,24 @@ func getData(blob *osmpbf.Blob, data []byte) ([]byte, error) { return blob.GetRaw(), nil case blob.ZlibData != nil: - return decompress(blob.GetZlibData(), (int)(blob.GetRawSize()), data) + size := int(blob.GetRawSize()) + l := size + bytes.MinRead + if cap(data) < l { + data = make([]byte, 0, l+l/10) + } else { + data = data[:0] + } + + out, err := decompress(blob.GetZlibData(), data) + if err != nil { + return nil, err + } + + if len(out) != size { + return nil, fmt.Errorf("raw blob data size %d but expected %d", len(out), size) + } + return out, nil default: return nil, errors.New("unknown blob data") } diff --git a/osmpbf/zlib_cgo.go b/osmpbf/zlib_cgo.go index e8e25ea..7397a35 100644 --- a/osmpbf/zlib_cgo.go +++ b/osmpbf/zlib_cgo.go @@ -4,22 +4,10 @@ package osmpbf import ( - "fmt" - deflate "github.com/4kills/go-libdeflate/v2" ) -func decompress(in []byte, size int, data []byte) ([]byte, error) { - if cap(data) > (int)(size) { - data = data[0:size] - } else { - data = nil - } - - _, buf, err := deflate.DecompressZlib(in, data) - if len(buf) != int(size) { - return nil, fmt.Errorf("raw blob data size %d but expected %d", len(buf), size) - } - +func decompress(in []byte, out []byte) ([]byte, error) { + _, buf, err := deflate.DecompressZlib(in, out) return buf, err } diff --git a/osmpbf/zlib_go.go b/osmpbf/zlib_go.go index 16ec1b8..593da1d 100644 --- a/osmpbf/zlib_go.go +++ b/osmpbf/zlib_go.go @@ -6,29 +6,18 @@ package osmpbf import ( "bytes" "compress/zlib" - "fmt" ) -func decompress(in []byte, size int, data []byte) ([]byte, error) { +func decompress(in []byte, out []byte) ([]byte, error) { r, err := zlib.NewReader(bytes.NewReader(in)) if err != nil { return nil, err } - // using the bytes.Buffer allows for the preallocation of the necessary space. - l := size + bytes.MinRead - if cap(data) < int(l) { - data = make([]byte, 0, l+l/10) - } else { - data = data[:0] - } - buf := bytes.NewBuffer(data) + buf := bytes.NewBuffer(out) if _, err = buf.ReadFrom(r); err != nil { return nil, err } - if buf.Len() != int(size) { - return nil, fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), size) - } return buf.Bytes(), nil } From 8acc889c3fcdf77a25ea06743d8914acde6da787 Mon Sep 17 00:00:00 2001 From: Paul Mach Date: Wed, 27 Dec 2023 22:57:39 -0800 Subject: [PATCH 5/5] fix lint errors: remove ioutil references --- .github/workflows/main.yml | 2 +- README.md | 2 +- annotate/way_test.go | 3 +-- change_test.go | 20 ++++++++++++-------- diff_test.go | 20 +++++++++++++++----- osm_test.go | 17 ----------------- osmgeojson/benchmarks_test.go | 4 ++-- osmgeojson/convert_test.go | 2 +- relation_test.go | 6 +++++- replication/changesets.go | 3 +-- replication/interval.go | 4 ++-- way_test.go | 6 +++++- 12 files changed, 46 insertions(+), 43 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index efe2bec..b6d3940 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,4 +29,4 @@ jobs: - name: codecov uses: codecov/codecov-action@v1 with: - file: ./profile.cov \ No newline at end of file + file: ./profile.cov diff --git a/README.md b/README.md index ce85096..0f5cef9 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ var c = jsoniter.Config{ MarshalFloatWith6Digits: true, }.Froze() -osmm.CustomJSONMarshaler = c +osm.CustomJSONMarshaler = c osm.CustomJSONUnmarshaler = c ``` diff --git a/annotate/way_test.go b/annotate/way_test.go index 42d35ce..01bc933 100644 --- a/annotate/way_test.go +++ b/annotate/way_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" "os" "reflect" "testing" @@ -132,7 +131,7 @@ func BenchmarkWays(b *testing.B) { } func loadTestdata(tb testing.TB, filename string) *osm.OSM { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { tb.Fatalf("unable to open file: %v", err) } diff --git a/change_test.go b/change_test.go index 444d754..085f60a 100644 --- a/change_test.go +++ b/change_test.go @@ -5,7 +5,7 @@ import ( "context" "encoding/json" "encoding/xml" - "io/ioutil" + "os" "testing" ) @@ -237,11 +237,13 @@ func TestChange_HistoryDatasource(t *testing.T) { } func BenchmarkChange_MarshalXML(b *testing.B) { - filename := "testdata/changeset_38162206.osc" - data := readFile(b, filename) + data, err := os.ReadFile("testdata/changeset_38162206.osc") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } c := &Change{} - err := xml.Unmarshal(data, c) + err = xml.Unmarshal(data, c) if err != nil { b.Fatalf("unable to unmarshal: %v", err) } @@ -270,7 +272,7 @@ func BenchmarkChange_MarshalXML(b *testing.B) { // } func BenchmarkChange_MarshalJSON(b *testing.B) { - data, err := ioutil.ReadFile("testdata/minute_871.osc") + data, err := os.ReadFile("testdata/minute_871.osc") if err != nil { b.Fatalf("could not read file: %v", err) } @@ -292,7 +294,7 @@ func BenchmarkChange_MarshalJSON(b *testing.B) { } func BenchmarkChange_UnmarshalJSON(b *testing.B) { - data, err := ioutil.ReadFile("testdata/minute_871.osc") + data, err := os.ReadFile("testdata/minute_871.osc") if err != nil { b.Fatalf("could not read file: %v", err) } @@ -320,8 +322,10 @@ func BenchmarkChange_UnmarshalJSON(b *testing.B) { } func BenchmarkChangeset_UnmarshalXML(b *testing.B) { - filename := "testdata/changeset_38162206.osc" - data := readFile(b, filename) + data, err := os.ReadFile("testdata/changeset_38162206.osc") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } b.ReportAllocs() b.ResetTimer() diff --git a/diff_test.go b/diff_test.go index 0955747..6ff329b 100644 --- a/diff_test.go +++ b/diff_test.go @@ -2,6 +2,7 @@ package osm import ( "encoding/xml" + "os" "reflect" "testing" ) @@ -72,10 +73,13 @@ func TestDiff_MarshalXML(t *testing.T) { } func TestDiff(t *testing.T) { - data := readFile(t, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + t.Fatalf("unable to read file: %v", err) + } diff := &Diff{} - err := xml.Unmarshal(data, &diff) + err = xml.Unmarshal(data, &diff) if err != nil { t.Errorf("unable to unmarshal: %v", err) } @@ -137,10 +141,13 @@ func TestDiff(t *testing.T) { } func BenchmarkDiff_Marshal(b *testing.B) { - data := readFile(b, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } diff := &Diff{} - err := xml.Unmarshal(data, &diff) + err = xml.Unmarshal(data, &diff) if err != nil { b.Fatalf("unmarshal error: %v", err) } @@ -156,7 +163,10 @@ func BenchmarkDiff_Marshal(b *testing.B) { } func BenchmarkDiff_Unmarshal(b *testing.B) { - data := readFile(b, "testdata/annotated_diff.xml") + data, err := os.ReadFile("testdata/annotated_diff.xml") + if err != nil { + b.Fatalf("unable to read file: %v", err) + } b.ReportAllocs() b.ResetTimer() diff --git a/osm_test.go b/osm_test.go index a80c807..7f57c8a 100644 --- a/osm_test.go +++ b/osm_test.go @@ -4,8 +4,6 @@ import ( "bytes" "encoding/json" "encoding/xml" - "io" - "os" "reflect" "testing" ) @@ -301,18 +299,3 @@ func TestOSM_MarshalXML(t *testing.T) { t.Errorf("incorrect marshal, got: %s", string(data)) } } - -func readFile(t testing.TB, filename string) []byte { - f, err := os.Open(filename) - if err != nil { - t.Fatalf("unable to open %s: %v", filename, err) - } - defer f.Close() - - data, err := io.ReadAll(f) - if err != nil { - t.Fatalf("unable to read file %s: %v", filename, err) - } - - return data -} diff --git a/osmgeojson/benchmarks_test.go b/osmgeojson/benchmarks_test.go index 4877187..3d25259 100644 --- a/osmgeojson/benchmarks_test.go +++ b/osmgeojson/benchmarks_test.go @@ -2,7 +2,7 @@ package osmgeojson import ( "encoding/xml" - "io/ioutil" + "os" "testing" "github.com/paulmach/osm" @@ -88,7 +88,7 @@ func BenchmarkConvert_NoIDsMetaMembership(b *testing.B) { } func parseFile(t testing.TB, filename string) *osm.OSM { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { t.Fatalf("could not read file: %v", err) } diff --git a/osmgeojson/convert_test.go b/osmgeojson/convert_test.go index 3209d0e..3c59e83 100644 --- a/osmgeojson/convert_test.go +++ b/osmgeojson/convert_test.go @@ -674,7 +674,7 @@ type rawFC struct { func jsonLoop(t *testing.T, fc *geojson.FeatureCollection) *rawFC { data, err := json.Marshal(fc) if err != nil { - t.Fatalf("unabled to marshal fc: %v", err) + t.Fatalf("unable to marshal fc: %v", err) } result := &rawFC{} diff --git a/relation_test.go b/relation_test.go index bb19e92..63b4b5e 100644 --- a/relation_test.go +++ b/relation_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "encoding/xml" + "os" "reflect" "testing" "time" @@ -199,7 +200,10 @@ func TestRelation_MarshalXML(t *testing.T) { } // blanket xml test - data = readFile(t, "testdata/relation-updates.osm") + data, err = os.ReadFile("testdata/relation-updates.osm") + if err != nil { + t.Fatalf("could not read file: %v", err) + } osm := &OSM{} err = xml.Unmarshal(data, &osm) diff --git a/replication/changesets.go b/replication/changesets.go index f149c8c..65eb273 100644 --- a/replication/changesets.go +++ b/replication/changesets.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "strconv" @@ -88,7 +87,7 @@ func (ds *Datasource) fetchChangesetState(ctx context.Context, n ChangesetSeqNum } } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/replication/interval.go b/replication/interval.go index 8c258bd..d322d85 100644 --- a/replication/interval.go +++ b/replication/interval.go @@ -6,7 +6,7 @@ import ( "context" "encoding/xml" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "time" @@ -219,7 +219,7 @@ func (ds *Datasource) fetchState(ctx context.Context, n SeqNum) (*State, error) } } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/way_test.go b/way_test.go index 59d7ab3..d669bbd 100644 --- a/way_test.go +++ b/way_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "encoding/xml" + "os" "reflect" "testing" "time" @@ -284,7 +285,10 @@ func TestWay_MarshalXML(t *testing.T) { } // blanket xml test - data = readFile(t, "testdata/way-updates.osm") + data, err = os.ReadFile("testdata/way-updates.osm") + if err != nil { + t.Fatalf("unable to read file: %v", err) + } osm := &OSM{} err = xml.Unmarshal(data, &osm)