diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d8e7f86..4e24408 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: 1.22 - name: Build run: go build -v ./... diff --git a/Dockerfile b/Dockerfile index af1c4a3..6181570 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-alpine AS builder +FROM golang:1.22-alpine AS builder LABEL maintainer="github:@rusq" WORKDIR /build @@ -6,10 +6,10 @@ COPY . . RUN go build -ldflags="-s -w" ./cmd/aklapi -FROM alpine:3.17 +FROM alpine:3.19.1 LABEL maintainer="github:@rusq" -RUN apk add --no-cache ca-certificates && apk --no-cache add tzdata +RUN apk add --no-cache ca-certificates WORKDIR /app diff --git a/cmd/aklapi/handlers.go b/cmd/aklapi/handlers.go index d342d1e..871c0fb 100644 --- a/cmd/aklapi/handlers.go +++ b/cmd/aklapi/handlers.go @@ -5,11 +5,12 @@ import ( "errors" "log" "net/http" + "time" "github.com/rusq/aklapi" ) -const dateFmt = "2006-01-02" +const dttmLayout = "2006-01-02" type rrResponse struct { Rubbish string `json:"rubbish,omitempty"` @@ -59,14 +60,22 @@ func rrHandler(w http.ResponseWriter, r *http.Request) { return } resp := rrResponse{ - Recycle: res.NextRecycle().Format(dateFmt), - Rubbish: res.NextRubbish().Format(dateFmt), - FoodScraps: res.NextFoodScraps().Format(dateFmt), + Recycle: timefmt(res.NextRecycle()), + Rubbish: timefmt(res.NextRubbish()), + FoodScraps: timefmt(res.NextFoodScraps()), Address: res.Address.Address, } respond(w, resp, http.StatusOK) } +// timefmt formats the time skipping empty time. +func timefmt(t time.Time) string { + if t.IsZero() { + return "" + } + return t.Format(dttmLayout) +} + func rrExtHandler(w http.ResponseWriter, r *http.Request) { res, err := rubbish(r) if err != nil { diff --git a/cmd/aklapi/main.go b/cmd/aklapi/main.go index 6d68f6f..10e697d 100644 --- a/cmd/aklapi/main.go +++ b/cmd/aklapi/main.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + _ "time/tzdata" "github.com/rusq/aklapi" "github.com/rusq/osenv/v2" @@ -41,6 +42,7 @@ var ( var tmpl = template.Must(template.New("index.html").Parse(rootHTML)) func main() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) flag.Parse() if *port == "" { log.Printf("no port specified, defaulting to %s", defaultPort) diff --git a/go.mod b/go.mod index 7a0cc38..6d734d6 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/rusq/aklapi -go 1.19 +go 1.22 require ( - github.com/PuerkitoBio/goquery v1.8.1 + github.com/PuerkitoBio/goquery v1.9.1 github.com/rusq/osenv/v2 v2.0.1 github.com/stretchr/testify v1.8.1 ) @@ -12,6 +12,6 @@ require ( github.com/andybalholm/cascadia v1.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.24.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 94dc9d5..0a373a8 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= +github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= @@ -28,11 +29,12 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/mocks_test.go b/mocks_test.go index e48f044..9367c8b 100644 --- a/mocks_test.go +++ b/mocks_test.go @@ -1,1389 +1,15 @@ package aklapi -const testHTML = ` - - - - - - - - Your collection day - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- -
- -
-
- Skip to main content - - - - - - - - -
- - Auckland Council
-
-
- Auckland Council -
-
- - Auckland Council
- -
-
-
-
- - myAUCKLAND
- - - - -
- -
-
- - - -
-
- - - -
-
- - - -
-
- -
-
- - - - - -
- - - - - - - - -
- - - -
-
- - - -
-
-
-
-
-
-
-
- - -

Your collection day

- -
- -

Ngā kōrero kohinga mōu

-

Red Square, Moscow

- -

- -
-
-

Household collection

-

Rubbish Rubbish

-

Collection day: Tuesday, weekly except after a public holiday.

- -

Recycle Recycling

-

Collection day: Tuesday, fortnightly except after a public holiday.

- Put bins out the night before or before 7am. - -
-
- -

Your next collection dates:

- -
Tuesday 11 FebruaryRubbish Recycle
- -
Tuesday 18 FebruaryRubbish
- -
Where you can put your rubbish and recycling for collection
-
What you can put in your recycle bin
-
What you can put in your rubbish bin
- -
-
- -
-

- Rubbish and recycling collection calendar

- PDF 420.3 KB
- -
-
-

Commercial collection

-

Rubbish Rubbish

-

Collection days: Tuesday, Thursday and Saturday, weekly. See public holidays for exceptions.

- -

Recycle Recycling

-

Collection day: Tuesday, fortnightly except after a public holiday.

- Put bins out the night before or before 7am. - -
-
- -

Your next collection dates:

- -
Tuesday 11 FebruaryRubbish Recycle
- -
Tuesday 18 FebruaryRubbish
- -
Where you can put your rubbish and recycling for collection
-
What you can put in your recycle bin
-
What you can put in your rubbish bin
- -
-
- -
- - -
-
-
-
-
-
- - - -
-
-
-
-
- -
-
- - - - - -
- - - -
- - -
-
- - -
-
-
-
- -
- -
- Terms and conditionsPrivacy policySite map
© Auckland Council 2019
New Zealand Government
- -
- -
-
- - - - -
- - - - - -
- - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -` - -var testHTMLcommercial = ` - - - - - - - - - - Your collection day - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
- - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - - -
- - - - -
- - - - - - - - -
- -
- -
- -
-
- Skip to main content - - - - - - - - -
- - Auckland Council
-
-
- Auckland Council -
-
- - Auckland Council
- -
-
-
-
- - myAUCKLAND
- - - - -
- -
- - Water conservation efforts
 - - More - - - How we are reducing water usage across the Auckland region. - Learn moreWater conservation efforts
 - - - - -
- -
-
- - - -
-
- - - -
-
- - - -
-
- -
-
- - - - - -
- - - - - - - - -
- - - -
-
- - - -
-
-
-
-
-
-
-
- - -

Your collection day

- -
- -

Ngā kōrero kohinga mōu

-

500 Queen Street, Auckland Central

- -

- -
-
-

Household collection

-

Rubbish Rubbish

-

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: -No changes to collection days.

- -

Recycle Recycling

-

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: -No changes to collection days.

-

Put bags out before collection times: Monday to Saturday, 5pm to 5.30pm and 12am to 4am
Sunday after 12am only

- -
- -
- - - -
-
-

Commercial collection

-

Rubbish Rubbish

-

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: -No changes to collection days.

- -

Recycle Recycling

-

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: -No changes to collection days.

-

Put bags out before collection times: Monday to Saturday, 5pm to 5.30pm and 12am to 4am
Sunday after 12am only

- -
- -
- -
- - -
-
-
-
-
-
- - - -
-
-
-
-
- -
-
- - - - - -
- - - -
- - -
-
- - -
-
-
-
- -
- -
- Terms and conditionsPrivacy policySite map
© Auckland Council 2020
New Zealand Government
- -
- -
-
- - - - -
- - - - - -
- - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -` +import ( + _ "embed" +) + +//go:generate curl -o test_assets/your_collection_day_500_queen_street.html https://www.aucklandcouncil.govt.nz/rubbish-recycling/rubbish-recycling-collections/Pages/collection-day-detail.aspx?an=12342478585 + +var ( + // testHTML is the test HTML content for private household. + // + //go:embed "test_assets/your_collection_day_500_queen_street.html" + testHTML string + testHTMLcommercial = testHTML +) diff --git a/rubbish.go b/rubbish.go index 3475fa8..bc462e6 100644 --- a/rubbish.go +++ b/rubbish.go @@ -125,15 +125,15 @@ type refuseParser struct { // Parse parses the auckland council rubbish webpage. func (p *refuseParser) parse(r io.Reader) ([]RubbishCollection, error) { - const datesSection = "#ctl00_SPWebPartManager1_g_dfe289d2_6a8a_414d_a384_fc25a0db9a6d_ctl00_pnlHouseholdBlock" - p.detail = make([]RubbishCollection, 2) + const datesSection = "#ctl00_SPWebPartManager1_g_dfe289d2_6a8a_414d_a384_fc25a0db9a6d_ctl00_pnlHouseholdBlock2" + p.detail = make([]RubbishCollection, 3) doc, err := goquery.NewDocumentFromReader(r) if err != nil { return nil, err } _ = doc.Find(datesSection). Children(). - Slice(1, 3). + Slice(1, 4). Each(p.parseLinks) // p.parseLinks populates p.detail for i := range p.detail { if err := (&p.detail[i]).parseDate(); err != nil { @@ -150,17 +150,11 @@ func (p *refuseParser) parse(r io.Reader) ([]RubbishCollection, error) { return p.detail, p.Err } -// parseLinks parses the links within selection +// parseLinks parses the links within selection and populates p.detail. func (p *refuseParser) parseLinks(el int, sel *goquery.Selection) { - sel.Children().Each(func(n int, sel *goquery.Selection) { + sel.Children().Children().Each(func(n int, sel *goquery.Selection) { switch n { case 0: - if dow.FindString(sel.Text()) == "" { - log.Println("unable to detect day of week") - return - } - p.detail[el].Day = sel.Text() - default: if sel.Text() == "Rubbish" { p.detail[el].Rubbish = true } else if sel.Text() == "Food scraps" { @@ -170,6 +164,12 @@ func (p *refuseParser) parseLinks(el int, sel *goquery.Selection) { } else { p.Err = fmt.Errorf("parse error: sel.Text = %q, el = %d, n = %d", sel.Text(), el, n) } + default: + if dow.FindString(sel.Text()) == "" { + log.Println("unable to detect day of week") + return + } + p.detail[el].Day = sel.Text() } }) } diff --git a/rubbish_test.go b/rubbish_test.go index a7a476a..5275ed6 100644 --- a/rubbish_test.go +++ b/rubbish_test.go @@ -27,16 +27,16 @@ func Test_parse(t *testing.T) { args{strings.NewReader(testHTML)}, &CollectionDayDetailResult{ Collections: []RubbishCollection{ - {Day: "Tuesday 11 February", - Date: adjustYear(time.Date(0, 02, 11, 0, 0, 0, 0, defaultLoc)), - Rubbish: true, - Recycle: true, - FoodScraps: false}, - {Day: "Tuesday 18 February", - Date: adjustYear(time.Date(0, 02, 18, 0, 0, 0, 0, defaultLoc)), + {Day: "Sunday 21 April", + Date: adjustYear(time.Date(0, 04, 21, 0, 0, 0, 0, defaultLoc)), Rubbish: true, Recycle: false, FoodScraps: false}, + {Day: "Sunday 21 April", + Date: adjustYear(time.Date(0, 04, 21, 0, 0, 0, 0, defaultLoc)), + Rubbish: false, + Recycle: true, + FoodScraps: false}, }, Address: nil, }, @@ -45,10 +45,18 @@ func Test_parse(t *testing.T) { args{strings.NewReader(testHTMLcommercial)}, &CollectionDayDetailResult{ Collections: []RubbishCollection{ - {Day: "Monday 24 February", - Date: adjustYear(time.Date(0, 02, 24, 0, 0, 0, 0, defaultLoc)), + { + Day: "Sunday 21 April", + Date: adjustYear(time.Date(0, 04, 21, 0, 0, 0, 0, defaultLoc)), Rubbish: true, - Recycle: true}, + Recycle: false, + }, + { + Day: "Sunday 21 April", + Date: adjustYear(time.Date(0, 04, 21, 0, 0, 0, 0, defaultLoc)), + Rubbish: false, + Recycle: true, + }, }, Address: nil, }, @@ -201,17 +209,17 @@ func TestCollectionDayDetail(t *testing.T) { &CollectionDayDetailResult{ Collections: []RubbishCollection{ { - Day: "Tuesday 11 February", - Date: adjustYear(time.Date(0, 2, 11, 0, 0, 0, 0, defaultLoc)), + Day: "Sunday 21 April", + Date: adjustYear(time.Date(0, 4, 21, 0, 0, 0, 0, defaultLoc)), Rubbish: true, - Recycle: true, + Recycle: false, FoodScraps: false, }, { - Day: "Tuesday 18 February", - Date: adjustYear(time.Date(0, 2, 18, 0, 0, 0, 0, defaultLoc)), - Rubbish: true, - Recycle: false, + Day: "Sunday 21 April", + Date: adjustYear(time.Date(0, 4, 21, 0, 0, 0, 0, defaultLoc)), + Rubbish: false, + Recycle: true, FoodScraps: false, }, }, diff --git a/test_assets/your_collection_day_500_queen_street.html b/test_assets/your_collection_day_500_queen_street.html new file mode 100644 index 0000000..05c5f5c --- /dev/null +++ b/test_assets/your_collection_day_500_queen_street.html @@ -0,0 +1,826 @@ + + + + + + + + + + Your collection day + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
+ + + + +
+ + + + + + + + +
+ +
+ +
+ +
+
+ Skip to main content + + + + + + + + +
+ + Auckland Council
+
+
+
+ Auckland Council +
+
+ + Auckland Council
+ +
+
+
+
+ myAUCKLAND
+ + + + +
+ +
+
+ + + +
+
+ + + +
+
+ + + +
+
+ +
+
+ + + + + +
+ + + + + + + + + + + +
+ + + +
+
+ + + +
+
+
+
+
+
+
+
+
+ +

Ngā kōrero kohinga mōu

+ +
+

+ Your collection day

+
+ +

500 Queen Street, Auckland Central

+

+ +
+ + + +
+ +
+
+

Household collection

+
+ +

Your next collection dates:

+
+ +
Rubbish Rubbish: + Sunday 21 April
+ +
+ +
+ +
+ +
Recycle Recycling: + Sunday 21 April
+ +
+ +
+

Put bags out before collection times: Monday to Saturday, 5pm to 5.30pm and 12am to 4am
Sunday after 12am only

+ +
+
+
+
+
How often do I put my bins out:
+

Rubbish Rubbish

+

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: +No changes to collection days.

+

Food scraps Food scraps

+

This property is listed as out of food scraps service area and will not receive collection service.

+

Recycle Recycling

+

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: +No changes to collection days.

+
+
+ +
+
+ + + +
+
+

Commercial collection

+
+ +

Your next collection dates:

+
+ +
Rubbish Rubbish: + Sunday 21 April
+ +
+ +
+
+ +
Recycle Recycling: + Sunday 21 April
+ +
+ +
+

Put bags out before collection times: Monday to Saturday, 5pm to 5.30pm and 12am to 4am
Sunday after 12am only

+ +
+
+
+
+
How often do I put my bins out:
+

Rubbish Rubbish

+

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: +No changes to collection days.

+

Recycle Recycling

+

Collection days: Monday to Saturday twice daily, Sunday, once daily
Public holidays: +No changes to collection days.

+
+
+ +
+
+ +
+ + +
+
+
+
+
+
+ + + +
+
+
+
+
+
+
+
+
+

Related topics

 +
+
+
+

+
+ +
+
+
+
+
+
+
+
+ + + + + +
+ + + +
+ + +
+
+ + +
+
+
+
+ +
+ +
+
+ Terms and conditionsPrivacy policySite map
© Auckland Council 2024
New Zealand Government

+ +
+ +
+
+ + + + +
+ + + + + +
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +