Skip to content

Commit

Permalink
Format
Browse files Browse the repository at this point in the history
  • Loading branch information
pops64 authored Aug 23, 2024
1 parent be9993c commit 2eeb0c9
Showing 1 changed file with 7 additions and 8 deletions.
15 changes: 7 additions & 8 deletions pkg/scrape/wetvr.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ package scrape

import (
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"fmt"

"github.com/gocolly/colly/v2"
"github.com/mozillazg/go-slugify"
Expand Down Expand Up @@ -96,15 +96,14 @@ func WetVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan<-
siteCollector.OnHTML(`div:has(p:contains("Latest"))`, func(e *colly.HTMLElement) {
// Check to make sure we aren't getting multiple firings from pages we have already incremented the count on
if !limitScraping {
if e.Request.URL.String() == `https://wetvr.com/?page=` + fmt.Sprint(pageCnt) {
pageCnt += 1
pageURL := e.Request.AbsoluteURL(`https://wetvr.com/?page=` + fmt.Sprint(pageCnt))
siteCollector.Visit(pageURL)
}
if e.Request.URL.String() == `https://wetvr.com/?page=`+fmt.Sprint(pageCnt) {
pageCnt += 1

pageURL := e.Request.AbsoluteURL(`https://wetvr.com/?page=` + fmt.Sprint(pageCnt))
siteCollector.Visit(pageURL)
}
}
})


siteCollector.OnHTML(`div:has(p:contains("Latest")) div[id^="r-"]`, func(e *colly.HTMLElement) {
sceneURL := e.Request.AbsoluteURL(e.ChildAttr("a", "href"))
Expand Down

0 comments on commit 2eeb0c9

Please sign in to comment.