Skip to content

Commit

Permalink
Fixes Db Locks and Missing SLR Ai Scripts (rebased #1381) (#1393)
Browse files Browse the repository at this point in the history
* Fixes Db Locks and Missing SLR Ai Scripts

* Fix Merge Issues

Multi-Axis scripts are only detected if you script the scene
  Rules when processing are not clear with only 4 examples
  Which ar are not consistent
There currently only 12 scenes flagged a Multi-Script
  Only 4 were AI + Human the other 8 were Human + Human
  There Multi-scripts are only flagged as Human Generated

* Fix script update condition check

Use to only process scenes not already flagged
but now need to handle multiple scripts

* Feature Tags in Heresphere for AI or Hand Crafted

This allos user to filter AI/Hand Crafted scripts in HSP
withour been forced to use a Saved Search

* Misx Merge Issue with BaDonink

* Remove Multi-Axis

* Remove Multiaxis Filter Attribute
  • Loading branch information
toshski authored Sep 5, 2023
1 parent 276c63b commit 450ee00
Show file tree
Hide file tree
Showing 13 changed files with 227 additions and 38 deletions.
7 changes: 7 additions & 0 deletions pkg/api/heresphere.go
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,13 @@ func (i HeresphereResource) getHeresphereScene(req *restful.Request, resp *restf
Name: file.Filename,
URL: fmt.Sprintf("%v://%v/api/dms/file/%v", getProto(req), req.Request.Host, file.ID),
})

if scene.HumanScript {
addFeatureTag("Hand Crafted Funscript")
}
if scene.AiScript {
addFeatureTag("AI Generated Funscript")
}
}

var heresphereSubtitlesFiles []HeresphereSubtitles
Expand Down
20 changes: 20 additions & 0 deletions pkg/api/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ type RequestSaveOptionsAdvanced struct {
UseImperialEntry bool `json:"useImperialEntry"`
}

type RequestSaveOptionsFunscripts struct {
ScrapeFunscripts bool `json:"scrapeFunscripts":`
}
type RequestSaveOptionsDLNA struct {
Enabled bool `json:"enabled"`
ServiceName string `json:"name"`
Expand Down Expand Up @@ -236,6 +239,10 @@ func (i ConfigResource) WebService() *restful.WebService {
ws.Route(ws.PUT("/interface/advanced").To(i.saveOptionsAdvanced).
Metadata(restfulspec.KeyOpenAPITags, tags))

// "Web Advanced UI options" section endpoints
ws.Route(ws.PUT("/funscripts").To(i.saveOptionsFunscripts).
Metadata(restfulspec.KeyOpenAPITags, tags))

// "Cache" section endpoints
ws.Route(ws.DELETE("/cache/reset/{cache}").To(i.resetCache).
Param(ws.PathParameter("cache", "Cache to reset - possible choices are `images`, `previews`, and `searchIndex`").DataType("string")).
Expand Down Expand Up @@ -396,6 +403,19 @@ func (i ConfigResource) saveOptionsAdvanced(req *restful.Request, resp *restful.

resp.WriteHeaderAndEntity(http.StatusOK, r)
}
func (i ConfigResource) saveOptionsFunscripts(req *restful.Request, resp *restful.Response) {
var r RequestSaveOptionsFunscripts
err := req.ReadEntity(&r)
if err != nil {
log.Error(err)
return
}

config.Config.Funscripts.ScrapeFunscripts = r.ScrapeFunscripts
config.SaveConfig()

resp.WriteHeaderAndEntity(http.StatusOK, r)
}

func (i ConfigResource) saveOptionsDeoVR(req *restful.Request, resp *restful.Response) {
var r RequestSaveOptionsDeoVR
Expand Down
2 changes: 2 additions & 0 deletions pkg/api/scenes.go
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,8 @@ func (i SceneResource) getFilters(req *restful.Request, resp *restful.Response)
outAttributes = append(outAttributes, "VRPorn Scraper")
outAttributes = append(outAttributes, "Stashdb Linked")
outAttributes = append(outAttributes, "Has Script Download")
outAttributes = append(outAttributes, "Has AI Generated Script")
outAttributes = append(outAttributes, "Has Human Generated Script")
type Results struct {
Result string
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ type ObjectConfig struct {
UseImperialEntry bool `default:"false" json:"useImperialEntry"`
ProgressTimeInterval int `default:"15" json:"progressTimeInterval"`
} `json:"advanced"`
Funscripts struct {
ScrapeFunscripts bool `default:"false" json:"scrapeFunscripts"`
} `json:"funscripts"`
Vendor struct {
TPDB struct {
ApiToken string `default:"" json:"apiToken"`
Expand Down
10 changes: 10 additions & 0 deletions pkg/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -728,6 +728,16 @@ func Migrate() {
return tx.Exec("update scenes set script_published = '0000-00-00' where script_published is null").Error
},
},
{
ID: "0067-scenes-add-ai-multi-human-script-types",
Migrate: func(tx *gorm.DB) error {
type Scene struct {
AiScript bool `json:"ai_script" gorm:"default:false" xbvrbackup:"ai_script"`
HumanScript bool `json:"human_script" gorm:"default:false" xbvrbackup:"human_script"`
}
return tx.AutoMigrate(Scene{}).Error
},
},
{
ID: "0068-scene-wishlist",
Migrate: func(tx *gorm.DB) error {
Expand Down
22 changes: 22 additions & 0 deletions pkg/models/model_scene.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ type Scene struct {
LegacySceneID string `json:"legacy_scene_id" xbvrbackup:"legacy_scene_id"`

ScriptPublished time.Time `json:"script_published" xbvrbackup:"script_published"`
AiScript bool `json:"ai_script" gorm:"default:false" xbvrbackup:"ai_script"`
HumanScript bool `json:"human_script" gorm:"default:false" xbvrbackup:"human_script"`

Description string `gorm:"-" json:"description" xbvrbackup:"-"`
Score float64 `gorm:"-" json:"_score" xbvrbackup:"-"`
Expand Down Expand Up @@ -451,6 +453,8 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
if ext.HasScriptDownload && o.ScriptPublished.IsZero() {
o.ScriptPublished = time.Now()
}
o.AiScript = ext.AiScript
o.HumanScript = ext.HumanScript

// Trailers
o.TrailerType = ext.TrailerType
Expand Down Expand Up @@ -544,6 +548,20 @@ func SceneCreateUpdateFromExternal(db *gorm.DB, ext ScrapedScene) error {
return nil
}

func SceneUpdateScriptData(db *gorm.DB, ext ScrapedScene) {
var o Scene
o.GetIfExistByPK(ext.InternalSceneId)

if o.ID != 0 {
if o.ScriptPublished.IsZero() || o.HumanScript != ext.HumanScript || o.AiScript != ext.AiScript {
o.ScriptPublished = time.Now()
o.HumanScript = ext.HumanScript
o.AiScript = ext.AiScript
o.Save()
}
}
}

type RequestSceneList struct {
DlState optional.String `json:"dlState"`
Limit optional.Int `json:"limit"`
Expand Down Expand Up @@ -808,6 +826,10 @@ func queryScenes(db *gorm.DB, r RequestSceneList) (*gorm.DB, *gorm.DB) {
where = `scenes.scene_id like "vrporn-%"`
case "Has Script Download":
where = "scenes.script_published > '0001-01-01 00:00:00+00:00'"
case "Has AI Generated Script":
where = "scenes.ai_script = 1"
case "Has Human Generated Script":
where = "scenes.human_script = 1"
}

if negate {
Expand Down
5 changes: 5 additions & 0 deletions pkg/models/model_scraper.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ type ScrapedScene struct {
TrailerSrc string `json:"trailer_source"`
ChromaKey string `json:"chromakey"`
HasScriptDownload bool `json:"has_script_Download"`
AiScript bool `json:"ai_script"`
HumanScript bool `json:"human_script"`

OnlyUpdateScriptData bool `default:"false" json:"only_update_script_data"`
InternalSceneId uint `json:"internal_id"`

ActorDetails map[string]ActorDetails `json:"actor_details"`
}
Expand Down
39 changes: 27 additions & 12 deletions pkg/scrape/badoink.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/mozillazg/go-slugify"
"github.com/nleeper/goment"
"github.com/thoas/go-funk"
"github.com/xbapps/xbvr/pkg/config"

"github.com/xbapps/xbvr/pkg/common"
"github.com/xbapps/xbvr/pkg/ffprobe"
Expand Down Expand Up @@ -112,9 +113,11 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
}
})

e.ForEach(`p.video-tags a[href^="/category/funscript"]`, func(id int, e *colly.HTMLElement) {
sc.HasScriptDownload = true
})
if config.Config.Funscripts.ScrapeFunscripts {
e.ForEach(`p.video-tags a[href^="/category/funscript"]`, func(id int, e *colly.HTMLElement) {
sc.HasScriptDownload = true
})
}

ctx := colly.NewContext()
ctx.Put("scene", sc)
Expand Down Expand Up @@ -200,16 +203,28 @@ func BadoinkSite(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
}
})

siteCollector.OnHTML(`div.video-card-info`, func(e *colly.HTMLElement) {
sceneURL := ""
e.ForEach(`div.video-card-info-main a`, func(id int, e *colly.HTMLElement) {
sceneURL = e.Request.AbsoluteURL(e.Attr("href"))
})

e.ForEach(`a[href^="/category/funscript"]`, func(id int, e *colly.HTMLElement) {
db.Model(&models.Scene{}).Where("scene_url = ? and script_published = '0000-00-00'", sceneURL).Update("script_published", time.Now())
if config.Config.Funscripts.ScrapeFunscripts {
siteCollector.OnHTML(`div.video-card-info`, func(e *colly.HTMLElement) {
sceneURL := ""
e.ForEach(`div.video-card-info-main a`, func(id int, e *colly.HTMLElement) {
sceneURL = e.Request.AbsoluteURL(e.Attr("href"))
})

e.ForEach(`a[href^="/category/funscript"]`, func(id int, e *colly.HTMLElement) {
var existingScene models.Scene
existingScene.GetIfExistURL(sceneURL)
if existingScene.ID != 0 && existingScene.ScriptPublished.IsZero() {
var sc models.ScrapedScene
sc.InternalSceneId = existingScene.ID
sc.HasScriptDownload = true
sc.OnlyUpdateScriptData = true
sc.HumanScript = false
sc.AiScript = false
out <- sc
}
})
})
})
}

if singleSceneURL != "" {
sceneCollector.Visit(singleSceneURL)
Expand Down
46 changes: 30 additions & 16 deletions pkg/scrape/czechvr.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ import (
"strconv"
"strings"
"sync"
"time"

"github.com/gocolly/colly/v2"
"github.com/mozillazg/go-slugify"
"github.com/nleeper/goment"
"github.com/thoas/go-funk"
"github.com/xbapps/xbvr/pkg/config"
"github.com/xbapps/xbvr/pkg/models"
)

Expand Down Expand Up @@ -98,9 +98,11 @@ func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan
}
})

e.ForEach(`div.interactive`, func(id int, e *colly.HTMLElement) {
sc.HasScriptDownload = true
})
if config.Config.Funscripts.ScrapeFunscripts {
e.ForEach(`div.interactive`, func(id int, e *colly.HTMLElement) {
sc.HasScriptDownload = true
})
}

// trailer details
sc.TrailerType = "heresphere"
Expand Down Expand Up @@ -151,19 +153,31 @@ func CzechVR(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out chan
siteCollector.Visit(pageURL)
})

siteCollector.OnHTML(`div.postTag`, func(e *colly.HTMLElement) {
sceneURL := ""
e.ForEach(`div.foto a`, func(id int, e *colly.HTMLElement) {
sceneURL = e.Request.AbsoluteURL(e.Attr("href"))
// If scene exist in database, there's no need to scrape
if !funk.ContainsString(knownScenes, sceneURL) {
sceneCollector.Visit(sceneURL)
}
})
e.ForEach(`div.interactive`, func(id int, e *colly.HTMLElement) {
db.Model(&models.Scene{}).Where("scene_url = ? and script_published = '0000-00-00'", sceneURL).Update("script_published", time.Now())
if config.Config.Funscripts.ScrapeFunscripts {
siteCollector.OnHTML(`div.postTag`, func(e *colly.HTMLElement) {
sceneURL := ""
e.ForEach(`div.foto a`, func(id int, e *colly.HTMLElement) {
sceneURL = e.Request.AbsoluteURL(e.Attr("href"))
// If scene exist in database, there's no need to scrape
if !funk.ContainsString(knownScenes, sceneURL) {
sceneCollector.Visit(sceneURL)
}
})
e.ForEach(`div.interactive`, func(id int, e *colly.HTMLElement) {
var existingScene models.Scene
existingScene.GetIfExistURL(sceneURL)
if existingScene.ID != 0 && existingScene.ScriptPublished.IsZero() {
var sc models.ScrapedScene
sc.InternalSceneId = existingScene.ID
sc.HasScriptDownload = true
sc.OnlyUpdateScriptData = true
sc.HumanScript = false
sc.AiScript = false
out <- sc
}
})
})
})
}

if singleSceneURL != "" {
sceneCollector.Visit(singleSceneURL)
Expand Down
65 changes: 57 additions & 8 deletions pkg/scrape/slrstudios.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"strconv"
"strings"
"sync"
"time"

"github.com/go-resty/resty/v2"
"github.com/gocolly/colly/v2"
Expand Down Expand Up @@ -247,10 +246,20 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
}
})
})
sc.HasScriptDownload = false
e.ForEach(`ul.c-meta--scene-specs a[href='/tags/sex-toy-scripts-vr']`, func(id int, e_a *colly.HTMLElement) {
sc.HasScriptDownload = true
})
if config.Config.Funscripts.ScrapeFunscripts {
sc.HasScriptDownload = false
sc.AiScript = false
sc.HumanScript = false
e.ForEach(`ul.c-meta--scene-specs a[href='/tags/sex-toy-scripts-vr']`, func(id int, e_a *colly.HTMLElement) {
sc.HasScriptDownload = true
sc.HumanScript = true
})
e.ForEach(`ul.c-meta--scene-specs a[href='/tags/sex-toy-scripts-ai-vr']`, func(id int, e_a *colly.HTMLElement) {
sc.HasScriptDownload = true
sc.AiScript = true
})
}

out <- sc
})

Expand All @@ -266,9 +275,49 @@ func SexLikeReal(wg *sync.WaitGroup, updateSite bool, knownScenes []string, out
isTransScene := strings.Contains(sceneURL, "/trans")

if isStraightScene || isTransScene {
e.ForEach(`div.c-grid-badge--fleshlight-badge`, func(id int, e_a *colly.HTMLElement) {
db.Model(&models.Scene{}).Where("scene_url = ? and script_published = '0000-00-00'", sceneURL).Update("script_published", time.Now())
})

if config.Config.Funscripts.ScrapeFunscripts {
var existingScene models.Scene
existingScene.GetIfExistURL(sceneURL)
if existingScene.ID != 0 {
fleshlightBadge := false
aiBadge := false
multiBadge := false

human := false
ai := false
e.ForEach(`div.c-grid-badge--fleshlight`, func(id int, e_a *colly.HTMLElement) {
fleshlightBadge = true
})
e.ForEach(`div.c-grid-badge--script-ai`, func(id int, e_a *colly.HTMLElement) {
ai = true
})
e.ForEach(`div.c-grid-badge--fleshlight-badge-multi`, func(id int, e_a *colly.HTMLElement) {
multiBadge = true
})

if fleshlightBadge {
human = true
}
if aiBadge {
ai = true
}
if multiBadge && !fleshlightBadge {
human = true
// ai = true
}

if existingScene.HumanScript != human || existingScene.AiScript != ai {
var sc models.ScrapedScene
sc.InternalSceneId = existingScene.ID
sc.HasScriptDownload = true
sc.OnlyUpdateScriptData = true
sc.HumanScript = human
sc.AiScript = ai
out <- sc
}
}
}

// If scene exist in database, there's no need to scrape
if !funk.ContainsString(knownScenes, sceneURL) {
Expand Down
8 changes: 7 additions & 1 deletion pkg/tasks/content.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,13 @@ func sceneDBWriter(wg *sync.WaitGroup, i *uint64, scenes <-chan models.ScrapedSc
if os.Getenv("DEBUG") != "" {
log.Printf("Saving %v", scene.SceneID)
}
models.SceneCreateUpdateFromExternal(db, scene)
if scene.OnlyUpdateScriptData {
if config.Config.Funscripts.ScrapeFunscripts {
models.SceneUpdateScriptData(db, scene)
}
} else {
models.SceneCreateUpdateFromExternal(db, scene)
}
atomic.AddUint64(i, 1)
if os.Getenv("DEBUG") != "" {
log.Printf("Saved %v", scene.SceneID)
Expand Down
Loading

0 comments on commit 450ee00

Please sign in to comment.