Skip to content

Commit

Permalink
Extend update script with version selector
Browse files Browse the repository at this point in the history
  • Loading branch information
x-way committed Dec 17, 2023
1 parent 74b1e63 commit 7b32237
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions update_data.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
#!/bin/bash

#VERSION="97e9fe30219e60092e107651abb379a38b342921"
#VERSION="v1.2.116"
VERSION="master"

datafile(){
url="$1"
name="$2"
Expand All @@ -19,9 +23,9 @@ END{
' > "${name}.go"
}
echo "Updating crawlers.go"
datafile "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt" "crawlers"
datafile "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/${VERSION}/raw/Crawlers.txt" "crawlers"
echo "Updating exclusions.go"
datafile "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Exclusions.txt" "exclusions"
datafile "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/${VERSION}/raw/Exclusions.txt" "exclusions"

echo "Patching files for re2 engine"
# XXX: golang re2 based regex engine does not support negative lookahead Yandex(?!Search)
Expand All @@ -31,8 +35,8 @@ sed -i "" -e '/^}$/i\
`YandexSearch`,' exclusions.go

echo "Updating testdata/crawlers.txt"
curl --progress-bar -o testdata/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/crawlers.txt
curl --progress-bar -o testdata/crawlers.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/${VERSION}/tests/crawlers.txt
echo "Updating testdata/devices.txt"
curl --progress-bar -o testdata/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/tests/devices.txt
curl --progress-bar -o testdata/devices.txt https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/${VERSION}/tests/devices.txt

echo "Updating completed"

0 comments on commit 7b32237

Please sign in to comment.