Skip to content

Commit

Permalink
Add "globoff" to curl
Browse files Browse the repository at this point in the history
This allows to use brackets '{}[]' uninterpreted by curl.

See hbz#1048.
  • Loading branch information
dr0i committed Jun 22, 2020
1 parent 34de43c commit 1ccb094
Showing 1 changed file with 3 additions and 3 deletions.
6 changes: 3 additions & 3 deletions scripts/getAllData.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@ HITS=1
FROM_INITIAL="00000101"
TO_INITIAL="19800101"

curl --header "Accept-Encoding: gzip" "http://lobid.org/resources/search?q=describedBy.dateCreated:[$FROM_INITIAL%20TO%20$TO_INITIAL]&format=bulk" > $FROM_INITIAL-$TO_INITIAL.jsonl.gz
curl --globoff --header "Accept-Encoding: gzip" "http://lobid.org/resources/search?q=describedBy.dateCreated:[$FROM_INITIAL%20TO%20$TO_INITIAL]&format=bulk" > $FROM_INITIAL-$TO_INITIAL.jsonl.gz

while [ $HITS != 0 ] ; do
TO=$(expr $FROM + $INC )
HITS=$(curl "http://lobid.org/resources/search?q=describedBy.dateCreated:[$FROM%20TO%20$TO]&size=1"|jq .totalItems)
HITS=$(curl --globoff "http://lobid.org/resources/search?q=describedBy.dateCreated:[$FROM%20TO%20$TO]&size=1"|jq .totalItems)
echo "getting $FROM to $TO with total items: $HITS"
curl --header "Accept-Encoding: gzip" "http://lobid.org/resources/search?q=describedBy.dateCreated:[$FROM%20TO%20$TO]&format=bulk" > $FROM-$TO.jsonl.gz
curl --globoff --header "Accept-Encoding: gzip" "http://lobid.org/resources/search?q=describedBy.dateCreated:[$FROM%20TO%20$TO]&format=bulk" > $FROM-$TO.jsonl.gz
GOT_HITS=$(zcat $FROM-$TO.jsonl.gz |wc -l)
echo "Got hits ($GOT_HITS), expected ($HITS)"
if [ $HITS != $GOT_HITS ]; then
Expand Down

0 comments on commit 1ccb094

Please sign in to comment.