Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update solr files from 6.0.0 to 8.11 #450

Merged
merged 2 commits into from
Jun 27, 2022
Merged

Update solr files from 6.0.0 to 8.11 #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b2f78d2
new: update solr files from 6.0.0 to 8.11
nickumia-reisys Jun 27, 2022
44696c3
fix: don't reference deleted files
nickumia-reisys Jun 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 0 additions & 67 deletions solr/currency.xml
View file
Open in desktop

This file was deleted.

42 changes: 0 additions & 42 deletions solr/elevate.xml
View file
Open in desktop

This file was deleted.

7 changes: 7 additions & 0 deletions solr/local_setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash

# Setup ckan core config
/app/solr/solr_setup.sh

# Start solr
su -c "init-var-solr; precreate-core ckan /tmp/ckan_config; chown -R 8983:8983 /var/solr/data; solr-fg" -m solr
2 changes: 1 addition & 1 deletion solr/migrate-solrcloud-schema.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ if ! (curl --get --fail --location-trusted --user $CKAN_SOLR_USER:$CKAN_SOLR_PA

# Zip solr configSet
cd solr && zip ckan_2.9_solr_config.zip \
currency.xml elevate.xml protwords.txt schema.xml solrconfig.xml stopwords.txt synonyms.txt
protwords.txt schema.xml solrconfig.xml stopwords.txt synonyms.txt

echo "Uploading config set..."
curl --fail --location-trusted --user $CKAN_SOLR_USER:$CKAN_SOLR_PASSWORD \
Expand Down
42 changes: 21 additions & 21 deletions solr/protwords.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#-----------------------------------------------------------------------
# Use a protected word file to protect against the stemmer reducing two
# unrelated words to the same base word.

# Some non-words that normally won't be encountered,
# just to test that they won't be stemmed.
dontstems
zwhacky

# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#-----------------------------------------------------------------------
# Use a protected word file to protect against the stemmer reducing two
# unrelated words to the same base word.
# Some non-words that normally won't be encountered,
# just to test that they won't be stemmed.
dontstems
zwhacky
10 changes: 7 additions & 3 deletions solr/schema.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down Expand Up @@ -155,11 +157,13 @@ schema. In this case the version should be set to the next CKAN version number.

<field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>

<!-- Required by ckanext-spatial for geospatial searching -->
<field name="bbox_area" type="float" indexed="true" stored="true" />
<field name="maxx" type="float" indexed="true" stored="true" />
<field name="maxy" type="float" indexed="true" stored="true" />
<field name="minx" type="float" indexed="true" stored="true" />
<field name="miny" type="float" indexed="true" stored="true" />
<!-- Required by ckanext-spatial for geospatial searching -->

<!-- Copy the title field into titleString, and treat as a string
(rather than text type). This allows us to sort on the titleString -->
Expand All @@ -179,8 +183,8 @@ schema. In this case the version should be set to the next CKAN version number.
</fields>

<uniqueKey>index_id</uniqueKey>
<defaultSearchField>text</defaultSearchField>
<solrQueryParser defaultOperator="AND"/>
<df>text</df>
<solrQueryParser q.op="AND"/>

<copyField source="url" dest="urls"/>
<copyField source="title" dest="title_ngram"/>
Expand All @@ -205,4 +209,4 @@ schema. In this case the version should be set to the next CKAN version number.
<copyField source="maintainer" dest="text"/>
<copyField source="author" dest="text"/>

</schema>
</schema>
42 changes: 42 additions & 0 deletions solr/solr_setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/bin/bash

mkdir -p /tmp/ckan_config

# Remove any restored EFS backups
rm -rf /var/solr/data/aws-backup-restore*
# TODO: replace existing (bad) core with the correct (good) restore directory

# Check if users already exist
SECURITY_FILE=/var/solr/data/security.json
if [ -f "$SECURITY_FILE" ]; then
echo "Solr ckan and authentication are set up already :)"
exit 0;
fi

# add solr authentication
cat <<SOLRAUTH > $SECURITY_FILE
{
"authentication":{
"blockUnknown": true,
"class":"solr.BasicAuthPlugin",
"credentials":{"catalog":"rJzrn+HooKn79Q+cfysdGKmMhJbtj0Q1bTokFud6f9o= eKuBUjAoBIkJAMYZxJU6HOKSchTAce+DoQrY5Vewu7I="},
"realm":"data.gov users",
"forwardCredentials": false
},
"authorization":{
"class":"solr.RuleBasedAuthorizationPlugin",
"permissions":[{"name":"security-edit",
"role":"admin"}],
"user-role":{"catalog":"admin"}
}}
SOLRAUTH

# add solr config files for ckan 2.9
wget -O /tmp/ckan_config/schema.xml https://raw.githubusercontent.com/GSA/inventory-app/main/solr/schema.xml
wget -O /tmp/ckan_config/protwords.txt https://raw.githubusercontent.com/GSA/inventory-app/main/solr/protwords.txt
wget -O /tmp/ckan_config/solrconfig.xml https://raw.githubusercontent.com/GSA/inventory-app/main/solr/solrconfig.xml
wget -O /tmp/ckan_config/stopwords.txt https://raw.githubusercontent.com/GSA/inventory-app/main/solr/stopwords.txt
wget -O /tmp/ckan_config/synonyms.txt https://raw.githubusercontent.com/GSA/inventory-app/main/solr/synonyms.txt

# group user solr:solr is 8983:8983 in solr docker image
chown -R 8983:8983 /var/solr/data/
Loading