Skip to content

Commit

Permalink
Merge pull request #744 from lehigh-university-libraries/585-black-li…
Browse files Browse the repository at this point in the history
…nter

Use black linter
  • Loading branch information
mjordan authored Feb 22, 2024
2 parents 2b686d5 + ef5f8f5 commit 6c50ccf
Show file tree
Hide file tree
Showing 25 changed files with 13,347 additions and 8,897 deletions.
1 change: 0 additions & 1 deletion .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
## Checklist

* [ ] Before opening this PR, have you opened an issue explaining what you want to to do?
* [ ] Have you run `pycodestyle --show-source --show-pep8 --ignore=E402,W504 --max-line-length=200 yourfile.py`?
* [ ] Have you included some configuration and/or CSV files useful for testing this PR?
* [ ] Have you written unit or integration tests if applicable?
* [ ] Does the code added in this PR require a version of Python that is higher than the current minimum version?
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/black.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: Lint

on: [push, pull_request]

jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: psf/black@stable
with:
options: "--check --verbose"
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ Metadata, files, and Drupal configurations are, in the real world, extremly comp

Using Workbench and reporting problems is the best way you can help make it better!

### Linting

All code must be formatted using [black](https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html)

You can automatically style your code [using black in your IDE of choice](https://black.readthedocs.io/en/stable/integrations/editors.html).

### Documentation and code

* If you have a suggestion for improving the documentation, please open an issue on [this repository's queue](https://github.com/mjordan/islandora_workbench/issues) and tag your issue "documentation".
Expand Down
346 changes: 189 additions & 157 deletions WorkbenchConfig.py

Large diffs are not rendered by default.

77 changes: 42 additions & 35 deletions i7Import/get_islandora_7_content.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/usr/bin/env python3

'''Script for exporting Islandora 7 content (metadata and OBJ datastreams). See
"""Script for exporting Islandora 7 content (metadata and OBJ datastreams). See
https://mjordan.github.io/islandora_workbench_docs/exporting_islandora_7_content/
for more info.
'''
"""

import os
import sys
Expand All @@ -20,8 +20,10 @@
############################

parser = argparse.ArgumentParser()
parser.add_argument('--config', required=True, help='Configuration file to use.')
parser.add_argument('--metadata_solr_request', required=False, help='Option to solr metadata request.')
parser.add_argument("--config", required=True, help="Configuration file to use.")
parser.add_argument(
"--metadata_solr_request", required=False, help="Option to solr metadata request."
)
args = parser.parse_args()
utils = i7ImportUtilities(args.config)
config = utils.config
Expand All @@ -31,85 +33,90 @@
#######################

logging.basicConfig(
filename=config['log_file_path'],
filename=config["log_file_path"],
level=logging.INFO,
filemode='a',
format='%(asctime)s - %(levelname)s - %(message)s',
datefmt='%d-%b-%y %H:%M:%S')
filemode="a",
format="%(asctime)s - %(levelname)s - %(message)s",
datefmt="%d-%b-%y %H:%M:%S",
)

if args.metadata_solr_request:
metadata_solr_request = utils.get_metadata_solr_request(args.metadata_solr_request)
else:
metadata_solr_request = utils.get_default_metadata_solr_request()
if config['debug']:
pretty_print = metadata_solr_request.replace('&', "\n&")
if config["debug"]:
pretty_print = metadata_solr_request.replace("&", "\n&")
print(f"Solr request: {pretty_print}")
utils.print_config()

try:
metadata_solr_response = requests.get(url=metadata_solr_request, allow_redirects=True)
metadata_solr_response = requests.get(
url=metadata_solr_request, allow_redirects=True
)
except requests.exceptions.RequestException as e:
logging.info("Solr Query failed.")
raise SystemExit(e)
if not metadata_solr_response.ok:
warning = ''
warning = ""
if len(metadata_solr_request) > 2000:
warning = 'The default query may be too long for a url request. See docs'
print(f"Illegal request: Server returned status of {metadata_solr_response.status_code} \n{warning} ")
warning = "The default query may be too long for a url request. See docs"
print(
f"Illegal request: Server returned status of {metadata_solr_response.status_code} \n{warning} "
)
sys.exit()
rows = metadata_solr_response.content.decode().splitlines()
logging.info(f"Processing {len(rows)} items.")
reader = csv.DictReader(rows)
headers = reader.fieldnames
# We add a 'sequence' column to store the Islandora 7.x property "isSequenceNumberOfxxx"/"isSequenceNumber".
headers.append('sequence')
headers.append("sequence")
# Add a column to store the files
headers.append('file')
if config['id_field'] not in headers:
headers.append(config['id_field'])
index = config['id_start_number']
headers.append("file")
if config["id_field"] not in headers:
headers.append(config["id_field"])
index = config["id_start_number"]

if config['fetch_files'] is True:
if not os.path.exists(config['obj_directory']):
os.makedirs(config['obj_directory'])
if config["fetch_files"] is True:
if not os.path.exists(config["obj_directory"]):
os.makedirs(config["obj_directory"])

row_count = 0
pbar = InitBar()
num_csv_rows = len(rows)
print(f"Processing {num_csv_rows -1}.")
with open(config['csv_output_path'], 'w', newline='') as csvfile:
with open(config["csv_output_path"], "w", newline="") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=headers)
writer.writeheader()
failed_pids = []
for row in reader:
rels_ext = utils.parse_rels_ext(row['PID'])
rels_ext = utils.parse_rels_ext(row["PID"])
if rels_ext:
for key, value in rels_ext.items():
if 'isSequenceNumber' in key:
row['sequence'] = str(value)
if "isSequenceNumber" in key:
row["sequence"] = str(value)
else:
failed_pids.append(row['PID'])
failed_pids.append(row["PID"])
logging.error(f"{row['PID']} was unsuccessful.")
continue
if config['fetch_files'] or config['get_file_url']:
if config["fetch_files"] or config["get_file_url"]:
row_count += 1
row_position = utils.get_percentage(row_count, num_csv_rows)
pbar(row_position)
for datastream in config['datastreams']:
file = utils.get_i7_asset(row['PID'], datastream)
for datastream in config["datastreams"]:
file = utils.get_i7_asset(row["PID"], datastream)
if file:
row['file'] = file
row["file"] = file
break

if config['id_field'] in headers:
row[config['id_field']] = index + reader.line_num - 2
if config["id_field"] in headers:
row[config["id_field"]] = index + reader.line_num - 2
writer.writerow(row)
if failed_pids:
output = 'The following PIDS returned no data:\n'
output = "The following PIDS returned no data:\n"
for pid in failed_pids:
output += f"{pid}\n"
print(output)
if config['debug']:
if config["debug"]:
with open("failure_report.txt", "w") as f:
f.write(output)
pbar(100)
Loading

0 comments on commit 6c50ccf

Please sign in to comment.