Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use black linter #744

Merged
merged 4 commits into from
Feb 22, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
## Checklist

* [ ] Before opening this PR, have you opened an issue explaining what you want to to do?
* [ ] Have you run `pycodestyle --show-source --show-pep8 --ignore=E402,W504 --max-line-length=200 yourfile.py`?
* [ ] Have you included some configuration and/or CSV files useful for testing this PR?
* [ ] Have you written unit or integration tests if applicable?
* [ ] Does the code added in this PR require a version of Python that is higher than the current minimum version?
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/black.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: Lint

on: [push, pull_request]

jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: psf/black@stable
with:
options: "--check --verbose"
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ Metadata, files, and Drupal configurations are, in the real world, extremly comp

Using Workbench and reporting problems is the best way you can help make it better!

### Linting

All code must be formatted using [black](https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html)

You can automatically style your code [using black in your IDE of choice](https://black.readthedocs.io/en/stable/integrations/editors.html).

### Documentation and code

* If you have a suggestion for improving the documentation, please open an issue on [this repository's queue](https://github.com/mjordan/islandora_workbench/issues) and tag your issue "documentation".
Expand Down
346 changes: 189 additions & 157 deletions WorkbenchConfig.py

Large diffs are not rendered by default.

77 changes: 42 additions & 35 deletions i7Import/get_islandora_7_content.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/usr/bin/env python3

'''Script for exporting Islandora 7 content (metadata and OBJ datastreams). See
"""Script for exporting Islandora 7 content (metadata and OBJ datastreams). See
https://mjordan.github.io/islandora_workbench_docs/exporting_islandora_7_content/
for more info.
'''
"""

import os
import sys
Expand All @@ -20,8 +20,10 @@
############################

parser = argparse.ArgumentParser()
parser.add_argument('--config', required=True, help='Configuration file to use.')
parser.add_argument('--metadata_solr_request', required=False, help='Option to solr metadata request.')
parser.add_argument("--config", required=True, help="Configuration file to use.")
parser.add_argument(
"--metadata_solr_request", required=False, help="Option to solr metadata request."
)
args = parser.parse_args()
utils = i7ImportUtilities(args.config)
config = utils.config
Expand All @@ -31,85 +33,90 @@
#######################

logging.basicConfig(
filename=config['log_file_path'],
filename=config["log_file_path"],
level=logging.INFO,
filemode='a',
format='%(asctime)s - %(levelname)s - %(message)s',
datefmt='%d-%b-%y %H:%M:%S')
filemode="a",
format="%(asctime)s - %(levelname)s - %(message)s",
datefmt="%d-%b-%y %H:%M:%S",
)

if args.metadata_solr_request:
metadata_solr_request = utils.get_metadata_solr_request(args.metadata_solr_request)
else:
metadata_solr_request = utils.get_default_metadata_solr_request()
if config['debug']:
pretty_print = metadata_solr_request.replace('&', "\n&")
if config["debug"]:
pretty_print = metadata_solr_request.replace("&", "\n&")
print(f"Solr request: {pretty_print}")
utils.print_config()

try:
metadata_solr_response = requests.get(url=metadata_solr_request, allow_redirects=True)
metadata_solr_response = requests.get(
url=metadata_solr_request, allow_redirects=True
)
except requests.exceptions.RequestException as e:
logging.info("Solr Query failed.")
raise SystemExit(e)
if not metadata_solr_response.ok:
warning = ''
warning = ""
if len(metadata_solr_request) > 2000:
warning = 'The default query may be too long for a url request. See docs'
print(f"Illegal request: Server returned status of {metadata_solr_response.status_code} \n{warning} ")
warning = "The default query may be too long for a url request. See docs"
print(
f"Illegal request: Server returned status of {metadata_solr_response.status_code} \n{warning} "
)
sys.exit()
rows = metadata_solr_response.content.decode().splitlines()
logging.info(f"Processing {len(rows)} items.")
reader = csv.DictReader(rows)
headers = reader.fieldnames
# We add a 'sequence' column to store the Islandora 7.x property "isSequenceNumberOfxxx"/"isSequenceNumber".
headers.append('sequence')
headers.append("sequence")
# Add a column to store the files
headers.append('file')
if config['id_field'] not in headers:
headers.append(config['id_field'])
index = config['id_start_number']
headers.append("file")
if config["id_field"] not in headers:
headers.append(config["id_field"])
index = config["id_start_number"]

if config['fetch_files'] is True:
if not os.path.exists(config['obj_directory']):
os.makedirs(config['obj_directory'])
if config["fetch_files"] is True:
if not os.path.exists(config["obj_directory"]):
os.makedirs(config["obj_directory"])

row_count = 0
pbar = InitBar()
num_csv_rows = len(rows)
print(f"Processing {num_csv_rows -1}.")
with open(config['csv_output_path'], 'w', newline='') as csvfile:
with open(config["csv_output_path"], "w", newline="") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=headers)
writer.writeheader()
failed_pids = []
for row in reader:
rels_ext = utils.parse_rels_ext(row['PID'])
rels_ext = utils.parse_rels_ext(row["PID"])
if rels_ext:
for key, value in rels_ext.items():
if 'isSequenceNumber' in key:
row['sequence'] = str(value)
if "isSequenceNumber" in key:
row["sequence"] = str(value)
else:
failed_pids.append(row['PID'])
failed_pids.append(row["PID"])
logging.error(f"{row['PID']} was unsuccessful.")
continue
if config['fetch_files'] or config['get_file_url']:
if config["fetch_files"] or config["get_file_url"]:
row_count += 1
row_position = utils.get_percentage(row_count, num_csv_rows)
pbar(row_position)
for datastream in config['datastreams']:
file = utils.get_i7_asset(row['PID'], datastream)
for datastream in config["datastreams"]:
file = utils.get_i7_asset(row["PID"], datastream)
if file:
row['file'] = file
row["file"] = file
break

if config['id_field'] in headers:
row[config['id_field']] = index + reader.line_num - 2
if config["id_field"] in headers:
row[config["id_field"]] = index + reader.line_num - 2
writer.writerow(row)
if failed_pids:
output = 'The following PIDS returned no data:\n'
output = "The following PIDS returned no data:\n"
for pid in failed_pids:
output += f"{pid}\n"
print(output)
if config['debug']:
if config["debug"]:
with open("failure_report.txt", "w") as f:
f.write(output)
pbar(100)
Loading