Skip to content

Commit

Permalink
refactoring of outputs
Browse files Browse the repository at this point in the history
  • Loading branch information
@deusebio
deusebio committed Jul 4, 2023
1 parent 858b641 commit be9832b
Show file tree
Hide file tree
Showing 11 changed files with 263 additions and 165 deletions.
155 changes: 79 additions & 76 deletions .github/workflows/e2e_test.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ jobs:
\"api_key\":\"${{ secrets.DISCOURSE_API_KEY }}\" \
}" >> $GITHUB_OUTPUT
- uses: actions/checkout@v3
- name: Setup Python
run: |
sudo apt update && sudo apt install python3-pip git
pip3 install -r requirements.txt
- name: Prepare for action to run
run: |
# Create metadata.yaml file
Expand Down Expand Up @@ -63,8 +67,6 @@ jobs:
run: echo '${{ steps.e2e-test-draft.outputs.reconcile }}'
- name: Check draft
run: |
sudo apt update && sudo apt install python3-pip git
pip3 install -r requirements.txt
PYTHONPATH=$(pwd) python3 prepare_check_cleanup/reconcile.py --action check-draft --action-kwargs '{"expected_url_results": []}' '${{ steps.e2e-test-draft.outputs.reconcile }}' '${{ steps.configuration.outputs.discourse }}'
- name: Create self test
id: e2e-test-create
Expand Down Expand Up @@ -135,77 +137,78 @@ jobs:
GITHUB_TOKEN='${{ secrets.GITHUB_TOKEN }}'
REPO='${{ github.repository }}'
PYTHONPATH=$(pwd) python3 prepare_check_cleanup/reconcile.py --action cleanup --action-kwargs "{\"github_token\": \"$GITHUB_TOKEN\", \"repo\": \"$REPO\"}" '${{ steps.e2e-test-create.outputs.reconcile }}' '${{ steps.configuration.outputs.discourse }}'
e2e-tests-migration:
permissions: write-all
runs-on: ubuntu-22.04
steps:
# Each job has to have this configuration because secrets can be passed through the output of
# another job
- name: Generate discourse configuration
id: configuration
run: |
echo "discourse= \
{ \
\"hostname\":\"discourse.charmhub.io\", \
\"category_id\":\"41\", \
\"api_username\":\"${{ secrets.DISCOURSE_API_USERNAME }}\", \
\"api_key\":\"${{ secrets.DISCOURSE_API_KEY }}\" \
}" >> $GITHUB_OUTPUT
- uses: actions/checkout@v3
- name: Prepare discourse topics
id: prepare-discourse
run: |
# Create topics on discourse
MIGRATION_DIR=migration_docs
mkdir $MIGRATION_DIR
echo "migration_docs" >> .gitignore
# need some values in the content that change on every commit as discourse looks for file
# similarity
INDEX_FILENAME=$MIGRATION_DIR/index.md
echo -e "# Charm Upload Automation Migration Test Index Title some uuid: $(uuidgen)\n" \
"placeholder index content for testing, " \
"repository: ${{ github.repository }}, " \
"branch: ${{ github.head_ref }}, " \
"commit sha: ${{ github.sha }}, " \
"some uuid: $(uuidgen)" \
> $INDEX_FILENAME
PAGE_FILENAME=$MIGRATION_DIR/page.md
echo -e "# Charm Upload Automation Migration Test Documentation Title some uuid: $(uuidgen)\n" \
"placeholder documentation content for testing, " \
"repository: ${{ github.repository }}, " \
"branch: ${{ github.head_ref }}, " \
"commit sha: ${{ github.sha }}, " \
"some uuid: $(uuidgen)" \
> $PAGE_FILENAME
# Create discourse topics
sudo apt update && sudo apt install python3-pip git
pip3 install -r requirements.txt
DISCOURSE_CONFIG='${{ steps.configuration.outputs.discourse }}'
PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action prepare --action-kwargs "{\"index_filename\": \"$INDEX_FILENAME\", \"page_filename\": \"$PAGE_FILENAME\", \"discourse_config\": $DISCOURSE_CONFIG}"
- name: Show created topics
run: echo '${{ steps.prepare-discourse.outputs.topics }}'
- name: Create metadata.yaml
run: |
# Create metadata.yaml file
echo "name: $(echo ${{ github.repository }} | sed 's:.*/::')-test" > metadata.yaml
echo "docs: ${{ steps.prepare-discourse.outputs.index_url }}" >> metadata.yaml
cat metadata.yaml
- name: Run migration
id: run-migration
uses: ./
with:
discourse_host: discourse.charmhub.io
discourse_api_username: ${{ secrets.DISCOURSE_API_USERNAME }}
discourse_api_key: ${{ secrets.DISCOURSE_API_KEY }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Check migration branch got created
run: |
PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action check-branch --action-kwargs '{"github_access_token": "${{ secrets.GITHUB_TOKEN }}"}'
- name: Check migration pull request got created
run: |
PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action check-pull-request --action-kwargs '{"github_access_token": "${{ secrets.GITHUB_TOKEN }}"}'
- name: Clean up
if: always()
run: |
DISCOURSE_CONFIG='${{ steps.configuration.outputs.discourse }}'
PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action cleanup --action-kwargs "{\"topics\": ${{ steps.prepare-discourse.outputs.topics }}, \"github_access_token\": \"${{ secrets.GITHUB_TOKEN }}\", \"discourse_config\": $DISCOURSE_CONFIG}"
# e2e-tests-migration:
# permissions: write-all
# runs-on: ubuntu-22.04
# steps:
# # Each job has to have this configuration because secrets can be passed through the output of
# # another job
# - name: Generate discourse configuration
# id: configuration
# run: |
# echo "discourse= \
# { \
# \"hostname\":\"discourse.charmhub.io\", \
# \"category_id\":\"41\", \
# \"api_username\":\"${{ secrets.DISCOURSE_API_USERNAME }}\", \
# \"api_key\":\"${{ secrets.DISCOURSE_API_KEY }}\" \
# }" >> $GITHUB_OUTPUT
# - uses: actions/checkout@v3
# - name: Prepare discourse topics
# id: prepare-discourse
# run: |
# # Create topics on discourse
# MIGRATION_DIR=migration_docs
# mkdir $MIGRATION_DIR
# echo "migration_docs" >> .gitignore
# # need some values in the content that change on every commit as discourse looks for file
# # similarity
# INDEX_FILENAME=$MIGRATION_DIR/index.md
# echo -e "# Charm Upload Automation Migration Test Index Title some uuid: $(uuidgen)\n" \
# "placeholder index content for testing, " \
# "repository: ${{ github.repository }}, " \
# "branch: ${{ github.head_ref }}, " \
# "commit sha: ${{ github.sha }}, " \
# "some uuid: $(uuidgen)" \
# > $INDEX_FILENAME
# PAGE_FILENAME=$MIGRATION_DIR/page.md
# echo -e "# Charm Upload Automation Migration Test Documentation Title some uuid: $(uuidgen)\n" \
# "placeholder documentation content for testing, " \
# "repository: ${{ github.repository }}, " \
# "branch: ${{ github.head_ref }}, " \
# "commit sha: ${{ github.sha }}, " \
# "some uuid: $(uuidgen)" \
# > $PAGE_FILENAME
# # Create discourse topics
# sudo apt update && sudo apt install python3-pip git
# pip3 install -r requirements.txt
# DISCOURSE_CONFIG='${{ steps.configuration.outputs.discourse }}'
# PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action prepare --action-kwargs "{\"index_filename\": \"$INDEX_FILENAME\", \"page_filename\": \"$PAGE_FILENAME\", \"discourse_config\": $DISCOURSE_CONFIG}"
# - name: Show created topics
# run: echo '${{ steps.prepare-discourse.outputs.topics }}'
# - name: Create metadata.yaml
# run: |
# # Create metadata.yaml file
# echo "name: $(echo ${{ github.repository }} | sed 's:.*/::')-test" > metadata.yaml
# echo "docs: ${{ steps.prepare-discourse.outputs.index_url }}" >> metadata.yaml
# cat metadata.yaml
# - name: Run migration
# id: run-migration
# uses: ./
# with:
# discourse_host: discourse.charmhub.io
# discourse_api_username: ${{ secrets.DISCOURSE_API_USERNAME }}
# discourse_api_key: ${{ secrets.DISCOURSE_API_KEY }}
# github_token: ${{ secrets.GITHUB_TOKEN }}
# - name: Check migration branch got created
# run: |
# PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action check-branch --action-kwargs '{"github_access_token": "${{ secrets.GITHUB_TOKEN }}"}'
# - name: Check migration pull request got created
# run: |
# PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action check-pull-request --action-kwargs '{"github_access_token": "${{ secrets.GITHUB_TOKEN }}"}'
# - name: Clean up
# if: always()
# run: |
# DISCOURSE_CONFIG='${{ steps.configuration.outputs.discourse }}'
# PYTHONPATH=$(pwd) python3 prepare_check_cleanup/migration.py --action cleanup --action-kwargs "{\"topics\": ${{ steps.prepare-discourse.outputs.topics }}, \"github_access_token\": \"${{ secrets.GITHUB_TOKEN }}\", \"discourse_config\": $DISCOURSE_CONFIG}"
22 changes: 17 additions & 5 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ def _serialize_for_github(urls_with_actions_dict: dict[str, str]) -> str:


def _write_github_output(
**urls_with_actions_dicts: dict[str, str],
migrate: types_.MigrateOutputs | None,
reconcile: types_.ReconcileOutputs | None
) -> None:
"""Writes results produced by the action to github_output.
Expand All @@ -104,9 +105,20 @@ def _write_github_output(
f"{GETTING_STARTED}"
)

output_dict = (
{
"index_url": reconcile.index_url,
"topics": reconcile.topics
} if reconcile else {}
) | (
{
"pr_action": migrate.action,
"pr_link": migrate.pull_request_url
} if migrate else {}
)

output: str = "".join(
f"{key}={_serialize_for_github(urls_with_actions_dict)}\n"
for key, urls_with_actions_dict in urls_with_actions_dicts.items()
f"{key}={_serialize_for_github(value)}" for key, value in output_dict.items()
)

logging.info("Output: %s", output)
Expand Down Expand Up @@ -156,7 +168,7 @@ def wrapper(*args: typing.Any, **kwargs: typing.Any) -> T:


@execute_in_tmpdir
def main_migrate(path: Path, user_inputs: types_.UserInputs) -> dict:
def main_migrate(path: Path, user_inputs: types_.UserInputs) -> types_.MigrateOutputs | None:
"""Main to migrate content from Discourse to Git repository.
Args:
Expand All @@ -171,7 +183,7 @@ def main_migrate(path: Path, user_inputs: types_.UserInputs) -> dict:


@execute_in_tmpdir
def main_reconcile(path: Path, user_inputs: types_.UserInputs) -> dict:
def main_reconcile(path: Path, user_inputs: types_.UserInputs) -> types_.ReconcileOutputs | None:
"""Main to reconcile content from Git repository to Discourse.
Args:
Expand Down
56 changes: 34 additions & 22 deletions src/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,25 @@
from .navigation_table import from_page as navigation_table_from_page
from .reconcile import run as get_reconcile_actions
from .repository import DEFAULT_BRANCH_NAME
from .types_ import ActionResult, UserInputs
from .types_ import (
ActionResult, UserInputs, PullRequestAction, ReconcileOutputs, MigrateOutputs, Url
)

GETTING_STARTED = (
"To get started with upload-charm-docs, "
"please refer to https://github.com/canonical/upload-charm-docs#getting-started"
)


def run_reconcile(clients: Clients, user_inputs: UserInputs) -> dict[str, str]:
def run_reconcile(clients: Clients, user_inputs: UserInputs) -> ReconcileOutputs | None:
"""Upload the documentation to charmhub.
Args:
clients: The clients to interact with things like discourse and the repository.
user_inputs: Configurable inputs for running upload-charm-docs.
Returns:
All the URLs that had an action with the result of that action.
ReconcileOutputs object with the result of the action. None, if there is no reconcile.
Raises:
InputError: if there are any problems with executing any of the actions.
Expand All @@ -45,14 +47,14 @@ def run_reconcile(clients: Clients, user_inputs: UserInputs) -> dict[str, str]:
"present in the repository"
)

return {}
return None

if clients.repository.is_same_commit(DOCUMENTATION_TAG, user_inputs.commit_sha):
logging.warning(
"Cannot run any reconcile to Discourse as we are at the same commit of the tag %s",
DOCUMENTATION_TAG,
)
return {}
return None

metadata = clients.repository.metadata
base_path = clients.repository.base_path
Expand Down Expand Up @@ -83,46 +85,50 @@ def run_reconcile(clients: Clients, user_inputs: UserInputs) -> dict[str, str]:
"One or more of the required actions could not be executed, see the log for details"
)

reports = run_all_actions(
index_url, reports = run_all_actions(
actions=actions,
index=index,
discourse=clients.discourse,
dry_run=user_inputs.dry_run,
delete_pages=user_inputs.delete_pages,
)
urls_with_actions: dict[str, str] = {
urls_with_actions: dict[Url, ActionResult] = {
str(report.location): report.result
for report in reports
if report.location is not None
and report.location != DRY_RUN_NAVLINK_LINK
and report.location != FAIL_NAVLINK_LINK
and report.location != DRY_RUN_NAVLINK_LINK
and report.location != FAIL_NAVLINK_LINK
}

if not user_inputs.dry_run:
clients.repository.tag_commit(
tag_name=DOCUMENTATION_TAG, commit_sha=user_inputs.commit_sha
)

return urls_with_actions
return ReconcileOutputs(
index_url=index_url,
topics=urls_with_actions,
documentation_tag=clients.repository.tag_exists(DOCUMENTATION_TAG)
)


def run_migrate(clients: Clients, user_inputs: UserInputs) -> dict[str, str]:
def run_migrate(clients: Clients, user_inputs: UserInputs) -> MigrateOutputs | None:
"""Migrate existing docs from charmhub to local repository.
Args:
clients: The clients to interact with things like discourse and the repository.
user_inputs: Configurable inputs for running upload-charm-docs.
Returns:
A single key-value pair dictionary containing a link to the Pull Request containing
migrated documentation as key and successful action result as value.
MigrateOutputs providing details on the action performed and a link to the
Pull Request containing migrated documentation. None if there is no migration.
"""
if not clients.repository.metadata.docs:
logging.warning(
"Cannot run migration from Discourse as there is no discourse "
"link available in metadata"
)
return {}
return None

logging.info("Tag exists: %s", str(clients.repository.tag_exists(DOCUMENTATION_TAG)))

Expand All @@ -144,18 +150,24 @@ def run_migrate(clients: Clients, user_inputs: UserInputs) -> dict[str, str]:
# Given there are NO diffs compared to the base, if a PR is open, it should be closed
if pull_request is not None:
pull_request.edit(state="closed")
return {}
return MigrateOutputs(
action=PullRequestAction.CLOSED,
pull_request_url=pull_request.html_url
)
return None

if pull_request is not None:
logging.info(
"upload-charm-documents pull request already open at %s", pull_request.html_url
)
clients.repository.update_pull_request(DEFAULT_BRANCH_NAME)
else:
if pull_request is None:
logging.info("PR not existing: creating a new one...")
pull_request = clients.repository.create_pull_request(user_inputs.base_branch)
return MigrateOutputs(
action=PullRequestAction.OPENED,
pull_request_url=pull_request.html_url
)

logging.info("upload-charm-documents pull request already open at %s", pull_request.html_url)
clients.repository.update_pull_request(DEFAULT_BRANCH_NAME)

return {pull_request.html_url: ActionResult.SUCCESS}
return MigrateOutputs(action=PullRequestAction.UPDATED, pull_request_url=pull_request.html_url)


def pre_flight_checks(clients: Clients, user_inputs: UserInputs) -> bool:
Expand Down
6 changes: 3 additions & 3 deletions src/action.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ def run_all(
discourse: Discourse,
dry_run: bool,
delete_pages: bool,
) -> list[types_.ActionReport]:
) -> tuple[str, list[types_.ActionReport]]:
"""Take the actions against the server.
Args:
Expand All @@ -430,7 +430,7 @@ def run_all(
delete_pages: Whether to delete pages that are no longer needed.
Returns:
The reports of taking all the requested action and the index action report.
A 2-element tuple with the index url and the reports of all the requested action.
"""
action_reports = [
_run_one(
Expand All @@ -446,4 +446,4 @@ def run_all(
index_action = reconcile.index_page(index=index, table_rows=table_rows)
index_action_report = _run_index(action=index_action, discourse=discourse, dry_run=dry_run)
action_reports.append(index_action_report)
return action_reports
return str(index_action_report.location), action_reports
Loading

0 comments on commit be9832b

Please sign in to comment.