From c6efc3c2c6bf70a3d11d13c5169018bc2daceaa0 Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Wed, 6 Sep 2023 09:30:37 +0200 Subject: [PATCH 1/9] initial version of Dockerfile --- Dockerfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a9dcd57 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +ARG PYTHON_VERSION=3.11-alpine + +FROM python:${PYTHON_VERSION} + +RUN apk add py3-virtualenv py3-pytest python3-dev gcc musl-dev openldap-dev + +WORKDIR /app + +ADD requirements.txt . +ADD requirements-minimal.txt . + +RUN virtualenv .venv +ENV VIRTUAL_ENV /app/.venv +ENV PATH /app/.venv/bin:$PATH + +RUN pip install --upgrade pip +RUN pip install -r requirements.txt From e7a0cd2e3daf1473e2241c66f03d0ef029a2bfc4 Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Mon, 11 Sep 2023 14:09:07 +0200 Subject: [PATCH 2/9] wip --- Dockerfile | 16 +++++++++++----- Makefile | 18 ++++++++++++++++++ tests/base_test.py | 32 ++++++++++++++++++++++++++++---- 3 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 Makefile diff --git a/Dockerfile b/Dockerfile index a9dcd57..aa0090d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,14 +4,20 @@ FROM python:${PYTHON_VERSION} RUN apk add py3-virtualenv py3-pytest python3-dev gcc musl-dev openldap-dev -WORKDIR /app +RUN virtualenv /.venv +ENV VIRTUAL_ENV /.venv +ENV PATH /.venv/bin:$PATH + +RUN adduser -D user + +RUN chown -R user:user /.venv + +USER user ADD requirements.txt . ADD requirements-minimal.txt . -RUN virtualenv .venv -ENV VIRTUAL_ENV /app/.venv -ENV PATH /app/.venv/bin:$PATH - RUN pip install --upgrade pip RUN pip install -r requirements.txt + +WORKDIR /app \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b58fa71 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +# Makefile + +all: pytest + +image: + docker build -t plsc . + +ldap_start: + etc/ldap_start.sh + +ldap_show: + etc/ldap_show.sh + +ldap_stop: + etc/ldap_stop.sh + +pytest: image ldap_start + docker run --rm -ti --network host -v ${PWD}:/app plsc pytest diff --git a/tests/base_test.py b/tests/base_test.py index eae7724..59fa1f4 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -2,8 +2,6 @@ from aiohttp import web -from gera2ld.pyserve import run_forever, start_server_aiohttp - from sldap import SLdap from sbs import SBS @@ -47,6 +45,7 @@ def do_get(request): return web.json_response({}, status=404) +DEFAULT_LOCAL_HOST = '127.0.0.1' DEFAULT_LOCAL_PORT = 3333 @@ -71,16 +70,41 @@ class BaseTest(TestCase): @classmethod def setUpClass(cls): def start_server(loop): + + async def init_web_server(handle, host, port): + server = web.ServerRunner(web.Server(handle)) + await server.setup() + + site = web.TCPSite( + server, + host=host, + port=port + ) + await site.start() + + return server + logger.debug("BaseTest start_server") handle = APIHandler() asyncio.set_event_loop(loop) - run_forever(start_server_aiohttp(handle, ':{}'.format(DEFAULT_LOCAL_PORT))) + + server = init_web_server( + handle, + DEFAULT_LOCAL_HOST, + DEFAULT_LOCAL_PORT + ) + + loop.run_until_complete(server) + loop.run_forever() + def check_server(): logger.debug("BaseTest check_server") sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(1) - while sock.connect_ex(('127.0.0.1', DEFAULT_LOCAL_PORT)) == 111: + while sock.connect_ex( + (DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT) + ) == 111: time.sleep(0.1) if not os.environ.get("SBS_URL", None): From c689af048207887257eb195f096c0e585e7cefd6 Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Mon, 11 Sep 2023 16:18:49 +0200 Subject: [PATCH 3/9] flake fix --- emails.py | 5 ++--- tests/base_test.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/emails.py b/emails.py index 1cc9f7a..dca42c8 100755 --- a/emails.py +++ b/emails.py @@ -120,13 +120,12 @@ def write_xls(contacts: contacts_type, fd: TextIOBase) -> None: workbook = xlsxwriter.Workbook(raw_fd, {'in_memory': True}) worksheet = workbook.add_worksheet() - columns = ("type", "id", "name", "role", "mail") - columns_width = ( 8, 6, 30, 20, 30) + columns = ("type", "id", "name", "role", "mail") + columns_width = (8, 6, 30, 20, 30) row_max = len(contacts) + 1 - 1 col_max = len(columns) - 1 - # write header for col_num, col_name in enumerate(columns): worksheet.write(0, col_num, col_name) diff --git a/tests/base_test.py b/tests/base_test.py index 59fa1f4..d7d0cb7 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -82,8 +82,8 @@ async def init_web_server(handle, host, port): ) await site.start() - return server - + return server + logger.debug("BaseTest start_server") handle = APIHandler() asyncio.set_event_loop(loop) @@ -97,7 +97,6 @@ async def init_web_server(handle, host, port): loop.run_until_complete(server) loop.run_forever() - def check_server(): logger.debug("BaseTest check_server") sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) From 983426284dfdca985f68743506355fc1a37f118e Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Mon, 11 Sep 2023 22:17:45 +0200 Subject: [PATCH 4/9] add .dockerignore --- .dockerignore | 8 ++++++++ Makefile | 2 ++ 2 files changed, 10 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9077b4b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.vscode +.editorignore +.flake* +.git* +.python* +Dockerfile +Makefile +etc \ No newline at end of file diff --git a/Makefile b/Makefile index b58fa71..8bfc979 100644 --- a/Makefile +++ b/Makefile @@ -16,3 +16,5 @@ ldap_stop: pytest: image ldap_start docker run --rm -ti --network host -v ${PWD}:/app plsc pytest + +clean: ldap_stop From a98e1c94e856a3399e1d73a7b11dbc806e9c7a37 Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Mon, 11 Sep 2023 22:25:55 +0200 Subject: [PATCH 5/9] wip --- Dockerfile | 8 ++++---- tests/base_test.py | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index aa0090d..5c70866 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,13 @@ FROM python:${PYTHON_VERSION} RUN apk add py3-virtualenv py3-pytest python3-dev gcc musl-dev openldap-dev -RUN virtualenv /.venv -ENV VIRTUAL_ENV /.venv -ENV PATH /.venv/bin:$PATH +ENV VENV /.venv +RUN virtualenv ${VENV} +ENV PATH ${VENV}/bin:$PATH RUN adduser -D user -RUN chown -R user:user /.venv +RUN chown -R user:user ${VENV} USER user diff --git a/tests/base_test.py b/tests/base_test.py index d7d0cb7..669ab8a 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -86,7 +86,6 @@ async def init_web_server(handle, host, port): logger.debug("BaseTest start_server") handle = APIHandler() - asyncio.set_event_loop(loop) server = init_web_server( handle, From e1612c2aad681656795ce95bfd536c88ccaec922 Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Tue, 12 Sep 2023 08:33:23 +0200 Subject: [PATCH 6/9] remove annoying warning on pending tasks during pytest --- tests/base_test.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index 669ab8a..a25ed60 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -85,16 +85,18 @@ async def init_web_server(handle, host, port): return server logger.debug("BaseTest start_server") - handle = APIHandler() server = init_web_server( - handle, + APIHandler(), DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT ) - loop.run_until_complete(server) - loop.run_forever() + try: + loop.run_until_complete(server) + loop.run_forever() + finally: + loop.close() def check_server(): logger.debug("BaseTest check_server") @@ -118,6 +120,8 @@ def check_server(): def tearDownClass(cls): if cls.loop: logger.debug("BaseTest tearDownClass") + for task in asyncio.all_tasks(cls.loop): + task.cancel() cls.loop.call_soon_threadsafe(cls.loop.stop) cls.x.join() From 345ef73c6d30a0172f53fecc1691455f128e0464 Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Tue, 12 Sep 2023 08:39:00 +0200 Subject: [PATCH 7/9] small readability changes --- tests/base_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index a25ed60..0d1e416 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -71,7 +71,7 @@ class BaseTest(TestCase): def setUpClass(cls): def start_server(loop): - async def init_web_server(handle, host, port): + async def init_api_server(handle, host, port): server = web.ServerRunner(web.Server(handle)) await server.setup() @@ -86,14 +86,14 @@ async def init_web_server(handle, host, port): logger.debug("BaseTest start_server") - server = init_web_server( + api_server = init_api_server( APIHandler(), DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT ) try: - loop.run_until_complete(server) + loop.run_until_complete(api_server) loop.run_forever() finally: loop.close() @@ -110,8 +110,9 @@ def check_server(): if not os.environ.get("SBS_URL", None): logger.debug("BaseTest setUpClass") cls.loop = asyncio.new_event_loop() - cls.x = threading.Thread(target=start_server, args=(cls.loop,), ) - cls.x.start() + cls.api = threading.Thread(target=start_server, args=(cls.loop,), ) + cls.api.start() + check_server() else: cls.loop = None @@ -123,7 +124,7 @@ def tearDownClass(cls): for task in asyncio.all_tasks(cls.loop): task.cancel() cls.loop.call_soon_threadsafe(cls.loop.stop) - cls.x.join() + cls.api.join() def setUp(self): """ Run a complete PLSC cycle, 1st ordered structure, 2nd flat structure... From ffd061a33b12ea726fcc012a0ab1963004c1394a Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Wed, 13 Sep 2023 09:49:47 +0200 Subject: [PATCH 8/9] add CI action to build image --- .github/workflows/image.yml | 42 +++++++++++++++++++++++++++++++++++++ .gitignore | 2 ++ 2 files changed, 44 insertions(+) create mode 100644 .github/workflows/image.yml diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..67cde92 --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,42 @@ +# +name: Create and publish a Docker image + +on: + push: + branches: ["docker"] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index bc924f9..fdd25a1 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,5 @@ config.d/ tests/api/ .vscode/ .theia/ +Pipfile +.python-version \ No newline at end of file From 9e5df2da17f7d0acb4d7f82e2c8900193fd01cd6 Mon Sep 17 00:00:00 2001 From: Harry Kodden Date: Wed, 13 Sep 2023 11:41:52 +0200 Subject: [PATCH 9/9] Update README --- .gitignore | 3 ++- Dockerfile | 12 ++++++------ Makefile | 2 +- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++----------- 4 files changed, 54 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index fdd25a1..acad345 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ tests/api/ .vscode/ .theia/ Pipfile -.python-version \ No newline at end of file +.python-version +plsc.yaml diff --git a/Dockerfile b/Dockerfile index 5c70866..9a31bde 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG PYTHON_VERSION=3.11-alpine FROM python:${PYTHON_VERSION} -RUN apk add py3-virtualenv py3-pytest python3-dev gcc musl-dev openldap-dev +RUN apk add py3-virtualenv py3-pytest python3-dev gcc musl-dev openldap-dev bash ENV VENV /.venv RUN virtualenv ${VENV} @@ -10,14 +10,14 @@ ENV PATH ${VENV}/bin:$PATH RUN adduser -D user -RUN chown -R user:user ${VENV} +WORKDIR /app -USER user +ADD . . +RUN chown -R user:user ${VENV} . -ADD requirements.txt . -ADD requirements-minimal.txt . +USER user RUN pip install --upgrade pip RUN pip install -r requirements.txt -WORKDIR /app \ No newline at end of file +CMD ["./run.sh"] \ No newline at end of file diff --git a/Makefile b/Makefile index 8bfc979..a19417f 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,6 @@ ldap_stop: etc/ldap_stop.sh pytest: image ldap_start - docker run --rm -ti --network host -v ${PWD}:/app plsc pytest + docker run --rm -ti --network host plsc pytest clean: ldap_stop diff --git a/README.md b/README.md index ed79153..331da46 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ For local development we advice to make use of Python Virtual Environment. The i This is an example of what we need to specify source and destination. [ More complex examples will follow ] -``` +```bash --- ldap: src: @@ -34,7 +34,8 @@ src: ### Local development Create a virtual environment and install the required python packages: -``` + +```bash python3 -m venv .venv source .venv/bin/activate pip install --upgrade pip @@ -42,10 +43,12 @@ pip install -r requirements.txt ``` ### Use -```sync_services``` will look for all ```labeledURI``` attributes in the source and create ```dc=``` branches on the destination containing all CO's that are linked to these services with bare ```People```, ```Groups``` and ```uid/gidNumberSequence``` structures. -```plsc``` will then mirror all ```People``` and ```Groups``` to the corresponding CO's in the destination LDAP, optionally converting attributes and dn's as defined in the code on the way: -``` +`sync_services` will look for all `labeledURI` attributes in the source and create `dc=` branches on the destination containing all CO's that are linked to these services with bare `People`, `Groups` and `uid/gidNumberSequence` structures. + +`plsc` will then mirror all `People` and `Groups` to the corresponding CO's in the destination LDAP, optionally converting attributes and dn's as defined in the code on the way: + +```bash # Here's the magic: Build the new person entry dst_entry = {} dst_entry['objectClass'] = ['inetOrgPerson', 'person', 'posixAccount'] @@ -56,7 +59,8 @@ pip install -r requirements.txt ``` And for groups: -``` + +```bash # Here's the magic: Build the new group entry m = re.search('^(?:GRP)?(?:CO)?(?:COU)?:(.*?)$', src_cn) dst_cn = src_type + "_" + m.group(1) if m.group(1) else "" @@ -72,7 +76,7 @@ And for groups: For local testing, you need a local LDAP to be started before running tests. When you have docker installed on your local machine, you can simple run: -``` +```bash etc/ldap_start.sh ``` @@ -82,7 +86,7 @@ If you do not have docker installed or wish to use an existing running LDAP serv You can specify LDAP connection and access constants in a local **.env** file, for exanple: -``` +```bash LDAP_URL="ldap://localhost:389" LDAP_ADMIN_PASSWORD="secret" LDAP_CONFIG_PASSWORD="config" @@ -105,20 +109,50 @@ When you omit the **SBS_URL** variable, the tests will run API requests agains t When all these preperations are completed, you can now run the tests: -``` +```bash pytest -v ``` After each Test cycle the contents of the LDAP can be inspected, for that run this command: -``` +```bash etc/ldap_show.sh ``` When finished you can tear down the local running LDAP instance, by: -``` +```bash etc/ldap_stop.sh ``` +# Using Make + +A Makefile is added to make life easier. Now you can run: + +```bash +make pytest +``` + +Or just make. This will build the docker image and run pytest next. + +```bash +make +``` + +# Docker + +A Dockerfile is added to produce an image that is capable of running PLSC. Since PLSC depends on an input YAML file holding the source and destionatination details, you have to provide sucan YANL file. +Using Docker you can do the following: + +Suppose you have prepared you YAML file (inspired by plsc.yml.example) and this file is 'my-plsc.yml' + +You have to mount this file into the container and then run **run.sh** with that file as parameter. + +Example: + +```bash +docker run -v ${pwd}/my-plsc.yml:/tmp/plsc.yaml plsc ./run.sh /tmp/plsc.yml +``` + +**run.sh** is the existing script that runs both _plsc_ordered.py_ and _plsc_flat_