From f6dc476a7df2a8c03488337665535c4633616e9d Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Tue, 12 Dec 2023 16:40:34 -0500 Subject: [PATCH 01/22] Setup allowing pytest for log_based syncs Fix for 'str' has no 'append' --- .github/workflows/test.yml | 14 +++++++++ docker-compose.yml | 27 +++++++++++++++++ tap_postgres/client.py | 4 ++- tap_postgres/tap.py | 6 ++-- tests/test_log_based.py | 60 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 tests/test_log_based.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f449ae5..4941cf3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,20 @@ jobs: run: | docker compose -f docker-compose.yml up -d + - name: Set up log based replication + run: | + docker exec tap-postgres-postgres_log_based-1 apt-get update + docker exec tap-postgres-postgres_log_based-1 apt-get upgrade -y + docker exec tap-postgres-postgres_log_based-1 apt-get install curl ca-certificates -y + docker exec tap-postgres-postgres_log_based-1 install -d /usr/share/postgresql-common/pgdg + docker exec tap-postgres-postgres_log_based-1 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + docker exec tap-postgres-postgres_log_based-1 echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list + docker exec tap-postgres-postgres_log_based-1 apt-get update + docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-server-dev-15 -y + docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-15-wal2json -y + docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" + + - uses: isbang/compose-action@v1.5.1 # Leaving comments as this makes testing the openssh docker image easier and should save someone some time later diff --git a/docker-compose.yml b/docker-compose.yml index b9c5ac06..1bcdf87f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,33 @@ services: ports: - "5433:5432" + postgres_no_ssl: + image: postgres:latest + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - "5432:5432" + + postgres_log_based: + image: postgres:latest + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - "5434:5432" + command: + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_replication_slots=10" + - "-c" + - "max_wal_senders=10" + + networks: inner: driver: bridge diff --git a/tap_postgres/client.py b/tap_postgres/client.py index 90dfdf78..b290fead 100644 --- a/tap_postgres/client.py +++ b/tap_postgres/client.py @@ -304,8 +304,10 @@ def schema(self) -> dict: """Override schema for log-based replication adding _sdc columns.""" schema_dict = typing.cast(dict, self._singer_catalog_entry.schema.to_dict()) for property in schema_dict["properties"].values(): - if "null" not in property["type"]: + if isinstance(property["type"], list): property["type"].append("null") + else: + property["type"] = [property["type"], "null"] if "required" in schema_dict: schema_dict.pop("required") schema_dict["properties"].update({"_sdc_deleted_at": {"type": ["string"]}}) diff --git a/tap_postgres/tap.py b/tap_postgres/tap.py index c04ed08b..3f5d1919 100644 --- a/tap_postgres/tap.py +++ b/tap_postgres/tap.py @@ -531,8 +531,10 @@ def catalog(self) -> Catalog: if new_stream.replication_method == "LOG_BASED": for property in new_stream.schema.properties.values(): if "null" not in property.type: - stream_modified = True - property.type.append("null") + if isinstance(property.type, list): + property.type.append("null") + else: + property.type = [property.type, "null"] if new_stream.schema.required: stream_modified = True new_stream.schema.required = None diff --git a/tests/test_log_based.py b/tests/test_log_based.py new file mode 100644 index 00000000..98993000 --- /dev/null +++ b/tests/test_log_based.py @@ -0,0 +1,60 @@ +import json + +import sqlalchemy +from sqlalchemy import Column, MetaData, Table +from sqlalchemy.dialects.postgresql import BIGINT, TEXT +from tap_postgres.tap import TapPostgres +from test_core import PostgresTestRunner + + +LOG_BASED_CONFIG = { + "host": "localhost", + "port": 5434, + "user": "postgres", + "password": "postgres", + "database": "postgres", +} + +def test_null_append(): + """LOG_BASED syncs failed with string property types. (issue #294). + + This test checks that even when a catalog contains properties with types represented + as strings (ex: "object") instead of arrays (ex: ["object"] or ["object", "null"]), + LOG_BASED replication can still append the "null" option to a property's type. + """ + table_name = "test_null_append" + engine = sqlalchemy.create_engine("postgresql://postgres:postgres@localhost:5434/postgres") + + metadata_obj = MetaData() + table = Table( + table_name, + metadata_obj, + Column("id", BIGINT, primary_key = True), + Column("data", TEXT, nullable = True) + ) + with engine.connect() as conn: + if table.exists(conn): + table.drop(conn) + metadata_obj.create_all(conn) + insert = table.insert().values(id=123, data="hello world") + conn.execute(insert) + tap = TapPostgres(config=LOG_BASED_CONFIG) + tap_catalog = json.loads(tap.catalog_json_text) + altered_table_name = f"public-{table_name}" + for stream in tap_catalog["streams"]: + if stream.get("stream") and altered_table_name not in stream["stream"]: + for metadata in stream["metadata"]: + metadata["metadata"]["selected"] = False + else: + stream["replication_method"] = "LOG_BASED" + stream["replication_key"] = "_sdc_lsn" + stream["schema"]["properties"]["data"]["type"] = "string" + for metadata in stream["metadata"]: + metadata["metadata"]["selected"] = True + if metadata["breadcrumb"] == []: + metadata["metadata"]["replication-method"] = "LOG_BASED" + + test_runner = PostgresTestRunner( + tap_class=TapPostgres, config=LOG_BASED_CONFIG, catalog=tap_catalog + ) + test_runner.sync_all() From 8ac5816a7b21e71cfcd430983d179424bd180108 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Tue, 12 Dec 2023 16:44:22 -0500 Subject: [PATCH 02/22] Remove postgres_no_ssl container --- docker-compose.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1bcdf87f..f3f512d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,15 +48,6 @@ services: ports: - "5433:5432" - postgres_no_ssl: - image: postgres:latest - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - ports: - - "5432:5432" - postgres_log_based: image: postgres:latest environment: From bd26f3d0e8cda57c61d1e15a2244fe151a35e60f Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Tue, 12 Dec 2023 16:48:49 -0500 Subject: [PATCH 03/22] Modify log_based setup --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba895951..724d7b30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,7 +51,7 @@ jobs: docker exec tap-postgres-postgres_log_based-1 apt-get install curl ca-certificates -y docker exec tap-postgres-postgres_log_based-1 install -d /usr/share/postgresql-common/pgdg docker exec tap-postgres-postgres_log_based-1 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc - docker exec tap-postgres-postgres_log_based-1 echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list + docker exec tap-postgres-postgres_log_based-1 sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' docker exec tap-postgres-postgres_log_based-1 apt-get update docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-server-dev-15 -y docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-15-wal2json -y From 81d615802d559b3ffacce36cde4e850a04fd20fc Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 08:58:28 -0500 Subject: [PATCH 04/22] Split log-based setup --- .github/workflows/test.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 724d7b30..dfdc586f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,17 +44,35 @@ jobs: run: | docker compose -f docker-compose.yml up -d - - name: Set up log based replication + - name: Set up log based replication A run: | docker exec tap-postgres-postgres_log_based-1 apt-get update + - name: Set up log based replication B + run: | docker exec tap-postgres-postgres_log_based-1 apt-get upgrade -y + - name: Set up log based replication C + run: | docker exec tap-postgres-postgres_log_based-1 apt-get install curl ca-certificates -y + - name: Set up log based replication D + run: | docker exec tap-postgres-postgres_log_based-1 install -d /usr/share/postgresql-common/pgdg + - name: Set up log based replication E + run: | docker exec tap-postgres-postgres_log_based-1 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + - name: Set up log based replication F + run: | docker exec tap-postgres-postgres_log_based-1 sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + - name: Set up log based replication G + run: | docker exec tap-postgres-postgres_log_based-1 apt-get update + - name: Set up log based replication H + run: | docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-server-dev-15 -y + - name: Set up log based replication I + run: | docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-15-wal2json -y + - name: Set up log based replication J + run: | docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" From 943261d1f7fd5f2f0129119fea4eedf528c9fcca Mon Sep 17 00:00:00 2001 From: Sebastian Smiley <46581584+sebastianswms@users.noreply.github.com> Date: Wed, 13 Dec 2023 08:59:18 -0500 Subject: [PATCH 05/22] checkfirst=True MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> --- tests/test_log_based.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_log_based.py b/tests/test_log_based.py index 98993000..f727a25a 100644 --- a/tests/test_log_based.py +++ b/tests/test_log_based.py @@ -33,8 +33,7 @@ def test_null_append(): Column("data", TEXT, nullable = True) ) with engine.connect() as conn: - if table.exists(conn): - table.drop(conn) + table.drop(conn, checkfirst=True) metadata_obj.create_all(conn) insert = table.insert().values(id=123, data="hello world") conn.execute(insert) From 5473f2b24d9b1e181e29ae7b9d440ea878e67a87 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 09:05:04 -0500 Subject: [PATCH 06/22] Add $PATH export --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dfdc586f..4526eb51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,6 +72,9 @@ jobs: run: | docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-15-wal2json -y - name: Set up log based replication J + run: | + docker exec tap-postgres-postgres_log_based-1 sh -c 'export PATH=/usr/lib/postgresql/15/bin:$PATH' + - name: Set up log based replication K run: | docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" From 81eda49ee0bb5a01e878bed3d6578fe1eca96dc0 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 09:12:24 -0500 Subject: [PATCH 07/22] Docker debugging --- .github/workflows/test.yml | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4526eb51..899d4204 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,34 +47,23 @@ jobs: - name: Set up log based replication A run: | docker exec tap-postgres-postgres_log_based-1 apt-get update - - name: Set up log based replication B - run: | docker exec tap-postgres-postgres_log_based-1 apt-get upgrade -y - - name: Set up log based replication C - run: | docker exec tap-postgres-postgres_log_based-1 apt-get install curl ca-certificates -y - - name: Set up log based replication D - run: | docker exec tap-postgres-postgres_log_based-1 install -d /usr/share/postgresql-common/pgdg - - name: Set up log based replication E - run: | docker exec tap-postgres-postgres_log_based-1 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc - - name: Set up log based replication F - run: | docker exec tap-postgres-postgres_log_based-1 sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - - name: Set up log based replication G - run: | docker exec tap-postgres-postgres_log_based-1 apt-get update - - name: Set up log based replication H - run: | docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-server-dev-15 -y - - name: Set up log based replication I - run: | docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-15-wal2json -y - - name: Set up log based replication J + - name: Set up log based replication B run: | docker exec tap-postgres-postgres_log_based-1 sh -c 'export PATH=/usr/lib/postgresql/15/bin:$PATH' - - name: Set up log based replication K + - name: Set up log based replication C + run: | + docker logs tap-postgres-postgres_log_based-1 + docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "\dx" + docker exec tap-postgres-postgres_log_based-1 ls -la /usr/lib/postgresql/15/bin + - name: Set up log based replication D run: | docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" From b4221ae44a71834963428eaca1dfdd2b05772d6c Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 11:07:36 -0500 Subject: [PATCH 08/22] Debugging reorganization --- .github/workflows/test.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 899d4204..db51f876 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,23 +47,15 @@ jobs: - name: Set up log based replication A run: | docker exec tap-postgres-postgres_log_based-1 apt-get update - docker exec tap-postgres-postgres_log_based-1 apt-get upgrade -y docker exec tap-postgres-postgres_log_based-1 apt-get install curl ca-certificates -y docker exec tap-postgres-postgres_log_based-1 install -d /usr/share/postgresql-common/pgdg docker exec tap-postgres-postgres_log_based-1 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc docker exec tap-postgres-postgres_log_based-1 sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' docker exec tap-postgres-postgres_log_based-1 apt-get update docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-server-dev-15 -y + docker exec tap-postgres-postgres_log_based-1 sh -c 'export PATH=/usr/lib/postgresql/15/bin:$PATH' docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-15-wal2json -y - name: Set up log based replication B - run: | - docker exec tap-postgres-postgres_log_based-1 sh -c 'export PATH=/usr/lib/postgresql/15/bin:$PATH' - - name: Set up log based replication C - run: | - docker logs tap-postgres-postgres_log_based-1 - docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "\dx" - docker exec tap-postgres-postgres_log_based-1 ls -la /usr/lib/postgresql/15/bin - - name: Set up log based replication D run: | docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" From 9cf1e21279c6430c846eca0b94aed4dc69a34c48 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 12:38:14 -0500 Subject: [PATCH 09/22] Switch to Dockerfile from docker exec --- .github/workflows/test.yml | 16 +--------------- Dockerfile | 12 ++++++++++++ docker-compose.yml | 4 +++- log_based/init.sql | 1 + 4 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 Dockerfile create mode 100644 log_based/init.sql diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index db51f876..a13a9b51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,24 +42,10 @@ jobs: - name: Set up Postgres container run: | + docker build . --tag log_based docker compose -f docker-compose.yml up -d - - - name: Set up log based replication A - run: | - docker exec tap-postgres-postgres_log_based-1 apt-get update - docker exec tap-postgres-postgres_log_based-1 apt-get install curl ca-certificates -y - docker exec tap-postgres-postgres_log_based-1 install -d /usr/share/postgresql-common/pgdg - docker exec tap-postgres-postgres_log_based-1 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc - docker exec tap-postgres-postgres_log_based-1 sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - docker exec tap-postgres-postgres_log_based-1 apt-get update - docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-server-dev-15 -y - docker exec tap-postgres-postgres_log_based-1 sh -c 'export PATH=/usr/lib/postgresql/15/bin:$PATH' - docker exec tap-postgres-postgres_log_based-1 apt-get install postgresql-15-wal2json -y - - name: Set up log based replication B - run: | docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" - - uses: isbang/compose-action@v1.5.1 # Leaving comments as this makes testing the openssh docker image easier and should save someone some time later diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7295c479 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM postgres:latest + +RUN apt-get update +RUN apt-mark hold locales +RUN apt-get install curl ca-certificates -y +RUN install -d /usr/share/postgresql-common/pgdg +RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc +RUN sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' +RUN apt-get update +RUN apt-get install postgresql-server-dev-15 -y +RUN sh -c 'export PATH=/usr/lib/postgresql/15/bin:$PATH' +RUN apt-get install postgresql-15-wal2json -y diff --git a/docker-compose.yml b/docker-compose.yml index f3f512d0..73c4b432 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,13 +49,15 @@ services: - "5433:5432" postgres_log_based: - image: postgres:latest + image: meltano/log_based # Locally built environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres ports: - "5434:5432" + volumes: + - ./log_based:/docker-entrypoint-initdb.d command: - "postgres" - "-c" diff --git a/log_based/init.sql b/log_based/init.sql new file mode 100644 index 00000000..4cd87906 --- /dev/null +++ b/log_based/init.sql @@ -0,0 +1 @@ +SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json'); \ No newline at end of file From 01c9576f6d25af09cd5b3af3ef7a21ee27b20aaa Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 12:58:45 -0500 Subject: [PATCH 10/22] Fix typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a13a9b51..8934eddb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: - name: Set up Postgres container run: | - docker build . --tag log_based + docker build . --tag meltano/log_based docker compose -f docker-compose.yml up -d docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" From 6aaca9d7deab42a04a31c997ee892d578d6e530e Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 13:00:46 -0500 Subject: [PATCH 11/22] Remove docker exec --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8934eddb..9cec2c80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,6 @@ jobs: run: | docker build . --tag meltano/log_based docker compose -f docker-compose.yml up -d - docker exec tap-postgres-postgres_log_based-1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('tappostgres', 'wal2json');" - uses: isbang/compose-action@v1.5.1 From 50b53c585d148f4710a9aa511534cb93c3a0d109 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 13:04:06 -0500 Subject: [PATCH 12/22] Import PostgresTestRunner --- tests/test_log_based.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_log_based.py b/tests/test_log_based.py index f727a25a..0048c0a4 100644 --- a/tests/test_log_based.py +++ b/tests/test_log_based.py @@ -4,7 +4,7 @@ from sqlalchemy import Column, MetaData, Table from sqlalchemy.dialects.postgresql import BIGINT, TEXT from tap_postgres.tap import TapPostgres -from test_core import PostgresTestRunner +from tests.test_core import PostgresTestRunner LOG_BASED_CONFIG = { From e4e54e4118916901eebcc4e0edc649bfcf2e6e85 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 13:08:25 -0500 Subject: [PATCH 13/22] Debugging --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9cec2c80..92f9bc35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,7 @@ jobs: run: | docker build . --tag meltano/log_based docker compose -f docker-compose.yml up -d + docker ps -a - uses: isbang/compose-action@v1.5.1 From 56329ea219e697d3c7eb052d8d3f9b4553be5478 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 14:34:33 -0500 Subject: [PATCH 14/22] Docker restructure --- .github/workflows/test.yml | 2 +- docker-compose.yml | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92f9bc35..1f24d70b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,6 @@ jobs: run: | docker build . --tag meltano/log_based docker compose -f docker-compose.yml up -d - docker ps -a - uses: isbang/compose-action@v1.5.1 @@ -74,6 +73,7 @@ jobs: poetry install - name: Run pytest run: | + docker ps -a poetry run pytest - name: Run lint run: | diff --git a/docker-compose.yml b/docker-compose.yml index 73c4b432..47c74f40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,22 +50,15 @@ services: postgres_log_based: image: meltano/log_based # Locally built + command: postgres -c wal_level=logica -c max_replication_slots=10 -c max_wal_senders=10 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - ports: - - "5434:5432" volumes: - ./log_based:/docker-entrypoint-initdb.d - command: - - "postgres" - - "-c" - - "wal_level=logical" - - "-c" - - "max_replication_slots=10" - - "-c" - - "max_wal_senders=10" + ports: + - "5434:5432" networks: From 15e6c06d5330a488fe80c7f1ad724f7266216351 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 14:37:50 -0500 Subject: [PATCH 15/22] GitHub actions debugging --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f24d70b..5ae902ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,6 +74,7 @@ jobs: - name: Run pytest run: | docker ps -a + docker logs tap-postgres-postgres_log_based-1 poetry run pytest - name: Run lint run: | From 79ba74c6f373eedf8ae653dd79bbcbaf187a5725 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 14:41:55 -0500 Subject: [PATCH 16/22] Fix typo --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 47c74f40..6b536fe0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,7 +50,7 @@ services: postgres_log_based: image: meltano/log_based # Locally built - command: postgres -c wal_level=logica -c max_replication_slots=10 -c max_wal_senders=10 + command: postgres -c wal_level=logical -c max_replication_slots=10 -c max_wal_senders=10 environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres From 33591d10b2f5f56315e549c3f6f75ef308b0422a Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 15:03:51 -0500 Subject: [PATCH 17/22] Debugging GitHub actions --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ae902ce..c164a2ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,9 +40,10 @@ jobs: sudo chown 999:999 ssl/server.key chmod 600 ssl/pkey.key - - name: Set up Postgres container + - name: Build Postgres container run: | docker build . --tag meltano/log_based + - name: Compose Postgres container docker compose -f docker-compose.yml up -d - uses: isbang/compose-action@v1.5.1 @@ -71,10 +72,12 @@ jobs: - name: Install dependencies run: | poetry install - - name: Run pytest + - name: Debugging run: | docker ps -a docker logs tap-postgres-postgres_log_based-1 + - name: Run pytest + run: | poetry run pytest - name: Run lint run: | From 7f0bc117410da7c13d327deaf9da6dc442584288 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 15:04:09 -0500 Subject: [PATCH 18/22] Debugging GitHub actions --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c164a2ca..57ac9c62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,7 @@ jobs: run: | docker build . --tag meltano/log_based - name: Compose Postgres container + run: | docker compose -f docker-compose.yml up -d - uses: isbang/compose-action@v1.5.1 From 3facd567b69dbdb0124fedbaaa8007bfecf719fd Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 15:29:52 -0500 Subject: [PATCH 19/22] Lock to postgres 15 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7295c479..d906333a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:latest +FROM postgres:15 RUN apt-get update RUN apt-mark hold locales From 3fbf99db0a0f0cfdf05cdc080aebea1966585306 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 15:35:18 -0500 Subject: [PATCH 20/22] linting, remove debugging code --- .github/workflows/test.yml | 4 ---- docker-compose.yml | 2 +- tap_postgres/tap.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 57ac9c62..c2921ae5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -73,10 +73,6 @@ jobs: - name: Install dependencies run: | poetry install - - name: Debugging - run: | - docker ps -a - docker logs tap-postgres-postgres_log_based-1 - name: Run pytest run: | poetry run pytest diff --git a/docker-compose.yml b/docker-compose.yml index 6b536fe0..0b8bc6a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: ipv4_address: 10.5.0.5 postgres_ssl: - image: postgres:latest + image: postgres:15 command: postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key -c ssl_ca_file=/var/lib/postgresql/ca.crt -c hba_file=/var/lib/postgresql/pg_hba.conf environment: POSTGRES_USER: postgres diff --git a/tap_postgres/tap.py b/tap_postgres/tap.py index 3f5d1919..0aebab29 100644 --- a/tap_postgres/tap.py +++ b/tap_postgres/tap.py @@ -515,7 +515,7 @@ def catalog_dict(self) -> dict: return self._catalog_dict @property - def catalog(self) -> Catalog: + def catalog(self) -> Catalog: # noqa: C901 """Get the tap's working catalog. Override to do LOG_BASED modifications. From b1eec9f465cb4730952c2a9932a5e520a8721948 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 15:42:57 -0500 Subject: [PATCH 21/22] Healthcheck and update to 16 (ensure healthcheck fails) --- Dockerfile | 2 +- docker-compose.yml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d906333a..db552590 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:15 +FROM postgres:16 RUN apt-get update RUN apt-mark hold locales diff --git a/docker-compose.yml b/docker-compose.yml index 0b8bc6a1..32cf7a3f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,11 @@ services: - ./log_based:/docker-entrypoint-initdb.d ports: - "5434:5432" + healthcheck: + test: [ "CMD", "pg_isready", "-q", "-d", "postgres", "-U", "postgres" ] + timeout: 45s + interval: 10s + retries: 10 networks: From b498d1082b71583b3cc277eaaeb06139d6465700 Mon Sep 17 00:00:00 2001 From: Sebastian Smiley Date: Wed, 13 Dec 2023 16:20:45 -0500 Subject: [PATCH 22/22] Add docker compose wait and fully update to postgres 16 --- .github/workflows/test.yml | 2 +- Dockerfile | 6 +++--- docker-compose.yml | 7 +------ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2921ae5..c1f0e2dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,7 +45,7 @@ jobs: docker build . --tag meltano/log_based - name: Compose Postgres container run: | - docker compose -f docker-compose.yml up -d + docker compose -f docker-compose.yml up -d --wait --wait-timeout=30 - uses: isbang/compose-action@v1.5.1 diff --git a/Dockerfile b/Dockerfile index db552590..478adf97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,6 @@ RUN install -d /usr/share/postgresql-common/pgdg RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc RUN sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" > /etc/apt/sources.list.d/pgdg.list' RUN apt-get update -RUN apt-get install postgresql-server-dev-15 -y -RUN sh -c 'export PATH=/usr/lib/postgresql/15/bin:$PATH' -RUN apt-get install postgresql-15-wal2json -y +RUN apt-get install postgresql-server-dev-16 -y +RUN sh -c 'export PATH=/usr/lib/postgresql/16/bin:$PATH' +RUN apt-get install postgresql-16-wal2json -y diff --git a/docker-compose.yml b/docker-compose.yml index 32cf7a3f..4a2b7978 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: ipv4_address: 10.5.0.5 postgres_ssl: - image: postgres:15 + image: postgres:16 command: postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key -c ssl_ca_file=/var/lib/postgresql/ca.crt -c hba_file=/var/lib/postgresql/pg_hba.conf environment: POSTGRES_USER: postgres @@ -59,11 +59,6 @@ services: - ./log_based:/docker-entrypoint-initdb.d ports: - "5434:5432" - healthcheck: - test: [ "CMD", "pg_isready", "-q", "-d", "postgres", "-U", "postgres" ] - timeout: 45s - interval: 10s - retries: 10 networks: