From a39916cef16a5e1592aacb25989efbdc00bf7ff4 Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 13:42:30 +0200 Subject: [PATCH 01/12] Remove BuildBuddy grpc client --- .github/workflows/deploy.yml | 3 +- .github/workflows/schedule.yml | 6 +- MODULE.bazel | 2 - WORKSPACE.bzlmod | 55 --------------- buildbuddy.yaml | 20 ------ requirements.in | 2 - requirements.txt | 18 ----- tools/buildbuddy/BUILD.bazel | 40 ----------- tools/buildbuddy/action.proto | 75 -------------------- tools/buildbuddy/client.py | 85 ----------------------- tools/buildbuddy/common.proto | 112 ------------------------------ tools/buildbuddy/file.proto | 61 ---------------- tools/buildbuddy/invocation.proto | 103 --------------------------- tools/buildbuddy/log.proto | 44 ------------ tools/buildbuddy/service.proto | 59 ---------------- tools/buildbuddy/target.proto | 77 -------------------- tools/buildbuddy/workflow.proto | 51 -------------- 17 files changed, 5 insertions(+), 808 deletions(-) delete mode 100644 tools/buildbuddy/BUILD.bazel delete mode 100644 tools/buildbuddy/action.proto delete mode 100644 tools/buildbuddy/client.py delete mode 100644 tools/buildbuddy/common.proto delete mode 100644 tools/buildbuddy/file.proto delete mode 100644 tools/buildbuddy/invocation.proto delete mode 100644 tools/buildbuddy/log.proto delete mode 100644 tools/buildbuddy/service.proto delete mode 100644 tools/buildbuddy/target.proto delete mode 100644 tools/buildbuddy/workflow.proto diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 851087517..99c3be2ce 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,5 +18,4 @@ jobs: - uses: ./.github/actions/setup-bazel with: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD }} - - name: Run client - run: bazel run --config buildbuddy --config buildbuddy_rbe //tools/buildbuddy:client -- "Deploy Provisioner" + - run: bazel run --config buildbuddy --config buildbuddy_rbe //provisioner:deploy diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 6bc93bbd7..8c3aeff13 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -2,6 +2,9 @@ name: Schedule on: schedule: - cron: "*/10 * * * *" + pull_request: + branches: + - "*" jobs: provisioner-validate: @@ -14,5 +17,4 @@ jobs: - uses: ./.github/actions/setup-bazel with: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD }} - - name: Run client - run: bazel run --config buildbuddy --config buildbuddy_rbe //tools/buildbuddy:client -- "Validate Provisioner" + - run: bazel run --config buildbuddy --config buildbuddy_rbe //provisioner:deploy_validate diff --git a/MODULE.bazel b/MODULE.bazel index c09553ea9..fef39ca47 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -100,5 +100,3 @@ use_repo(npm, "npm") # ------------------------------------ buildifier ------------------------------------ # # NOTE: don't upgrade to 6.1.0 because this has a bug where tests always pass bazel_dep(name = "buildifier_prebuilt", version = "6.0.0.1") -bazel_dep(name = "rules_proto", version = "5.3.0-21.7") -bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index ad4384f45..7b557be24 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -1,60 +1,5 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") -# ------------------------------------ rules_go ------------------------------------ # -http_archive( - name = "io_bazel_rules_go", - sha256 = "6dc2da7ab4cf5d7bfc7c949776b1b7c733f05e56edc4bcd9022bb249d2e2a996", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.39.1/rules_go-v0.39.1.zip", - ], -) - -# ------------------------------------ rules_proto_grpc ------------------------------------ # -http_archive( - name = "rules_proto_grpc", - sha256 = "928e4205f701b7798ce32f3d2171c1918b363e9a600390a25c876f075f1efc0a", - strip_prefix = "rules_proto_grpc-4.4.0", - urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/releases/download/4.4.0/rules_proto_grpc-4.4.0.tar.gz"], -) - -load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") - -rules_proto_grpc_toolchains() - -rules_proto_grpc_repos() - -load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") - -rules_proto_dependencies() - -rules_proto_toolchains() - -load("@rules_proto_grpc//python:repositories.bzl", rules_proto_grpc_python_repos = "python_repos") - -rules_proto_grpc_python_repos() - -load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") - -grpc_deps() - -load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") - -grpc_extra_deps() - -load("@rules_python//python:pip.bzl", "pip_parse") -load("@python3//:defs.bzl", "interpreter") - -pip_parse( - name = "rules_proto_grpc_py3_deps", - python_interpreter_target = interpreter, - requirements_lock = "@rules_proto_grpc//python:requirements.txt", -) - -load("@rules_proto_grpc_py3_deps//:requirements.bzl", "install_deps") - -install_deps() - # ------------------------------------ rules_docker ------------------------------------ # # https://github.com/bazelbuild/rules_docker/pull/2201 http_archive( diff --git a/buildbuddy.yaml b/buildbuddy.yaml index b10a1d69b..9c3326577 100644 --- a/buildbuddy.yaml +++ b/buildbuddy.yaml @@ -11,23 +11,3 @@ actions: - "*" bazel_commands: - "test //... @rules_task//... --config buildbuddy --config buildbuddy_rbe" - - - name: "Deploy Provisioner" - user: buildbuddy - container_image: "ubuntu-20.04" - triggers: - push: - branches: - - "reserved-for-buildbuddy-api__deploy-provisioner" - bazel_commands: - - "run //provisioner:deploy --config buildbuddy --config buildbuddy_rbe" - - - name: "Validate Provisioner" - user: buildbuddy - container_image: "ubuntu-20.04" - triggers: - push: - branches: - - "reserved-for-buildbuddy-api__validate-provisioner" - bazel_commands: - - "run //provisioner:deploy_validate --config buildbuddy --config buildbuddy_rbe" diff --git a/requirements.in b/requirements.in index 67b4958c6..3dcc70275 100644 --- a/requirements.in +++ b/requirements.in @@ -13,6 +13,4 @@ pyinfra==2.7 sqlalchemy==2.0.15; semver==2.13.0; bazel-runfiles==0.22.0; -grpclib[protobuf]==0.4.4; -googleapis-common-protos==1.59.0; requests==2.31.0; \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a26b55686..973b1ac65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -660,7 +660,6 @@ googleapis-common-protos==1.59.0 \ --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \ --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f # via - # -r requirements.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http greenlet==2.0.2 \ @@ -776,17 +775,10 @@ grpcio==1.51.3 \ # via # opentelemetry-exporter-otlp-proto-grpc # pulumi -grpclib[protobuf]==0.4.4 \ - --hash=sha256:80c7f5179b8d05d1192a092db1b60b4934a0d52e9919358de75c912c269485e5 - # via -r requirements.in h11==0.14.0 \ --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \ --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761 # via httpcore -h2==4.1.0 \ - --hash=sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d \ - --hash=sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb - # via grpclib home-assistant-bluetooth==1.10.0 \ --hash=sha256:a5fc5dc13088f92cf157ecba153972fded22e042d2507b2933850b5cfc8b82e0 \ --hash=sha256:e810a2db9d3d542779c46a4202fb3f1de54692e2631257a00b8aa2634c7ebde0 @@ -795,10 +787,6 @@ homeassistant==2023.6.0 \ --hash=sha256:744b3303de4423018c47666da5eecff307a1e056ea189064b28ee70af4e43c37 \ --hash=sha256:9e445fbe2880b1544129d41482da07b3996492182b0b0684a97ec6bd42b6e8e2 # via pytest-homeassistant-custom-component -hpack==4.0.0 \ - --hash=sha256:84a076fad3dc9a9f8063ccb8041ef100867b1878b25ef0ee63847a5d53818a6c \ - --hash=sha256:fc41de0c63e687ebffde81187a948221294896f6bdc0ae2312708df339430095 - # via h2 httpcore==0.17.2 \ --hash=sha256:125f8375ab60036db632f34f4b627a9ad085048eef7cb7d2616fea0f739f98af \ --hash=sha256:5581b9c12379c4288fe70f43c710d16060c10080617001e6b22a3b6dbcbefd36 @@ -809,10 +797,6 @@ httpx==0.24.1 \ # via # homeassistant # respx -hyperframe==6.0.1 \ - --hash=sha256:0ec6bafd80d8ad2195c4f03aacba3a8265e57bc4cff261e802bf39970ed02a15 \ - --hash=sha256:ae510046231dc8e9ecb1a6586f63d2347bf4c8905914aa84ba585ae85f28a914 - # via h2 idna==3.4 \ --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 @@ -1021,7 +1005,6 @@ multidict==6.0.4 \ --hash=sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba # via # aiohttp - # grpclib # yarl mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ @@ -1232,7 +1215,6 @@ protobuf==4.23.2 \ --hash=sha256:efabbbbac1ab519a514579ba9ec52f006c28ae19d97915951f69fa70da2c9e91 # via # googleapis-common-protos - # grpclib # opentelemetry-proto # pulumi psutil==5.9.5 \ diff --git a/tools/buildbuddy/BUILD.bazel b/tools/buildbuddy/BUILD.bazel deleted file mode 100644 index 93aae081b..000000000 --- a/tools/buildbuddy/BUILD.bazel +++ /dev/null @@ -1,40 +0,0 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") -load("@rules_proto_grpc//python:defs.bzl", "python_grpclib_library") -load("@pip-setup//:requirements.bzl", "requirement") - -proto_library( - name = "api_v1_proto", - srcs = [ - "action.proto", - "common.proto", - "file.proto", - "invocation.proto", - "log.proto", - "service.proto", - "target.proto", - "workflow.proto", - ], - visibility = ["//visibility:public"], - deps = [ - "@com_google_protobuf//:duration_proto", - "@com_google_protobuf//:timestamp_proto", - "@go_googleapis//google/rpc:status_proto", - ], -) - -python_grpclib_library( - name = "lib_grpc", - protos = [":api_v1_proto"], -) - -py_binary( - name = "client", - srcs = [ - "client.py", - ], - deps = [ - requirement("googleapis-common-protos"), - requirement("grpclib"), - ":lib_grpc", - ], -) diff --git a/tools/buildbuddy/action.proto b/tools/buildbuddy/action.proto deleted file mode 100644 index cc5d3cd06..000000000 --- a/tools/buildbuddy/action.proto +++ /dev/null @@ -1,75 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -import "tools/buildbuddy/file.proto"; - -// Request passed into GetAction -message GetActionRequest { - // The selector defining which action(s) to retrieve. - ActionSelector selector = 1; - - // The next_page_token value returned from a previous request, if any. - string page_token = 2; -} - -// Response from calling GetAction -message GetActionResponse { - // Actions matching the request, possibly capped a server limit. - repeated Action action = 1; - - // Token to retrieve the next page of results, or empty if there are no - // more results in the list. - string next_page_token = 2; -} - -// An action that happened as part of a configured target. This action could be -// a build, a test, or another type of action. -message Action { - // The resource ID components that identify the Action. - message Id { - // The Invocation ID. - string invocation_id = 1; - - // The Target ID. - string target_id = 2; - - // The Configuration ID. - string configuration_id = 3; - - // The Action ID. - string action_id = 4; - } - - // The resource ID components that identify the Action. - Id id = 1; - - // A list of file references for action level files. - repeated File file = 2; - - // The label of the target that generated this action. - string target_label = 3; -} - -// The selector used to specify which actions to return. -message ActionSelector { - // Required: The Invocation ID. - // All actions returned will be scoped to this invocation. - string invocation_id = 1; - - // Optional: The Target ID. - // If set, all actions returned will be scoped to this target. - string target_id = 2; - - // Optional: The Configuration ID. - // If set, all actions returned will be scoped to this configuration. - string configuration_id = 3; - - // Optional: The Action ID. - // If set, only the action with this action id will be returned. - string action_id = 4; - - // Optional: The Target label. - // If set, only the action with this target label will be returned. - string target_label = 5; -} diff --git a/tools/buildbuddy/client.py b/tools/buildbuddy/client.py deleted file mode 100644 index 5ec817574..000000000 --- a/tools/buildbuddy/client.py +++ /dev/null @@ -1,85 +0,0 @@ -import pprint -from tools.buildbuddy.service_grpc import ApiServiceStub -from tools.buildbuddy.invocation_pb2 import GetInvocationRequest, InvocationSelector -from tools.buildbuddy.workflow_pb2 import ExecuteWorkflowRequest -from tools.buildbuddy.log_pb2 import GetLogRequest, LogSelector -from grpclib.client import Channel -import asyncio -import os -import sys - - -async def start_workflow(stub, action_name): - token = os.environ["BUILDBUDDY_API_KEY"] - reply = await stub.ExecuteWorkflow( - ExecuteWorkflowRequest( - repo_url="https://github.com/mvgijssel/setup", - ref="master", - action_names=[action_name], - ), - metadata={"x-buildbuddy-api-key": token}, - ) - - print(reply) - - action_status = reply.action_statuses[0] - - if action_status.status.code: - print(action_status.status.message) - sys.exit(1) - - return action_status.invocation_id - - -async def get_log(stub, invocation_id): - token = os.environ["BUILDBUDDY_API_KEY"] - reply = await stub.GetLog( - GetLogRequest(selector=LogSelector(invocation_id=invocation_id)), - metadata={"x-buildbuddy-api-key": token}, - ) - # print(reply) - return reply - - -async def get_invocation(stub, invocation_id): - token = os.environ["BUILDBUDDY_API_KEY"] - reply = await stub.GetInvocation( - GetInvocationRequest( - selector=InvocationSelector(invocation_id=invocation_id), - include_metadata=True, - ), - metadata={"x-buildbuddy-api-key": token}, - ) - # print(reply) - return reply.invocation[0] - - -async def main(): - _, action_name = sys.argv - - async with Channel("app.buildbuddy.io", 1986, ssl=True) as channel: - stub = ApiServiceStub(channel) - - invocation_id = await start_workflow(stub, action_name=action_name) - - while True: - invocation = await get_invocation(stub, invocation_id) - - if invocation.duration_usec > 0: - break - - print("Not yet ready. Sleeping 5...") - await asyncio.sleep(5) - - log = await get_log(stub, invocation_id) - - print(log.log.contents) - - if invocation.success: - sys.exit(0) - else: - sys.exit(1) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tools/buildbuddy/common.proto b/tools/buildbuddy/common.proto deleted file mode 100644 index 9dedafa16..000000000 --- a/tools/buildbuddy/common.proto +++ /dev/null @@ -1,112 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -import "google/protobuf/duration.proto"; -import "google/protobuf/timestamp.proto"; - -enum Status { - // The implicit default enum value. Should never be set. - STATUS_UNSPECIFIED = 0; - - // Displays as "Building". Means the target is compiling, linking, etc. - BUILDING = 1; - - // Displays as "Built". Means the target was built successfully. - // If testing was requested, it should never reach this status: it should go - // straight from BUILDING to TESTING. - BUILT = 2; - - // Displays as "Broken". Means build failure such as compile error. - FAILED_TO_BUILD = 3; - - // Displays as "Testing". Means the test is running. - TESTING = 4; - - // Displays as "Passed". Means the test was run and passed. - PASSED = 5; - - // Displays as "Failed". Means the test was run and failed. - FAILED = 6; - - // Displays as "Timed out". Means the test didn't finish in time. - TIMED_OUT = 7; - - // Displays as "Cancelled". Means the build or test was cancelled. - // E.g. User hit control-C. - CANCELLED = 8; - - // Displays as "Tool Failed". Means the build or test had internal tool - // failure. - TOOL_FAILED = 9; - - // Displays as "Incomplete". Means the build or test did not complete. This - // might happen when a build breakage or test failure causes the tool to stop - // trying to build anything more or run any more tests, with the default - // bazel --nokeep_going option or the --notest_keep_going option. - INCOMPLETE = 10; - - // Displays as "Flaky". Means the aggregate status contains some runs that - // were successful, and some that were not. - FLAKY = 11; - - // Displays as "Unknown". Means the tool uploading to the server died - // mid-upload or does not know the state. - UNKNOWN = 12; - - // Displays as "Skipped". Means building and testing were skipped. - // (E.g. Restricted to a different configuration.) - SKIPPED = 13; -} - -// These correspond to the suffix of the rule name. Eg cc_test has type TEST. -enum TargetType { - // Unspecified by the build system. - TARGET_TYPE_UNSPECIFIED = 0; - - // An application e.g. ios_application. - APPLICATION = 1; - - // A binary target e.g. cc_binary. - BINARY = 2; - - // A library target e.g. java_library - LIBRARY = 3; - - // A package - PACKAGE = 4; - - // Any test target, in bazel that means a rule with a '_test' suffix. - TEST = 5; -} - -// Indicates how big the user indicated the test action was. -enum TestSize { - // Unspecified by the user. - TEST_SIZE_UNSPECIFIED = 0; - - // Unit test taking less than 1 minute. - SMALL = 1; - - // Integration tests taking less than 5 minutes. - MEDIUM = 2; - - // End-to-end tests taking less than 15 minutes. - LARGE = 3; - - // Even bigger than LARGE. - ENORMOUS = 4; - - // Something that doesn't fit into the above categories. - OTHER_SIZE = 5; -} - -// The timing of a particular Invocation, Action, etc. The start_time is -// specified, stop time can be calculated by adding duration to start_time. -message Timing { - // The time the resource started running. This is in UTC Epoch time. - google.protobuf.Timestamp start_time = 1; - - // The duration for which the resource ran. - google.protobuf.Duration duration = 2; -} diff --git a/tools/buildbuddy/file.proto b/tools/buildbuddy/file.proto deleted file mode 100644 index b817e5448..000000000 --- a/tools/buildbuddy/file.proto +++ /dev/null @@ -1,61 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -// Request object for GetFile -message GetFileRequest { - // File URI corresponding to the `uri` field in the File message. - // - // If the BuildBuddy instance supports ZSTD transcoding, the literal string - // "/blobs/" in the URI (third-to-last path segment) may be replaced with - // "/compressed-blobs/zstd/", and the server will return a compressed payload. - // - // Examples: - // * Uncompressed blob with remote instance name of "ci": - // bytestream://remote.buildbuddy.io/ci/blobs/09e6fe6e1fd8c8734339a0a84c3c7a0eb121b57a45d21cfeb1f265bffe4c4888/216 - // - // * zstd-compressed blob with no remote instance name: - // bytestream://remote.buildbuddy.io/compressed-blobs/zstd/09e6fe6e1fd8c8734339a0a84c3c7a0eb121b57a45d21cfeb1f265bffe4c4888/216 - string uri = 1; -} - -// Response object for GetFile -message GetFileResponse { - // The file data. - bytes data = 1; -} - -// A file associated with a BuildBuddy build. -message File { - string name = 1; - string uri = 2; - string hash = 3; - int64 size_bytes = 4; -} - -// Request object for DeleteFile -message DeleteFileRequest { - // URI of file to delete. - // - // CAS URI format: - // /// - // Action cache URI format: - // //ac// - // - // Examples: - // * CAS artifact: - // compressed-blobs/zstd/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/2084 - // - // * CAS artifact with remote_instance_name - // my_remote_instance_name/blobs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/2084 - // - // * Action cache artifact: - // blobs/ac/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/2084 - // - // * Action cache artifact with remote_instance_name - // my_remote_instance_name/blobs/ac/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/2084 - string uri = 1; -} - -// Response object for DeleteFile -message DeleteFileResponse {} diff --git a/tools/buildbuddy/invocation.proto b/tools/buildbuddy/invocation.proto deleted file mode 100644 index 302587abe..000000000 --- a/tools/buildbuddy/invocation.proto +++ /dev/null @@ -1,103 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -// Request passed into GetInvocation -message GetInvocationRequest { - // The selector defining which invocations(s) to retrieve. - InvocationSelector selector = 1; - - // If true, includes additional build metadata. - bool include_metadata = 2; - - // The next_page_token value returned from a previous request, if any. - string page_token = 3; -} - -// Response from calling GetInvocation -message GetInvocationResponse { - // Invocations matching the request invocation, possibly capped by a - // server limit. - repeated Invocation invocation = 1; - - // Token to retrieve the next page of results, or empty if there are no - // more results in the list. - string next_page_token = 2; -} - -// Each Invocation represents metadata associated with a given invocation. -message Invocation { - // The resource ID components that identify the Invocation. - message Id { - // The Invocation ID. - string invocation_id = 1; - } - - // The resource ID components that identify the Invocation. - Id id = 1; - - // Whether or not the build was successful. - bool success = 3; - - // The user who performed this build. - string user = 4; - - // The duration of this build, from start to finish. - int64 duration_usec = 5; - - // The host this build was executed on. - string host = 6; - - // The command performed (usually "build" or "test"). - string command = 7; - - // The build patterns specified for this build. - string pattern = 8; - - // The number of actions performed. - int64 action_count = 9; - - // The time this invocation was created and updated, respectively. Invocations - // are created as soon as the first event is received from the client and - // updated with subsequent events until they are finalized. - int64 created_at_usec = 13; - int64 updated_at_usec = 14; - - // A URL to the git repo this invocation was for. - string repo_url = 15; - - // The commit SHA that this invocation was for. - string commit_sha = 16; - - // The role played by this invocation. Ex: "CI" - string role = 19; - - // The git branch that this invocation was for. - string branch_name = 20; - - // The invocation's build metadata. Only included if include_metadata = true. - repeated InvocationMetadata build_metadata = 21; - - // The invocation's workspace status. - // Only included if include_metadata = true. - repeated InvocationMetadata workspace_status = 22; -} - -// Key value pair containing invocation metadata. -message InvocationMetadata { - string key = 1; - string value = 2; -} - -// The selector used to specify which invocations to return. -message InvocationSelector { - // One invocation_id or commit_sha is required. - - // Optional: The Invocation ID. - // Return only the invocation with this invocation ID. - string invocation_id = 1; - - // Optional: The commmit SHA. - // If set, only the invocations with this commit SHA will be returned. - string commit_sha = 2; -} diff --git a/tools/buildbuddy/log.proto b/tools/buildbuddy/log.proto deleted file mode 100644 index 9b9280750..000000000 --- a/tools/buildbuddy/log.proto +++ /dev/null @@ -1,44 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -// Request passed into GetLog -message GetLogRequest { - // The selector defining which logs(s) to retrieve. - LogSelector selector = 1; - - // The next_page_token value returned from a previous request, if any. - string page_token = 3; -} - -// Response from calling GetLog -message GetLogResponse { - // Log matching the request, possibly capped by a server limit. - Log log = 1; - - // Token to retrieve the next page of the log, or empty if there are no - // more logs. - string next_page_token = 2; -} - -// Each Log represents a chunk of build logs. -message Log { - // The resource ID components that identify the Log. - message Id { - // The Invocation ID. - string invocation_id = 1; - } - - // The resource ID components that identify the Log. - Id id = 1; - - // The contents of the log. - string contents = 3; -} - -// The selector used to specify which logs to return. -message LogSelector { - // Required: The Invocation ID. - // Return only the logs associated with this invocation ID. - string invocation_id = 1; -} diff --git a/tools/buildbuddy/service.proto b/tools/buildbuddy/service.proto deleted file mode 100644 index e7a95fb8d..000000000 --- a/tools/buildbuddy/service.proto +++ /dev/null @@ -1,59 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -import "tools/buildbuddy/action.proto"; -import "tools/buildbuddy/file.proto"; -import "tools/buildbuddy/invocation.proto"; -import "tools/buildbuddy/log.proto"; -import "tools/buildbuddy/target.proto"; -import "tools/buildbuddy/workflow.proto"; - -// This is the public interface used to programatically retrieve information -// from BuildBuddy. -// -// Requests can be made over: -// - gRPC connecting on port 1985 -// - gRPCS connecting on port 1986 -// - REST by making a POST request to /api/v1/ENDPOINT_NAME -// Example: /api/v1/GetTarget -// -// Requests require setting an API key using the header (or metadata for GRPC -// requests) x-buildbuddy-api-key: YOUR_API_KEY which can be set in your -// config.yaml file for on-prem BuildBuddy, or retreived by emailing -// developers@buildbuddy.io if using the buildbuddy.io cloud hosted service. -// -// API access is available to BuildBuddy Enterprise customers. Email -// enterprise@buildbuddy.io for information on BuildBuddy Enterprise. -// -// If there's information you'd like to access programatically that's not -// included in this API, email developers@buildbuddy.io -service ApiService { - // Retrieves a list of invocations or a specific invocation matching the given - // request selector. - rpc GetInvocation(GetInvocationRequest) returns (GetInvocationResponse); - - // Retrieves the logs for a specific invocation. - rpc GetLog(GetLogRequest) returns (GetLogResponse); - - // Retrieves a list of targets or a specific target matching the given - // request selector. - rpc GetTarget(GetTargetRequest) returns (GetTargetResponse); - - // Retrieves a list of targets or a specific target matching the given - // request selector. - rpc GetAction(GetActionRequest) returns (GetActionResponse); - - // Streams the File with the given uri. - // - Over gRPC returns a stream of bytes to be stitched together in order. - // - Over HTTP this simply returns the requested file. - rpc GetFile(GetFileRequest) returns (stream GetFileResponse); - - // Delete the File with the given uri. - rpc DeleteFile(DeleteFileRequest) returns (DeleteFileResponse); - - // Execute a workflow for the given URL and branch. - // Github App authentication is required. The API does not support running - // legacy workflows. - rpc ExecuteWorkflow(ExecuteWorkflowRequest) returns (ExecuteWorkflowResponse); -} diff --git a/tools/buildbuddy/target.proto b/tools/buildbuddy/target.proto deleted file mode 100644 index 195ae9563..000000000 --- a/tools/buildbuddy/target.proto +++ /dev/null @@ -1,77 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -import "tools/buildbuddy/common.proto"; - -// Request passed into GetTarget -message GetTargetRequest { - // The selector defining which target(s) to retrieve. - TargetSelector selector = 1; - - // The next_page_token value returned from a previous request, if any. - string page_token = 3; -} - -// Response from calling GetTarget -message GetTargetResponse { - // Targets matching the request invocation, possibly capped by a - // server limit. - repeated Target target = 1; - - // Token to retrieve the next page of results, or empty if there are no - // more results in the list. - string next_page_token = 2; -} - -// Each Target represents data for a given target in a given Invocation. -message Target { - // The resource ID components that identify the Target. - message Id { - // The Invocation ID. - string invocation_id = 1; - - // The Target ID. - string target_id = 2; - } - - // The resource ID components that identify the Target. - Id id = 1; - - // The label of the target Ex: //server/test:foo - string label = 2; - - // The aggregate status of the target. - Status status = 3; - - // When this target started and its duration. - Timing timing = 4; - - // The type of the target rule. Ex: java_binary - string rule_type = 5; - - // Tags applied to this target (if any). - repeated string tag = 6; - - // The language of the target rule. Ex: java, go, sh - string language = 7; -} - -// The selector used to specify which targets to return. -message TargetSelector { - // Required: The Invocation ID. - // All actions returned will be scoped to this invocation. - string invocation_id = 1; - - // Optional: The Target ID. - // If set, only the target with this target id will be returned. - string target_id = 2; - - // Optional: Tag - // If set, only targets with this tag will be returned. - string tag = 3; - - // Optional: The Target label. - // If set, only the target with this target label will be returned. - string label = 4; -} diff --git a/tools/buildbuddy/workflow.proto b/tools/buildbuddy/workflow.proto deleted file mode 100644 index 886d465cb..000000000 --- a/tools/buildbuddy/workflow.proto +++ /dev/null @@ -1,51 +0,0 @@ -syntax = "proto3"; - -package api.v1; - -import "google/rpc/status.proto"; - -message ExecuteWorkflowRequest { - // URL of the repo the workflow is running for - // Ex. "https://github.com/some-user/acme" - string repo_url = 1; - // Reference for where the workflow should be run (currently only branch names - // are supported) Ex. "cool-feature" - string ref = 2; - - // OPTIONAL FIELDS - - // Names of the workflow actions to execute. Correspond to actions - // in buildbuddy.yaml, or actions in the default workflow config if - // buildbuddy.yaml is missing from the repo. - // - // If this is not set, will execute all actions in the workflow config. - // - // Ex. ["Browser Tests", "Docker tests"] - repeated string action_names = 3; - // Whether to run the workflow in a clean container. This prevents all - // existing workflow containers from being reused, so using this flag is not - // encouraged. - bool clean = 4; - // VISIBILITY build metadata used for the workflow invocation. - // Workflow invocations are private by default, but this can be - // set to "PUBLIC" to make the workflow invocation public. - string visibility = 5; -} - -message ExecuteWorkflowResponse { - message ActionStatus { - // Corresponds to an action in buildbuddy.yaml, or an action name in the - // default workflow config if buildbuddy.yaml is missing from the repo. Ex. - // "Test all targets" - string action_name = 1; - - // The BuildBuddy invocation ID from executing the action. - string invocation_id = 2; - - // The GRPC status from executing the action. - google.rpc.Status status = 3; - } - - // A list of the actions executed by the API. - repeated ActionStatus action_statuses = 1; -} From 8b4cc9f15db98d76a037b7737b16f23f2e5a3cbf Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 13:52:23 +0200 Subject: [PATCH 02/12] setup arm exporter --- .../deploys/monitoring/files/docker-compose.yml.j2 | 12 ++++++++++++ .../monitoring/files/nri-prometheus-config.yaml | 7 +++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 index cafa40176..ab5ba5f07 100644 --- a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 +++ b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 @@ -55,3 +55,15 @@ services: limits: cpus: '0.5' # Maximum of 0.5 CPU cores memory: '100M' # Maximum of 512 megabytes of memory + + arm_exporter: + container_name: arm_exporter + image: carlosedp/arm_exporter:latest@sha256:c2510142e3824686cba8af75826737a8158b25648e29867e262d26f553de5211 + restart: always + environment: + - SETUP_ENV={{ setup_env }} + deploy: + resources: + limits: + cpus: '0.5' # Maximum of 0.5 CPU cores + memory: '100M' # Maximum of 512 megabytes of memory \ No newline at end of file diff --git a/provisioner/deploys/monitoring/files/nri-prometheus-config.yaml b/provisioner/deploys/monitoring/files/nri-prometheus-config.yaml index f72c0afd7..d0eb278ec 100644 --- a/provisioner/deploys/monitoring/files/nri-prometheus-config.yaml +++ b/provisioner/deploys/monitoring/files/nri-prometheus-config.yaml @@ -5,10 +5,10 @@ cluster_name: "provisioner" # standalone: true # How often the integration should run. Defaults to 30s. -scrape_duration: "1m" +scrape_duration: "30s" # The HTTP client timeout when fetching data from targets. Defaults to 5s. -# scrape_timeout: "5s" +scrape_timeout: "15s" # How old must the entries used for calculating the counters delta be # before the telemetry emitter expires them. Defaults to 5m. @@ -61,6 +61,9 @@ require_scrape_enabled_label_for_nodes: true targets: - description: github_exporter urls: ["http://github_exporter:9504/metrics"] + + - description: arm_exporter + urls: ["http://arm_exporter:9243/metrics"] # targets: # - description: Secure etcd example # urls: From 58681dc9f995abb2585429e7f71b85521842da91 Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 13:55:41 +0200 Subject: [PATCH 03/12] fix schedule --- .github/workflows/deploy.yml | 2 ++ .github/workflows/schedule.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 99c3be2ce..34b7d434b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,3 +19,5 @@ jobs: with: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD }} - run: bazel run --config buildbuddy --config buildbuddy_rbe //provisioner:deploy + env: + ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD: ${{ secrets.ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD }} diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 8c3aeff13..ef0e2a017 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -18,3 +18,5 @@ jobs: with: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD }} - run: bazel run --config buildbuddy --config buildbuddy_rbe //provisioner:deploy_validate + env: + ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD: ${{ secrets.ONEPASSWORD_SERVICE_ACCOUNT_TOKEN_PROD }} From 90c532f92f1affae45965fd4115ff78ac38e9c02 Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:04:23 +0200 Subject: [PATCH 04/12] updated tests --- provisioner/test_provisioner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/provisioner/test_provisioner.py b/provisioner/test_provisioner.py index 0cda8a636..a5fd1ca57 100644 --- a/provisioner/test_provisioner.py +++ b/provisioner/test_provisioner.py @@ -106,6 +106,7 @@ def test_nri_prometheus_config(host): config = host.file("/opt/monitoring/nri-prometheus-config.yaml") assert config.exists assert config.contains("http://github_exporter:9504/metrics") + assert config.contains("http://arm_exporter:9243/metrics") def test_otel_collector_service(host): @@ -134,6 +135,11 @@ def test_otel_collector_health(host): ) +def test_arm_exporter_service(host): + arm_exporter = host.docker("arm_exporter") + assert arm_exporter.is_running + + def test_microk8s_installed(host): assert "microk8s" in host.check_output("snap list") From f7e9e883456c486faa20cc38d11c7607742cd23f Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:12:51 +0200 Subject: [PATCH 05/12] optionally start arm_exporter --- provisioner/deploys/monitoring/files/docker-compose.yml.j2 | 4 +++- provisioner/deploys/monitoring/tasks/install_monitoring.py | 3 ++- provisioner/test_provisioner.py | 5 ----- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 index ab5ba5f07..c598b4eb6 100644 --- a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 +++ b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 @@ -56,6 +56,7 @@ services: cpus: '0.5' # Maximum of 0.5 CPU cores memory: '100M' # Maximum of 512 megabytes of memory +{% if is_arm %} arm_exporter: container_name: arm_exporter image: carlosedp/arm_exporter:latest@sha256:c2510142e3824686cba8af75826737a8158b25648e29867e262d26f553de5211 @@ -66,4 +67,5 @@ services: resources: limits: cpus: '0.5' # Maximum of 0.5 CPU cores - memory: '100M' # Maximum of 512 megabytes of memory \ No newline at end of file + memory: '100M' # Maximum of 512 megabytes of memory +{% endif %} \ No newline at end of file diff --git a/provisioner/deploys/monitoring/tasks/install_monitoring.py b/provisioner/deploys/monitoring/tasks/install_monitoring.py index 0fd36148e..67ba8182c 100644 --- a/provisioner/deploys/monitoring/tasks/install_monitoring.py +++ b/provisioner/deploys/monitoring/tasks/install_monitoring.py @@ -31,6 +31,7 @@ def install_monitoring(): github_exporter_token=github_exporter_token, new_relic_license_key=new_relic_license_key, setup_env=host.data.setup_env, + is_arm=host.get_fact(DebArch) == "arm64", ) nri_prometheus_config = files.put( @@ -61,7 +62,7 @@ def install_monitoring(): server.shell( name="Start the monitoring service", commands=[ - "docker compose -f /opt/monitoring/docker-compose.yml up -d --force-recreate", + "docker compose -f /opt/monitoring/docker-compose.yml up -d --force-recreate --remove-orphans", ], _sudo=True, ) diff --git a/provisioner/test_provisioner.py b/provisioner/test_provisioner.py index a5fd1ca57..59962cd67 100644 --- a/provisioner/test_provisioner.py +++ b/provisioner/test_provisioner.py @@ -135,11 +135,6 @@ def test_otel_collector_health(host): ) -def test_arm_exporter_service(host): - arm_exporter = host.docker("arm_exporter") - assert arm_exporter.is_running - - def test_microk8s_installed(host): assert "microk8s" in host.check_output("snap list") From 96c6e35bcd4f8d5d204515b9b6c5ac99463dcea8 Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:27:17 +0200 Subject: [PATCH 06/12] setup cron for docker prune --- .../deploys/docker/tasks/install_docker.py | 15 +++++++++++++++ provisioner/test_provisioner.py | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/provisioner/deploys/docker/tasks/install_docker.py b/provisioner/deploys/docker/tasks/install_docker.py index dc519c89c..e19716b55 100644 --- a/provisioner/deploys/docker/tasks/install_docker.py +++ b/provisioner/deploys/docker/tasks/install_docker.py @@ -78,3 +78,18 @@ def install_docker(): enabled=True, _sudo=True, ) + + apt.packages( + name="Install cron", + packages=["cron"], + update=True, + cache_time=24 * 60 * 60, + _sudo=True, + ) + + server.crontab( + name="Prune Docker every day at 00:00", + command="docker system prune -a -f --volumes", + minute="0", + hour="0", + ) diff --git a/provisioner/test_provisioner.py b/provisioner/test_provisioner.py index 59962cd67..f99c41bec 100644 --- a/provisioner/test_provisioner.py +++ b/provisioner/test_provisioner.py @@ -67,6 +67,10 @@ def test_user_added_to_docker_group(host): assert "docker" in host.user("ubuntu").groups +def test_docker_system_prune_in_cron(host): + "0 0 * * * docker system prune -a -f --volumes" in host.check_output("crontab -l") + + def test_newrelic_infra_installed(host): newrelic_infra = host.package("newrelic-infra") assert newrelic_infra.is_installed From 9eecd657d96da6e93446d9fa68750ced20335b8a Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:29:26 +0200 Subject: [PATCH 07/12] remove schedule from pull request --- .github/workflows/schedule.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index ef0e2a017..c26f3cb8e 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -2,9 +2,6 @@ name: Schedule on: schedule: - cron: "*/10 * * * *" - pull_request: - branches: - - "*" jobs: provisioner-validate: From 71cdb921e204b6d2a8e581199c80aa3f8e04dad5 Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 14:38:43 +0200 Subject: [PATCH 08/12] setup cron for required reboot --- provisioner/BUILD.bazel | 1 + .../deploys/docker/tasks/install_docker.py | 1 + .../deploys/monitoring/files/reboot.sh | 6 +++++ .../monitoring/tasks/install_monitoring.py | 26 +++++++++++++++++++ provisioner/test_provisioner.py | 4 +++ 5 files changed, 38 insertions(+) create mode 100644 provisioner/deploys/monitoring/files/reboot.sh diff --git a/provisioner/BUILD.bazel b/provisioner/BUILD.bazel index e98da4ddd..66750f76f 100644 --- a/provisioner/BUILD.bazel +++ b/provisioner/BUILD.bazel @@ -31,6 +31,7 @@ pyinfra_run( "deploys/monitoring/files/newrelic-infra.yml.j2", "deploys/monitoring/files/nri-prometheus-config.yaml", "deploys/monitoring/files/otel-collector-config.yaml", + "deploys/monitoring/files/reboot.sh", "deploys/network/files/99_config.yaml", "deploys/teleport/files/teleport.yaml.j2", ], diff --git a/provisioner/deploys/docker/tasks/install_docker.py b/provisioner/deploys/docker/tasks/install_docker.py index e19716b55..47e48f422 100644 --- a/provisioner/deploys/docker/tasks/install_docker.py +++ b/provisioner/deploys/docker/tasks/install_docker.py @@ -92,4 +92,5 @@ def install_docker(): command="docker system prune -a -f --volumes", minute="0", hour="0", + _sudo=True, ) diff --git a/provisioner/deploys/monitoring/files/reboot.sh b/provisioner/deploys/monitoring/files/reboot.sh new file mode 100644 index 000000000..e68491495 --- /dev/null +++ b/provisioner/deploys/monitoring/files/reboot.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# Copied from https://askubuntu.com/questions/829526/ubuntu-server-reboot-command-in-crontab-trigger-only-if-required +if [ -f /var/run/reboot-required ]; then + /sbin/shutdown -r now +fi \ No newline at end of file diff --git a/provisioner/deploys/monitoring/tasks/install_monitoring.py b/provisioner/deploys/monitoring/tasks/install_monitoring.py index 67ba8182c..0f01f8623 100644 --- a/provisioner/deploys/monitoring/tasks/install_monitoring.py +++ b/provisioner/deploys/monitoring/tasks/install_monitoring.py @@ -122,3 +122,29 @@ def install_monitoring(): enabled=True, _sudo=True, ) + + apt.packages( + name="Install cron", + packages=["cron"], + update=True, + cache_time=24 * 60 * 60, + _sudo=True, + ) + + files.put( + name="Copy reboot script", + src="provisioner/deploys/monitoring/files/reboot.sh", + dest="/opt/monitoring/reboot.sh", + _sudo=True, + user="root", + group="root", + mode="0744", + ) + + server.crontab( + name="Reboot at 01:00 when required", + command="/opt/monitoring/reboot.sh", + minute="0", + hour="1", + _sudo=True, + ) diff --git a/provisioner/test_provisioner.py b/provisioner/test_provisioner.py index f99c41bec..acef68601 100644 --- a/provisioner/test_provisioner.py +++ b/provisioner/test_provisioner.py @@ -139,6 +139,10 @@ def test_otel_collector_health(host): ) +def test_required_reboot_in_cron(host): + "0 1 * * * /opt/monitoring/reboot.sh" in host.check_output("crontab -l") + + def test_microk8s_installed(host): assert "microk8s" in host.check_output("snap list") From 567a3054cac1957dd7228c121e5e8a31bb32990f Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 15:23:28 +0200 Subject: [PATCH 09/12] setup log forwarding --- provisioner/BUILD.bazel | 1 + provisioner/deploys/monitoring/files/logging.yml | 10 ++++++++++ .../deploys/monitoring/tasks/install_monitoring.py | 10 ++++++++++ 3 files changed, 21 insertions(+) create mode 100644 provisioner/deploys/monitoring/files/logging.yml diff --git a/provisioner/BUILD.bazel b/provisioner/BUILD.bazel index 66750f76f..d9911f097 100644 --- a/provisioner/BUILD.bazel +++ b/provisioner/BUILD.bazel @@ -28,6 +28,7 @@ pyinfra_run( data = [ "deploys/microk8s/files/cmdline.txt", "deploys/monitoring/files/docker-compose.yml.j2", + "deploys/monitoring/files/logging.yml", "deploys/monitoring/files/newrelic-infra.yml.j2", "deploys/monitoring/files/nri-prometheus-config.yaml", "deploys/monitoring/files/otel-collector-config.yaml", diff --git a/provisioner/deploys/monitoring/files/logging.yml b/provisioner/deploys/monitoring/files/logging.yml new file mode 100644 index 000000000..751d59f60 --- /dev/null +++ b/provisioner/deploys/monitoring/files/logging.yml @@ -0,0 +1,10 @@ +logs: + - name: docker-logs + file: /var/lib/docker/containers/*/*.log # Path to multiple folders and files + logtype: docker + + - name: system-logs + file: /var/log/*/*.log + + - name: systemd-teleport + systemd: teleport diff --git a/provisioner/deploys/monitoring/tasks/install_monitoring.py b/provisioner/deploys/monitoring/tasks/install_monitoring.py index 0f01f8623..4cfd95a5a 100644 --- a/provisioner/deploys/monitoring/tasks/install_monitoring.py +++ b/provisioner/deploys/monitoring/tasks/install_monitoring.py @@ -114,6 +114,16 @@ def install_monitoring(): _sudo=True, ) + files.put( + name="Copy New Relic logging config", + src="provisioner/deploys/monitoring/files/logging.yml", + dest="/etc/newrelic-infra/logging.d/logging.yml", + _sudo=True, + user="root", + group="root", + mode="0644", + ) + systemd.service( name="Enable the New Relic service", service="newrelic-infra.service", From a2a55d6c08eb438ae164d50f0f3270b1cb16cf1a Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 16:24:08 +0200 Subject: [PATCH 10/12] setup containerName log tag for docker containers --- provisioner/BUILD.bazel | 1 + provisioner/deploys/docker/files/daemon.json | 8 ++++++++ provisioner/deploys/docker/tasks/install_docker.py | 13 ++++++++++++- .../deploys/monitoring/files/docker-compose.yml.j2 | 8 ++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 provisioner/deploys/docker/files/daemon.json diff --git a/provisioner/BUILD.bazel b/provisioner/BUILD.bazel index d9911f097..fb404a362 100644 --- a/provisioner/BUILD.bazel +++ b/provisioner/BUILD.bazel @@ -26,6 +26,7 @@ pyinfra_run( "--data install_teleport=True", ], data = [ + "deploys/docker/files/daemon.json", "deploys/microk8s/files/cmdline.txt", "deploys/monitoring/files/docker-compose.yml.j2", "deploys/monitoring/files/logging.yml", diff --git a/provisioner/deploys/docker/files/daemon.json b/provisioner/deploys/docker/files/daemon.json new file mode 100644 index 000000000..07a6b7897 --- /dev/null +++ b/provisioner/deploys/docker/files/daemon.json @@ -0,0 +1,8 @@ +{ + "log-driver": "json-file", + "log-opts": { + "max-size": "10m", + "max-file": "3", + "labels": "containerName" + } +} diff --git a/provisioner/deploys/docker/tasks/install_docker.py b/provisioner/deploys/docker/tasks/install_docker.py index 47e48f422..b755aa59c 100644 --- a/provisioner/deploys/docker/tasks/install_docker.py +++ b/provisioner/deploys/docker/tasks/install_docker.py @@ -1,4 +1,4 @@ -from pyinfra.operations import apt, server, systemd +from pyinfra.operations import apt, server, systemd, files from pyinfra import host from pyinfra.facts.server import LsbRelease from pyinfra.api.deploy import deploy @@ -71,11 +71,22 @@ def install_docker(): _sudo=True, ) + files.put( + name="Copy Docker daemon config", + src="provisioner/deploys/docker/files/daemon.json", + dest="/etc/docker/daemon.json", + _sudo=True, + user="root", + group="root", + mode="0644", + ) + systemd.service( name="Enable the docker service", service="docker.service", running=True, enabled=True, + restarted=True, _sudo=True, ) diff --git a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 index c598b4eb6..92f8f7da4 100644 --- a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 +++ b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 @@ -21,6 +21,8 @@ services: limits: cpus: '0.5' # Maximum of 0.5 CPU cores memory: '100M' # Maximum of 512 megabytes of memory + labels: + containerName: "github_exporter" nri-prometheus: container_name: nri-prometheus @@ -36,6 +38,8 @@ services: limits: cpus: '0.5' # Maximum of 0.5 CPU cores memory: '100M' # Maximum of 512 megabytes of memory + labels: + containerName: "nri-prometheus" otel-collector: container_name: otel-collector @@ -55,6 +59,8 @@ services: limits: cpus: '0.5' # Maximum of 0.5 CPU cores memory: '100M' # Maximum of 512 megabytes of memory + labels: + containerName: "otel-collector" {% if is_arm %} arm_exporter: @@ -68,4 +74,6 @@ services: limits: cpus: '0.5' # Maximum of 0.5 CPU cores memory: '100M' # Maximum of 512 megabytes of memory + labels: + containerName: "arm_exporter" {% endif %} \ No newline at end of file From 9cf17cf35e2640993a3c7a517484263b913c7c9b Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 19:05:46 +0200 Subject: [PATCH 11/12] working proper log tagging of docker logs --- provisioner/BUILD.bazel | 2 ++ .../files/docker-logs-fluentbit.conf | 33 +++++++++++++++++++ .../files/docker-parser-fluentbit.conf | 5 +++ .../deploys/monitoring/files/logging.yml | 5 +-- .../monitoring/tasks/install_monitoring.py | 20 +++++++++++ 5 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf create mode 100644 provisioner/deploys/monitoring/files/docker-parser-fluentbit.conf diff --git a/provisioner/BUILD.bazel b/provisioner/BUILD.bazel index fb404a362..9a9349fe7 100644 --- a/provisioner/BUILD.bazel +++ b/provisioner/BUILD.bazel @@ -29,6 +29,8 @@ pyinfra_run( "deploys/docker/files/daemon.json", "deploys/microk8s/files/cmdline.txt", "deploys/monitoring/files/docker-compose.yml.j2", + "deploys/monitoring/files/docker-logs-fluentbit.conf", + "deploys/monitoring/files/docker-parser-fluentbit.conf", "deploys/monitoring/files/logging.yml", "deploys/monitoring/files/newrelic-infra.yml.j2", "deploys/monitoring/files/nri-prometheus-config.yaml", diff --git a/provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf b/provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf new file mode 100644 index 000000000..8f361cfd6 --- /dev/null +++ b/provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf @@ -0,0 +1,33 @@ +[INPUT] + Name tail + Path /var/lib/docker/containers/*/*.log + Buffer_Max_Size 128k + Mem_Buf_Limit 16384k + Skip_Long_Lines On + Path_Key filePath + Tag docker-logs + DB /var/db/newrelic-infra/newrelic-integrations/logging/fb.db + Parser docker + +[FILTER] + Name record_modifier + Match docker-logs + Record fb.input tail + +[FILTER] + Name nest + Match docker-logs + Operation lift + Nested_under attrs + Add_prefix attrs. + +[FILTER] + Name modify + Match docker-logs + Copy attrs.containerName containerName + +# [OUTPUT] +# Name file +# Format out_file +# Path /tmp/docker-logs +# Match docker-logs diff --git a/provisioner/deploys/monitoring/files/docker-parser-fluentbit.conf b/provisioner/deploys/monitoring/files/docker-parser-fluentbit.conf new file mode 100644 index 000000000..920114b0a --- /dev/null +++ b/provisioner/deploys/monitoring/files/docker-parser-fluentbit.conf @@ -0,0 +1,5 @@ +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L %z \ No newline at end of file diff --git a/provisioner/deploys/monitoring/files/logging.yml b/provisioner/deploys/monitoring/files/logging.yml index 751d59f60..1084c0e19 100644 --- a/provisioner/deploys/monitoring/files/logging.yml +++ b/provisioner/deploys/monitoring/files/logging.yml @@ -1,7 +1,8 @@ logs: - name: docker-logs - file: /var/lib/docker/containers/*/*.log # Path to multiple folders and files - logtype: docker + fluentbit: + config_file: /etc/newrelic-infra/logging.d/docker-logs-fluentbit.conf + parsers_file: /etc/newrelic-infra/logging.d/docker-parser-fluentbit.conf - name: system-logs file: /var/log/*/*.log diff --git a/provisioner/deploys/monitoring/tasks/install_monitoring.py b/provisioner/deploys/monitoring/tasks/install_monitoring.py index 4cfd95a5a..f4b2feffd 100644 --- a/provisioner/deploys/monitoring/tasks/install_monitoring.py +++ b/provisioner/deploys/monitoring/tasks/install_monitoring.py @@ -124,6 +124,26 @@ def install_monitoring(): mode="0644", ) + files.put( + name="Copy New Relic docker fluentbit logging config", + src="provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf", + dest="/etc/newrelic-infra/logging.d/docker-logs-fluentbit.conf", + _sudo=True, + user="root", + group="root", + mode="0644", + ) + + files.put( + name="Copy New Relic docker fluentbit parser", + src="provisioner/deploys/monitoring/files/docker-parser-fluentbit.conf", + dest="/etc/newrelic-infra/logging.d/docker-parser-fluentbit.conf", + _sudo=True, + user="root", + group="root", + mode="0644", + ) + systemd.service( name="Enable the New Relic service", service="newrelic-infra.service", From 29c03d6c01712fbcf84f891ee7cb0cc5fcace644 Mon Sep 17 00:00:00 2001 From: mvgijssel <6029816+mvgijssel@users.noreply.github.com> Date: Thu, 8 Jun 2023 19:16:05 +0200 Subject: [PATCH 12/12] enable propagating env label as well --- provisioner/deploys/docker/files/daemon.json | 2 +- provisioner/deploys/monitoring/files/docker-compose.yml.j2 | 7 ++++--- .../deploys/monitoring/files/docker-logs-fluentbit.conf | 6 ------ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/provisioner/deploys/docker/files/daemon.json b/provisioner/deploys/docker/files/daemon.json index 07a6b7897..00dfef92f 100644 --- a/provisioner/deploys/docker/files/daemon.json +++ b/provisioner/deploys/docker/files/daemon.json @@ -3,6 +3,6 @@ "log-opts": { "max-size": "10m", "max-file": "3", - "labels": "containerName" + "labels": "containerName,env" } } diff --git a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 index 92f8f7da4..0b6db5dab 100644 --- a/provisioner/deploys/monitoring/files/docker-compose.yml.j2 +++ b/provisioner/deploys/monitoring/files/docker-compose.yml.j2 @@ -15,7 +15,6 @@ services: - GITHUB_EXPORTER_COLLECTOR_RUNNERS=false - GITHUB_EXPORTER_COLLECTOR_BILLING=false - GITHUB_EXPORTER_COLLECTOR_ADMIN=false - - SETUP_ENV={{ setup_env }} deploy: resources: limits: @@ -23,6 +22,7 @@ services: memory: '100M' # Maximum of 512 megabytes of memory labels: containerName: "github_exporter" + env: "{{ setup_env }}" nri-prometheus: container_name: nri-prometheus @@ -40,6 +40,7 @@ services: memory: '100M' # Maximum of 512 megabytes of memory labels: containerName: "nri-prometheus" + env: "{{ setup_env }}" otel-collector: container_name: otel-collector @@ -61,14 +62,13 @@ services: memory: '100M' # Maximum of 512 megabytes of memory labels: containerName: "otel-collector" + env: "{{ setup_env }}" {% if is_arm %} arm_exporter: container_name: arm_exporter image: carlosedp/arm_exporter:latest@sha256:c2510142e3824686cba8af75826737a8158b25648e29867e262d26f553de5211 restart: always - environment: - - SETUP_ENV={{ setup_env }} deploy: resources: limits: @@ -76,4 +76,5 @@ services: memory: '100M' # Maximum of 512 megabytes of memory labels: containerName: "arm_exporter" + env: "{{ setup_env }}" {% endif %} \ No newline at end of file diff --git a/provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf b/provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf index 8f361cfd6..ca9c4cb99 100644 --- a/provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf +++ b/provisioner/deploys/monitoring/files/docker-logs-fluentbit.conf @@ -19,12 +19,6 @@ Match docker-logs Operation lift Nested_under attrs - Add_prefix attrs. - -[FILTER] - Name modify - Match docker-logs - Copy attrs.containerName containerName # [OUTPUT] # Name file