Skip to content

Commit

Permalink
Update schema descriptions to match docs (#283)
Browse files Browse the repository at this point in the history 
I used mostly text from the db extractor, but also from some other
extractors. Where I could not find descriptions for the attributes, I
tried rewriting them in the same voice as the docs.
  • Loading branch information
@mathialo
mathialo authored Jan 19, 2024
1 parent 6e228db commit 17dc731
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 72 deletions.
10 changes: 8 additions & 2 deletions schema/base_config.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,18 @@
"description": "Base configuration object",
"properties": {
"version": {
"type": ["string", "integer"],
"type": [
"string",
"integer"
],
"description": "Configuration file version"
},
"type": {
"type": "string",
"enum": ["local", "remote"],
"enum": [
"local",
"remote"
],
"default": "local",
"description": "Configuration file type. Either `local`, meaning the full config is loaded from this file, or `remote`, which means that only the `cognite` section is loaded from this file, and the rest is loaded from extraction pipelines."
},
Expand Down
79 changes: 45 additions & 34 deletions schema/cognite_config.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,48 +2,48 @@
"$id": "cognite_config.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"description": "Configuration for connecting to Cognite Data Fusion (CDF)",
"description": "The cognite section describes which CDF project the extractor will load data into and how to connect to the project.",
"properties": {
"project": {
"type": "string",
"description": "CDF project to connect to"
"description": "Insert the CDF project name."
},
"idp-authentication": {
"type": "object",
"description": "Configure authentication using OIDC tokens",
"description": "The `idp-authentication` section enables the extractor to authenticate to CDF using an external identity provider (IdP), such as Microsoft Entra ID (formerly Azure Active Directory).",
"properties": {
"authority": {
"type": "string",
"default": "https://login.microsoftonline.com/",
"description": "Authority using together with `tenant` to authenticate against azure tenants."
"description": "Insert the authority together with `tenant` to authenticate against Azure tenants."
},
"client-id": {
"type": "string",
"description": "Service principal client id."
"description": "Enter the service principal client id from the IdP."
},
"tenant": {
"type": "string",
"description": "Azure tenant"
"description": "Enter the Azure tenant."
},
"token-url": {
"type": "string",
"description": "URL used to obtain tokens given client credentials."
"description": "Insert the URL to fetch tokens from."
},
"secret": {
"type": "string",
"description": "Service principal client secret"
"description": "Enter the service principal client secret from the IdP."
},
"resource": {
"type": "string",
"description": "Resource parameter passed along with request"
"description": "Resource parameter passed along with token requests."
},
"audience": {
"type": "string",
"description": "Audience parameter passed along with request"
"description": "Audience parameter passed along with token requests."
},
"scopes": {
"type": "array",
"description": "A list of scopes requested for the token",
"description": "Enter a list of scopes requested for the token",
"items": {
"type": "string",
"description": "A scope requested for the token"
Expand All @@ -52,65 +52,76 @@
"min-ttl": {
"type": "integer",
"default": 30,
"description": "Requested minimum time-to-live in seconds for the token"
"description": "Insert the minimum time in seconds a token will be valid. If the cached token expires in less than `min-ttl` seconds, it will be refreshed even if it is still valid."
},
"certificate": {
"type": "object",
"description": "Authenticate with a client certificate",
"unevaluatedProperties": false,
"required": ["path"],
"required": [
"path"
],
"properties": {
"authority-url": {
"type": "string",
"description": "Authentication authority URL"
},
"path": {
"type": "string",
"description": "Path to the .pem or .pfx certificate to be used for authentication"
"description": "Enter the path to the .pem or .pfx certificate to be used for authentication"
},
"password": {
"type": "string",
"description": "Certificate password"
"description": "Enter the password for the key file, if it is encrypted."
}
}
}
},
"oneOf": [
{ "required": ["tenant", "secret"] },
{ "required": ["token-url", "secret"] },
{ "required": ["certificate"] }
{
"required": [
"tenant",
"secret"
]
},
{
"required": [
"token-url",
"secret"
]
},
{
"required": [
"certificate"
]
}
],
"unevaluatedProperties": false,
"required": ["client-id"]
"required": [
"client-id"
]
},
"data-set": {
"description": "Data set the extractor writes to",
"description": "Enter a data set the extractor should write data into",
"$ref": "either_id.schema.json"
},
"extraction-pipeline": {
"description": "Extraction pipeline used for remote config and reporting status",
"description": "Enter the extraction pipeline used for remote config and reporting statuses",
"$ref": "either_id.schema.json"
},
"security-categories": {
"description": "Security category to attach to any data written by the extractor (where relevant)",
"type": "array",
"items": {
"type": "integer"
}
},
"host": {
"type": "string",
"default": "https://api.cognitedata.com",
"description": "Cognite service URL."
"description": "Insert the base URL of the CDF project."
},
"timeout": {
"type": "integer",
"description": "Timeout on requests to CDF, in seconds",
"description": "Enter the timeout on requests to CDF, in seconds.",
"default": 30
},
"external-id-prefix": {
"type": "string",
"description": "Prefix on externalId used when creating CDF resources"
"description": "Prefix on external ID used when creating CDF resources"
},
"connection": {
"type": "object",
Expand All @@ -123,7 +134,7 @@
},
"status-forcelist": {
"type": "string",
"description": "HTTP status codes to retry. Defaults to {429, 502, 503, 504}"
"description": "HTTP status codes to retry. Defaults to 429, 502, 503 and 504"
},
"max-retries": {
"type": "integer",
Expand All @@ -145,10 +156,10 @@
"default": 50,
"description": "The maximum number of connections which will be kept in the SDKs connection pool."
},
"disable_ssl": {
"disable-ssl": {
"type": "boolean",
"default": false,
"description": "Whether or not to disable SSL."
"description": "Whether or not to disable SSL verification."
},
"proxies": {
"type": "object",
Expand Down
34 changes: 24 additions & 10 deletions schema/logging_config.schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,54 @@
"$id": "logging_config.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"description": "Configuration for logging to file or console",
"description": "The optional `logger` section sets up logging to a console and files.",
"properties": {
"console": {
"type": "object",
"description": "Configuration for logging to the console",
"description": "Include the console section to enable logging to a standard output, such as a terminal window.",
"unevaluatedProperties": false,
"properties": {
"level": {
"type": "string",
"description": "Minimum level of messages sent to console",
"enum": ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "FATAL", "CRITICAL"],
"description": "Select the verbosity level for console logging. Valid options, in decreasing verbosity levels, are `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.",
"enum": [
"DEBUG",
"INFO",
"WARNING",
"ERROR",
"CRITICAL"
],
"default": "INFO"
}
}
},
"file": {
"type": "object",
"description": "Configuration for logging to a rotating file",
"description": "Include the file section to enable logging to a file. The files are rotated daily.",
"unevaluatedProperties": false,
"required": ["path"],
"required": [
"path"
],
"properties": {
"level": {
"type": "string",
"description": "Minimum level of messages sent to file",
"enum": ["NOTSET", "DEBUG", "INFO", "WARNING", "ERROR", "FATAL", "CRITICAL"],
"description": "Select the verbosity level for file logging. Valid options, in decreasing verbosity levels, are `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL`.",
"enum": [
"DEBUG",
"INFO",
"WARNING",
"ERROR",
"CRITICAL"
],
"default": "INFO"
},
"path": {
"type": "string",
"description": "Path to log file"
"description": "Insert the path to the log file."
},
"retention": {
"type": "integer",
"description": "Number of log files to keep. Files are rotated daily",
"description": "Specify the number of days to keep logs for.",
"default": 7
}
}
Expand Down
32 changes: 16 additions & 16 deletions schema/metrics_config.schema.json
View file
Original file line number Diff line number Diff line change
@@ -1,43 +1,43 @@
{
"$id": "metrics_config.schema.json",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"description": "Configuration for publishing prometheus metrics, supports pushing to CDF timeseries instead",
"description": "The `metrics` section describes where to send metrics on extractor performance for remote monitoring of the extractor. We recommend sending metrics to a [Prometheus pushgateway](https://prometheus.io/), but you can also send metrics as time series in the CDF project.",
"type": "object",
"properties": {
"push-gateways": {
"type": "array",
"description": "List of prometheus push gateway destinations",
"description": "List of prometheus pushgateway configurations",
"items": {
"type": "object",
"description": "Configuration for pushing to a prometheus PushGateway",
"description": "The push-gateways sections contain a list of metric destinations.",
"unevaluatedProperties": false,
"properties": {
"host": {
"type": "string",
"description": "PushGateway host"
"description": "Enter the address of the host to push metrics to."
},
"job-name": {
"type": "string",
"description": "Job name, uniquely identifies this extractor in prometheus"
"description": "Enter the value of the `exported_job` label to associate metrics with. This separates several deployments on a single pushgateway, and should be unique."
},
"username": {
"type": "string",
"description": "PushGateway username"
"description": "Enter the credentials for the pushgateway."
},
"password": {
"type": "string",
"description": "PushGateway password"
"description": "Enter the credentials for the pushgateway."
},
"clear-after": {
"type": [
"null",
"integer"
],
"description": "Clear metrics after this many seconds when the extractor stops. Default is disabled"
"description": "Enter the number of seconds to wait before clearing the pushgateway. When this parameter is present, the extractor will stall after the run is complete before deleting all metrics from the pushgateway. The recommended value is at least twice that of the scrape interval on the pushgateway. This is to ensure that the last metrics are gathered before the deletion. Default is disabled."
},
"push-interval": {
"type": "integer",
"description": "Interval in seconds between each push to prometheus",
"description": "Enter the interval in seconds between each push.",
"default": 30
}
}
Expand All @@ -53,30 +53,30 @@
"properties": {
"external-id-prefix": {
"type": "string",
"description": "Prefix on external id on created metrics"
"description": "Prefix on external ID used when creating CDF time series to store metrics."
},
"asset-name": {
"type": "string",
"description": "Name of created root asset for the metrics"
"description": "Enter the name for a CDF asset that will have all the metrics time series attached to it."
},
"asset-external-id": {
"type": "string",
"description": "ExternalId of created root asset for the metrics"
"description": "Enter the external ID for a CDF asset that will have all the metrics time series attached to it."
},
"push-interval": {
"type": "integer",
"description": "Interval in seconds between each push to CDF",
"description": "Enter the interval in seconds between each push to CDF",
"default": 30
},
"data-set": {
"type": "string",
"description": "Data set the metrics timeseries created under"
"description": "Data set the metrics will be created under",
"$ref": "either_id.schema.json"
}
}
},
"server": {
"type": "object",
"description": "Configure a port to expose prometheus metrics for scraping",
"description": "The extractor can also be configured to expose a HTTP server with prometheus metrics for scraping",
"unevaluatedProperties": false,
"properties": {
"host": {
Expand Down
Loading

0 comments on commit 17dc731

Please sign in to comment.