Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add load_succeeded/3-0-1 #1350

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add load_succeeded/3-0-1 #1350

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 192 additions & 0 deletions schemas/com.snowplowanalytics.monitoring.batch/load_succeeded/jsonschema/3-0-1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
{
"$schema": "http://iglucentral.com/schemas/com.snowplowanalytics.self-desc/schema/jsonschema/1-0-0#",
"description": "Schema for a data load succeeding",
"self": {
"vendor": "com.snowplowanalytics.monitoring.batch",
"name": "load_succeeded",
"format": "jsonschema",
"version": "3-0-1"
},

"type": "object",
"properties": {
"shredding": {
"description": "Information about the batch taken from transformation step, isomorphic to shredding_complete schema",
"type": "object",
"properties": {
"base": {
"description": "Blob storage path to the root of the batch",
"type": "string",
"format": "uri",
"maxLength": 1024
},
"compression": {
"description": "File compression type",
"enum": ["GZIP", "NONE"]
},
"typesInfo": {
"description": "Info about schemas used in events and output formats in the respective batch",
"type": "object",
"oneOf": [
{
"properties": {
"transformation": {
"description": "Type of the transformation",
"enum": ["SHREDDED"]
},
"types": {
"description": "Set of Iglu URIs and the format they were shredded into",
"type": "array",
"items": {
"properties": {
"schemaKey": {
"description": "Iglu URI",
"type": "string",
"maxLength": 256
},
"format": {
"description": "File format into which the entities were shredded",
"enum": ["TSV", "JSON"]
},
"snowplowEntity": {
"description": "Type of the self-describing JSON in the event",
"enum": ["SELF_DESCRIBING_EVENT", "CONTEXT"]
}
},
"required": ["schemaKey", "format", "snowplowEntity"],
"additionalProperties": false
}
}
},
"required": ["transformation", "types"],
"additionalProperties": false
},
{
"properties": {
"transformation": {
"description": "Type of the transformation",
"enum": ["WIDEROW"]
},
"fileFormat": {
"description": "Output file format",
"enum": ["JSON", "PARQUET"]
},
"types": {
"description": "Set of Iglu URIs and the format they were shredded into",
"type": "array",
"items": {
"properties": {
"schemaKey": {
"description": "Iglu URI",
"type": "string",
"maxLength": 256
},
"snowplowEntity": {
"description": "Type of the self-describing JSON in the event",
"enum": ["SELF_DESCRIBING_EVENT", "CONTEXT"]
}
},
"required": ["schemaKey", "snowplowEntity"],
"additionalProperties": false
}
}
},
"required": ["transformation", "fileFormat", "types"],
"additionalProperties": false
}
]
},
"timestamps": {
"description": "Set of timestamps associated with the batch",
"type": "object",
"properties": {
"jobStarted": {
"description": "Time when the batch started being shredded",
"type": "string",
"format": "date-time"
},
"jobCompleted": {
"description": "Time when the batch shredding has been finished (and the message being prepared)",
"type": "string",
"format": "date-time"
},
"min": {
"description": "The earliest collector_stamp available in the batch",
"type": ["string", "null"],
"format": "date-time"
},
"max": {
"description": "The latest collector_tstamp available in the batch",
"type": ["string", "null"],
"format": "date-time"
}
},
"required": ["jobStarted", "jobCompleted", "min", "max"]
},
"processor": {
"description": "Identificator of a shredder sent the message",
"type": "object",
"properties": {
"artifact": {
"description": "Name of the artifact",
"type": "string",
"maxLength": 64
},
"version": {
"description": "Semantic Version of the artifact",
"type": "string",
"maxLength": 16
}
},
"required": ["artifact", "version"]
},
"count": {
"description": "Count of events in the batch, null means the count could not be calculated",
"type": ["object", "null"],
"properties": {
"good": {
"description": "Amount of good events in the batch",
"type": "integer",
"minimum": 0
}
}
}
},
"required": ["base", "compression", "typesInfo", "timestamps", "processor"],
"additionalProperties": false
},
"application": {
"description": "Loader's name and version",
"type": "string",
"maxLength": 128
},
"attempt": {
"description": "How many attempts were taken to load the batch",
"type": "integer",
"minimum": 0
},
"loadingStarted": {
"description": "Time when loader started loading (first attempt)",
"type": "string",
"format": "date-time"
},
"loadingCompleted": {
"description": "Time when loader finished loading (should match the ingestion timestamp in manifest)",
"type": "string",
"format": "date-time"
},
"recoveryTableNames": {
"description": "List of recovery table names loaded in this batch",
"type": "array",
"items": {
"type": "string"
}
},
"tags": {
"description": "Set of key value pairs providing additional information",
"type": "object"
}
},
"additionalProperties": false,
"required": ["shredding", "application", "attempt", "loadingStarted", "loadingCompleted", "tags"]
}