-
Notifications
You must be signed in to change notification settings - Fork 59
Cloud Pipeline Log Storage
Pavel Silin edited this page Mar 26, 2020
·
15 revisions
GitHub issue: https://github.com/epam/cloud-pipeline/issues/989
This Tutorial goes through a process of manual setup infrastructure for GxP logging store feature (for cloud-pipeline version: c6bdd60).
- Custom docker for cp-search-elk pod:
- install s3-repository plugin (https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html)
- setup s3 credentials to elastic keystore (https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3-client.html)
- install and setup elastic curator, to be able to manage snapshot life cycle with elastic 6.8 OR use elastic 7.x (snapshot life cycle is integrated) (https://www.elastic.co/guide/en/elasticsearch/client/curator/5.8/installation.html)
- Register elastic ingest pipelines (for api and edge logs)
- Create index lifecycle policy, index template and index for security logs
- Register repository for snapshots
- Install Filebeat for api-srv pod
- Install Filebeat for edge pod
- Configure Kibana server pod
FROM elasticsearch:6.8.3
RUN bin/elasticsearch-plugin install repositiry-s3
RUN wget https://packages.elastic.co/curator/5/centos/7/Packages/elasticsearch-curator-5.8.1-1.x86_64.rpm && \
rpm -vi elasticsearch-curator-5.8.1-1.x86_64.rpm && \
rm elasticsearch-curator-5.8.1-1.x86_64.rpm && mkdir ~/.curator
RUN yum update -y && yum install cronie -q -y
ADD curator.yml ~/.curator/curator.yml
ADD curator-actions.yml ~/.curator/curator-actions.yml
ADD get-aws-profile.sh /usr/local/bin/get-aws-profile
RUN chmod +x /usr/local/bin/get-aws-profile
ADD init.sh /init.sh
RUN chmod +x /init.sh
client:
hosts:
- 127.0.0.1
port: 9200
url_prefix:
use_ssl: False
certificate:
client_cert:
client_key:
ssl_no_validate: False
http_auth:
timeout: 30
master_only: False
logging:
loglevel: DEBUG
logfile:
logformat: default
blacklist: ['elasticsearch', 'urllib3']
actions:
1:
action: snapshot
description: >-
Snapshot security_log- prefixed indices older than 1 hour (based on index
creation_date).
options:
allow_ilm_indices: True
repository: ${CP_LOGS_ELASTIC_BACKUP_REPO:-logs_backup_repo}
name: snapshot-%Y%m%d%H
ignore_unavailable: False
include_global_state: True
partial: False
wait_for_completion: True
skip_repo_fs_check: False
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: ${CP_SECURITY_LOGS_ELASTIC_PREFIX:-security_log}
- filtertype: age
source: creation_date
direction: older
unit: days
unit_count: 1
2:
action: delete_snapshots
description: >-
Delete snapshots from the selected repository older than 30 days
(based on creation_date), for 'snapshot-' prefixed snapshots.
options:
repository: ${CP_LOGS_ELASTIC_BACKUP_REPO:-logs_backup_repo}
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: snapshot-
exclude:
- filtertype: age
source: creation_date
direction: older
unit: days
unit_count: 30
#!/bin/bash
cfg_parser ()
{
IFS=$'\n' && ini=( $(<$1) ) # convert to line-array
ini=( ${ini[*]//;*/} ) # remove comments ;
ini=( ${ini[*]//\#*/} ) # remove comments #
ini=( ${ini[*]/\ =/=} ) # remove tabs before =
ini=( ${ini[*]/=\ /=} ) # remove tabs be =
ini=( ${ini[*]/\ /} )
ini=( ${ini[*]/\ *=\ /=} ) # remove anything with a space around =
ini=( ${ini[*]/#[/\}$'\n'cfg.section.} ) # set section prefix
ini=( ${ini[*]/%]/ \(} ) # convert text2function (1)
ini=( ${ini[*]/=/=\( } ) # convert item to array
ini=( ${ini[*]/%/ \)} ) # close array parenthesis
ini=( ${ini[*]/%\\ \)/ \\} ) # the multiline trick
ini=( ${ini[*]/%\( \)/\(\) \{} ) # convert text2function (2)
ini=( ${ini[*]/%\} \)/\}} ) # remove extra parenthesis
ini[0]="" # remove first element
ini[${#ini[*]} + 1]='}' # add the last brace
eval "$(echo "${ini[*]}")" # eval the result
}
# echo a message to standard error (used for messages not intended
# to be parsed by scripts, such as usage messages, warnings or errors)
echo_stderr() {
echo "$@" >&2
}
#
# Parse options
#
display_usage ()
{
echo_stderr "Usage: $0 [--credentials=<path>] [--profile=<name>] [--key|--secret|--session-token]"
echo_stderr " Default --credentials is '~/.aws/credentials'"
echo_stderr " Default --profile is 'default'"
echo_stderr " By default environment variables are generate, e.g."
echo_stderr " source \$($0 --profile=myprofile)"
echo_stderr " You can specify one of --key, --secret, -or --session-token to get just that value, with no line break,"
echo_stderr " FOO_KEY=\$($0 --profile=myprofile --key)"
echo_stderr " FOO_SECRET=\$($0 --profile=myprofile --secret)"
echo_stderr " FOO_SESSION_TOKEN=\$($0 --profile=myprofile --session-token)"
}
for i in "$@"
do
case $i in
--credentials=*)
CREDENTIALS="${i#*=}"
shift # past argument=value
;;
--profile=*)
PROFILE="${i#*=}"
shift # past argument=value
;;
--key)
SHOW_KEY=true
shift # past argument with no value
;;
--secret)
SHOW_SECRET=true
shift # past argument with no value
;;
--session-token)
SHOW_SESSION_TOKEN=true
shift # past argument with no value
;;
--help)
display_usage
exit 0
;;
*)
# unknown option
echo "Unknown option $1"
display_usage
exit 1
;;
esac
done
#
# Check options
#
CREDENTIALS=${CREDENTIALS:-/root/.cloud/credentials}
PROFILE=${PROFILE:-default}
SHOW_KEY=${SHOW_KEY:-false}
SHOW_SECRET=${SHOW_SECRET:-false}
SHOW_SESSION_TOKEN=${SHOW_SESSION_TOKEN:-false}
if [[ "${SHOW_KEY}" = true && "${SHOW_SECRET}" = true ]]; then
echo_stderr "Can only specify one of --key or --secret"
display_usage
exit 2
fi
#
# Parse and display
#
if [[ ! -r "${CREDENTIALS}" ]]; then
echo_stderr "File not found: '${CREDENTIALS}'"
exit 3
fi
cfg_parser "${CREDENTIALS}"
if [[ $? -ne 0 ]]; then
echo_stderr "Parsing credentials file '${CREDENTIALS}' failed"
exit 4
fi
cfg.section.${PROFILE}
if [[ $? -ne 0 ]]; then
echo_stderr "Profile '${PROFILE}' not found"
exit 5
fi
if [[ "${SHOW_KEY}" = false && "${SHOW_SECRET}" = false && "${SHOW_SESSION_TOKEN}" = false ]]; then
echo "export AWS_ACCESS_KEY_ID=${aws_access_key_id}"
echo "export AWS_SECRET_ACCESS_KEY=${aws_secret_access_key}"
echo "export AWS_SESSION_TOKEN=${aws_session_token}"
elif [[ "${SHOW_KEY}" = true ]]; then
echo -n "${aws_access_key_id}"
elif [[ "${SHOW_SECRET}" = true ]]; then
echo -n "${aws_secret_access_key}"
elif [[ "${SHOW_SESSION_TOKEN}" = true ]]; then
echo -n "${aws_session_token}"
else
echo_stderr "Unknown error"
exit 9
fi
exit 0
echo eval(get-aws-profile --key) | bin/elasticsearch-keystore add s3.client.default.access_key
echo eval(get-aws-profile --secret) | bin/elasticsearch-keystore add s3.client.default.secret_key
ulimit -n 65536 && sysctl -w vm.max_map_count=262144 && /usr/local/bin/docker-entrypoint.sh &
ILM_POLICY='{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover": {
"max_age": "1d"
}
}
},
"delete": {
"min_age": "20d",
"actions": {
"delete": {}
}
}
}
}
}'
curl -H 'Content-Type: application/json' -XPUT localhost:9200/_ilm/policy/security_log_policy -d "$ILM_POLICY"
INDEX_TEMPLATE="{
\"index_patterns\": [\"${CP_SECURITY_LOGS_ELASTIC_PREFIX:-security_log}-*\"],
\"settings\": {
\"number_of_shards\": 1,
\"number_of_replicas\": 0,
\"index.lifecycle.name\": \"security_log_policy\",
\"index.lifecycle.rollover_alias\": \"security_log\"
}
}"
curl -H 'Content-Type: application/json' -XPUT localhost:9200/_template/security_log_template -d "$INDEX_TEMPLATE"
INDEX="{
\"aliases\": {
\"${CP_SECURITY_LOGS_ELASTIC_PREFIX:-security_log}\": {}
}
}"
curl -H 'Content-Type: application/json' -XPUT localhost:9200/%3C${CP_SECURITY_LOGS_ELASTIC_PREFIX:-security_log}-%7Bnow%2Fm%7Byyyy.MM.dd%7D%7D-0000001%3E -d "$INDEX_TEMPLATE"
EDGE_PIPELINE='{
"description" : "Log data extraction pipeline from EDGE",
"processors": [
{
"grok": {
"field": "message",
"patterns": [".* \\[SECURITY\\] Application: %{GREEDYDATA:application}; User: %{DATA:user};.*"]
}
},
{
"rename": {
"field": "fields.type",
"target_field": "type"
}
},
{
"rename": {
"field": "fields.service",
"target_field": "service_name"
}
}
]
}'
curl -H 'Content-Type: application/json' -XPUT localhost:9200/_ingest/pipeline/edge -d "$EDGE_PIPELINE"
API_SRV_PIPELINE='{
"description" : "Log data extraction pipeline from API server",
"processors": [
{
"rename": {
"field": "fields.type",
"target_field": "type"
}
},
{
"rename": {
"field": "fields.service",
"target_field": "service_name"
}
}
]
}'
curl -H 'Content-Type: application/json' -XPUT localhost:9200/_ingest/pipeline/api_server -d "$API_SRV_PIPELINE"
LOG_BACKUP_REPO="{
\"type\": \"s3\",
\"settings\": {
\"bucket\": \"${CP_LOG_ELASTIC_BACKUP_REPO:-log_backup_repo}\"
}
}"
curl -H 'Content-Type: application/json' -XPUT localhost:9200/_snapshot/log_backup_repo -d "$LOG_BACKUP_REPO"
cat > /etc/cron.d/curator-cron <<EOL
0 0 * * * curator --config ~/.curator/curator.yml ~/.curator/curator-actions.yml
EOL
chmod 0644 /etc/cron.d/curator-cron
crontab /etc/cron.d/curator-cron
crond
wait
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: cp-log-store-kibana
namespace: default
spec:
replicas: 1
template:
metadata:
namespace: default
labels:
cloud-pipeline/cp-log-store-kibana: "true"
spec:
nodeSelector:
cloud-pipeline/cp-log-store-kibana: "true"
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: cp-log-store-kibana
image: docker.elastic.co/kibana/kibana:6.8.3
securityContext:
privileged: true
imagePullPolicy: "IfNotPresent"
command: ["bash"]
args: ["-c", "/usr/local/bin/kibana-docker"]
env:
- name: ELASTICSEARCH_HOSTS
value: "http://${CP_SEARCH_ELK_ELASTIC_INTERNAL_HOST:-cp-search-elk.default.svc.cluster.local}:${CP_SEARCH_ELK_ELASTIC_INTERNAL_PORT:-30091}"
readinessProbe:
httpGet:
path: /
port: 5601
scheme: HTTP
initialDelaySeconds: 5
periodSeconds: 10
apiVersion: v1
kind: Service
metadata:
labels:
cloud-pipeline/cp-log-store-kibana: "true"
name: cp-log-store-kibana
namespace: default
spec:
ports:
- protocol: TCP
port: ${CP_SEARCH_KIBANA_PORT:-45601}
targetPort: 5601
name: cp-log-store-kibana-port
selector:
cloud-pipeline/cp-log-store-kibana: "true"
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.8.3-x86_64.rpm
rpm -vi filebeat-6.8.3-x86_64.rpm
setup.template.name: "security_log"
setup.template.pattern: "security_log*"
output.elasticsearch:
hosts: ["http://${CP_SEARCH_ELK_ELASTIC_INTERNAL_HOST}:${CP_SEARCH_ELK_ELASTIC_INTERNAL_PORT}"]
pipeline: "api_server"
index: "security_log"
filebeat:
inputs:
- paths:
- /opt/api/logs/security.json
json.keys_under_root: true
json.add_error_key: true
fields:
service: api-srv
type: security
processors:
- drop_fields:
fields: ["instant", "beat", "log", "loggerFqcn", "threadId", "log_context", "offset", "threadPriority", "endOfBatch", "agent", "log.offset", "esc" ]
setup.template.name: "security_log"
setup.template.pattern: "security_log*"
output.elasticsearch:
hosts: ["http://${CP_SEARCH_ELK_ELASTIC_INTERNAL_HOST:-cp-search-elk.default.svc.cluster.local}:${CP_SEARCH_ELK_ELASTIC_INTERNAL_PORT:-30091}"]
pipeline: "edge"
index: "security_log"
filebeat:
inputs:
- paths:
- /etc/nginx/logs/error.log
fields:
service: edge
type: security
include_lines: ['SECURITY']
processors:
- drop_fields:
fields: ["log", "offset", "beat", "agent", "log.offset", "esc" ]