Export GCP Stackdriver Log With Filebeat
This is a bash script to configure GCP project to export logs by creating a Pub/Sub sink topic and let filebeat to subscribe to that sink topic by the filebeat google cloud module.
#!/bin/sh
# author: me 😃
# $ bash gcloud-admin.sh -h Required parameters:
# -id|--project-id: gcloud project id
# -svs|--svs-account: gcloud service account name to collect logs
# Optional parameters:
# -h|--help: Print this message
readonly ARGS="$@"
readonly dependencies=( "gcloud" )
processArgs(){
while [[ "$#" -gt 0 ]];
do
key="$1"
case "$key" in
-h|--help)
PRINT_HELP=true
shift
;;
-id|--project-id)
PROJECT_ID="$2"
shift
;;
-svs|--svs-account)
SVS_ACCOUNT="$2"
shift
;;
esac
shift
done
}
checkDependencies() {
local unmet_dependencies=false
for dependency in "${dependencies[@]}" ; do
command -v "${dependency}" >/dev/null 2>&1 || {
echo >&2 "${dependency} required";
unmet_dependencies=true
}
done
if [ "${unmet_dependencies}" = true ] ; then
echo "Please install unmet dependencies above before running."
exit 1
fi
}
printHelp() {
echo " Required parameters:"
echo " -id|--project-id: gcloud project id "
echo " -svs|--svs-account: gcloud service account name to collect logs"
echo " Optional parameters:"
echo " -h|--help: Print this message"
exit 0
}
updateProjectPolicy(){
local project_id="${1}"
local new_policy=${project_id}-policy.yaml
gcloud projects get-iam-policy ${project_id} >> ${new_policy}
cat <<EOF >> ${new_policy}
auditConfigs:
- auditLogConfigs:
- logType: ADMIN_READ
- logType: DATA_READ
- logType: DATA_WRITE
service: allServices
EOF
gcloud projects set-iam-policy ${project_id} ${new_policy}
echo "updated ${project_id} iam policy"
}
enableLoggingAPI(){
local project_id="${1}"
echo "enable Stackdriver Logging API for ${project_id}"
gcloud services enable logging.googleapis.com --project ${project_id}
echo "enable Cloud Pub/Sub API for ${project_id}"
gcloud services enable pubsub.googleapis.com --project ${project_id}
}
createSink(){
local project_id="${1}"
local sink_name=${project_id}-audit-logs
gcloud pubsub topics create --project ${project_id} ${sink_name}
gcloud logging sinks create --project ${project_id} \
${sink_name} \
pubsub.googleapis.com/projects/${project_id}/topics/${sink_name} \
--log-filter="logName:projects/${project_id}/logs/cloudaudit.googleapis.com%2Fdata_access"
gcloud logging sinks describe ${sink_name} --project ${project_id}
}
svsAccountPermission(){
local project_id="${1}"
local sink_name=${project_id}-audit-logs
local svs_account="${2}"
echo "give ${svs_account} access to ${project_id}"
gcloud projects add-iam-policy-binding ${project_id} \
--member serviceAccount:${svs_account} \
--role roles/editor
echo "give the topics's roles/pubsub.editor access to the ${svs_account}"
local new_topic_policy=$(gcloud beta pubsub topics get-iam-policy \
projects/${project_id}/topics/${sink_name} --format json \
| jq --arg svs ${svs_account} '.bindings += [{"members": [$svs],"role": "roles/pubsub.editor"}]')
gcloud beta pubsub topics add-iam-policy-binding ${sink_name} \
--project ${project_id} \
--member serviceAccount:${svs_account} \
--role roles/pubsub.editor
}
main(){
processArgs ${ARGS}
if [ "${PRINT_HELP}" = true ] ; then
printHelp
fi
checkDependencies
updateProjectPolicy "${PROJECT_ID}"
enableLoggingAPI "${PROJECT_ID}"
createSink "${PROJECT_ID}"
svsAccountPermission "${PROJECT_ID}" \
"${SVS_ACCOUNT}"
}
main
Read other posts