I am trying to configure logstash to read the logs that are generated in my Amazon S3 but were not successful. The following are the details:
- I installed logstash for ec2 instance
- My logs - all gz files in s3 bucket
- The conf file looks like this:
input {
s3 {
access_key_id => "MY_ACCESS_KEY_ID"
bucket => "MY_BUCKET"
region => "MY_REGION"
secret_access_key => "MY_SECRET_ACESS_KEY"
prefix => "/"
type => "s3"
add_field => {source => gzfiles}
}
}
filter {
if [type] == "s3" {
csv {
columns => ["date", "time", "x-edge-location", "sc-bytes", "c-ip", "cs-method", "Host", "cs-uri-stem", "sc-status", "Referer", "User-Agent", "cs-uri-query", "Cookie", "x-edge-result-type", "x-edge-request-id"]
}
}
if ([message] = ~ / ^ # /) {
drop {}
}
}
output {
elasticsearch {
host => "ELASTICSEARCH_URL" protocol => "http"
}
}
amazon-s3 amazon-web-services logstash logstash-configuration logstash-file
user1697058
source share