blob: e46d2bac144db523ce896ca9111d79f3df818f99 [file] [log] [blame]
input {
beats {
## Add a id to plugin configuration. Can be anything unique.
id => 'beats_plugin'
######## Connection configurations ########
## The port to listen on.
port => {{.Values.service.externalPort}}
## Close Idle clients after the specified time in seconds. Default is 60 seconds
#client_inactivity_timeout => 60
######## Security configurations ########
## Enable encryption. Default false.
#ssl => $filebeat_ssl
## ssl certificate path.
#ssl_certificate => $filebeat_ssl_certificate
## SSL key to use.
#ssl_key => $filebeat_ssl_key
##SSL key passphrase to use.
#ssl_key_passphrase => $filebeat_ssl_key_passphrase
## Value can be any of: none, peer, force_peer.
#ssl_verify_mode => $filebeat_ssl_verify_mode
## Time in milliseconds for an incomplete ssl handshake to timeout. Default is 10000 ms.
#ssl_handshake_timeout => 10000
include_codec_tag => false
}
}
filter {
grok {
break_on_match => false
match => {
"source" => ["/var/log/onap/(?<componentName>[^/]+)/",
"/var/log/onap/%{GREEDYDATA:componentLogFile}"
]
}
}
# Filter for log4j xml events
if "</log4j:event>" in [message] {
#mutate { add_field => { "orgmsg_log4j" => "%{message}" } } # Copy of orginal msg for debug
#Filter to parse xml event and retrieve data
xml {
source => "message"
store_xml => false
remove_namespaces => true
target => "xml_content"
xpath => [ "/event/message/text()", "logmsg" ,
"/event/@logger", "Logger",
"/event/@timestamp", "Timestamp",
"/event/@level", "loglevel",
"/event/@thread", "Thread",
"/event/throwable/text()", "Exceptionthrowable",
"/event/NDC/text()", "NDCs",
"/event/properties/data/@name","mdcname",
"/event/properties/data/@value","mdcvalue"]
}
#Ruby filter to iterate and separate MDCs into documents
ruby {
code => '
$i = 0
$num = 0
if event.get("[mdcname]")
$num = event.get("[mdcname]").length
end
if $num != 0
until $i > $num do
if event.get("[mdcname]").at($i) and event.get("[mdcvalue]").at($i)
event.set(event.get("[mdcname]").at($i), event.get("[mdcvalue]").at($i))
end
$i=$i+1
end
end
'
}
#Validations
if [Exceptionthrowable]
{
mutate {
replace => {
"exceptionmessage" => "%{[Exceptionthrowable]}"
}
}
}
if [NDCs]
{
mutate {
replace => {
"NDC" => "%{[NDCs]}"
}
}
}
mutate {
replace => {
"Logger" =>"%{[Logger]}"
"logmsg" =>"%{[logmsg]}"
"Timestamp" =>"%{[Timestamp]}"
"loglevel" =>"%{[loglevel]}"
"message" => "%{logmsg}"
"Thread" => "%{[Thread]}"
}
remove_field => ["mdcname", "mdcvalue", "logmsg","Exceptionthrowable","NDCs"]
}
if [Timestamp]
{
date {
match => ["Timestamp", "UNIX_MS"]
target => "Timestamp"
}
}
}
# Filter for logback events
else {
#mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug
mutate {
gsub => [
'message', ' = ', '=',
'message', '= ', '=null',
'message', '=\t', '=null ', #This null is followed by a tab
'message', '\t$', '\t'
]
}
# The grok below parses the message field for all current logback patterns used by oom components.
# Example logback pattern: %d{&quot;yyyy-MM-dd'T'HH:mm:ss.SSSXXX&quot;, UTC}|%X{RequestId}|%msg
# Example grok pattern: %{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}
# Use the following command to find all logback patterns in oom directory: find oom -name "logback*xml" -exec grep "property.*attern.*value" {} \;|sort|uniq
grok {
match => {
"message" => [
"%{TIMESTAMP_ISO8601:Timestamp}\\t[%{GREEDYDATA:Thread}]\\t%{GREEDYDATA:loglevel}\\t%{JAVACLASS:Logger}\\t%{GREEDYDATA:MDCs}\\t%{GREEDYDATA:message}",
"%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:TargetVirtualEntity}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}",
"%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}",
"%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Timer}\|\[%{GREEDYDATA:caller}\]\|%{GREEDYDATA:message}",
"%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:ErrorCode}\|%{GREEDYDATA:ErrorDesc}\|%{GREEDYDATA:message}",
"%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ClassName}\|%{GREEDYDATA:message}",
"%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}",
"\[%{TIMESTAMP_ISO8601:Timestamp}\|%{LOGLEVEL:loglevel}\|%{GREEDYDATA:Logger}\|%{GREEDYDATA:Thread}\] %{GREEDYDATA:message}"
]
}
overwrite => ["message"]
}
# The MDCs are key value pairs that are seperated by "," or "\t". Extra space characters are trimmed from the keys and values.
kv {
source => "MDCs"
field_split => ",\t"
trim_key => "\s"
trim_value => "\s"
remove_field => [ "MDCs" ]
}
if (![Timestamp] and [EndTimestamp]) {
mutate { add_field => { "Timestamp" => "%{EndTimestamp}" } }
}
date {
match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ]
target => "Timestamp"
}
mutate {
remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8"]
}
if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") {
#Parse kvps in message
kv {
field_split => "\s"
trim_key => "\s"
trim_value => "\s"
}
#If Request Id is missing and DID is present use as RequestId
if (![RequestId] and [DID] =~ /.+/) {
mutate { add_field => { "RequestId" => "%{DID}" } }
}
}
} #Close else statement for logback events
} #Close filter
output {
elasticsearch {
id => 'onap_es'
######### Security configurations #########
user => "elastic"
password => "changeme"
## The .cer or .pem file to validate the server's certificate
#cacert => $es_cacert
## The keystore used to present a certificate to the server. It can be either .jks or .p12
#keystore => $es_keystore
#keystore_password => $es_keystore_password
## Enable SSL/TLS secured communication to Elasticsearch cluster.
## Default is not set which in that case depends on the protocol specidfied in hosts list
#ssl => $es_ssl
## Option to validate the server's certificate. Default is true
#ssl_certificate_verification => $es_ssl_certificate_verification
## The JKS truststore to validate the server's certificate.
#truststore => $es_truststore
#truststore_password => $es_truststore_password
######### Elasticsearchcluster and host configurations #########
##can specify one or a list of hosts. If sniffing is set, one is enough and others will be auto-discovered
hosts => ["http://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}:{{.Values.config.elasticsearchPort}}"]
## This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. Default is false.
sniffing => true
## How long to wait, in seconds, between sniffing attempts. Default is 5 seconds.
#sniffing_delay => 5
## Set the address of a forward HTTP proxy.
#proxy => $es_proxy
##Use this if you must run Elasticsearch behind a proxy that remaps the root path for the Elasticsearch HTTP API lives
#path => $es_path
######### Elasticsearch request configurations #########
## This setting defines the maximum sized bulk request Logstash will make.
#flush_size => ?
######### Document configurations #########
index => "logstash-%{+YYYY.MM.dd}"
document_type => "logs"
## This can be used to associate child documents with a parent using the parent ID.
#parent => "abcd'
}
}