Logstash Grok查找_grokparsefailure的源



这就是我试图找到_grokparsefailure来源的方法

echo'<30>2022:10:24-15:08:28 utm-1 httpproxy[28052]:id=";0003";严重性=";信息";sys=";SecureWeb";sub=";http";name=";http访问";动作=";通过";method=";CONNECT";srcip=";10.134.240.227";dstip="user="group="ad_domain="statuscode=";407〃;cached=";0";profile=";默认HTTPProfile(默认Web过滤器配置文件(";filteraction=";((";size=";2505〃;request=";0x20cc5800";url=";https://vcsa.vmware.com/"referer="error="authtime=";1〃;dnstime=";0";aptptime=";0";cattime=";0";avscantime=";0";fullreqtime=";201〃;设备=";3〃;auth=";1〃;ua=";Apache HttpClient/4.5.13(Java/1.8.0_321(";exceptions="异常";url,ssl,certcheck,certdate"'|/usr/share/logstash/bin/logstash-f/etc/logstash/conf.d/10-utm.stdin.test--调试

10-utm.stdin.test:

# This is a Grok Pattern form Sophos SG UTM Log's
input { stdin { } }
output { stdout { codec => rubydebug } }
filter {
grok {
add_tag => [ "Line7" ]
}  
if "sophos-utm" in [tags] {
grok {
add_tag => [ "Line11" ]
break_on_match => true
match => ["message",'<%{INT:utm_syslog_pri}>(?:%{YEAR}):(%{MONTHNUM}):(?:%{MONTHDAY})-(?:%{HOUR}):(?:%{MINUTE}):(?:%{SECOND}) (?:%{SYSLOGHOST}) (?:%{SYSLOGPROG}): id="%{INT:utm_id}" .* sub="%{DATA:utm_sub}"']
match => ["message",'<%{INT:utm_syslog_pri}>(?:%{YEAR}):(%{MONTHNUM}):(?:%{MONTHDAY})-(?:%{HOUR}):(?:%{MINUTE}):(?:%{SECOND}) (?:%{SYSLOGHOST}) (?:%{SYSLOGPROG}): id="%{INT:utm_id}"']
match => ["message",'<%{INT:utm_syslog_pri}>(?:%{YEAR}):(%{MONTHNUM}):(?:%{MONTHDAY})-(?:%{HOUR}):(?:%{MINUTE}):(?:%{SECOND}) (?:%{SYSLOGHOST}) (?:%{SYSLOGPROG}): [%{DATA:utm_security2}:.*]']
match => ["message",'<%{INT:utm_syslog_pri}>(?:%{YEAR}):(%{MONTHNUM}):(?:%{MONTHDAY})-(?:%{HOUR}):(?:%{MINUTE}):(?:%{SECOND}) (?:%{SYSLOGHOST}) (?:%{SYSLOGPROG}):']

overwrite => ["MONTHNUM", "MONTHDAY", "HOUR", "MINUTE", "SECOND", "SYSLOGHOST", "SYSLOGPROG", "id"]
#tag_on_failure => []
}
grok {
add_tag => [ "Line22" ]
}
if "_grokparsefailure" in [tags] {
grok {
add_tag => [ "Line26" ]
}
}
if [process][name] == "httpd" {
if [utm_security2] {
grok {
match => ["message",'.*[client %{IP:utm_srcip}].* [msg "%{DATA:utm_msg}"].* [data "%{DATA:utm_data}"].* [severity "%{LOGLEVEL:utm_severity}"].* [tag "OWASP_TOP_10/%{DATA:utm_owasptop10}"].* [hostname "%{DATA:utm_hostname}"].* [uri "%{DATA:utm_uri}"]']
}
if [utm_owasptop10] == "A1" {
mutate {
replace => ["utm_owasptop10","Injection (SQL,OS,XXE,LDAP)"]
}
}
if [utm_owasptop10] == "A2" {
mutate {
replace => ["utm_owasptop10","Broken Authentification and Session Management"]
}
}
if [utm_owasptop10] == "A3" {
mutate {
replace => ["utm_owasptop10","Cross-Site Scripting"]
}
}
if [utm_owasptop10] == "A4" {
mutate {
replace => ["utm_owasptop10","Broken Access Control"]
}
}
if [utm_owasptop10] == "A5" {
mutate {
replace => ["utm_owasptop10","Security Misconfiguration"]
}
}
if [utm_owasptop10] == "A6" {
mutate {
replace => ["utm_owasptop10","Sensitive Data Exposure"]
}
}
if [utm_owasptop10] == "A7" {
mutate {
replace => ["utm_owasptop10","Insufficient Attack Protection"]
}
}
if [utm_owasptop10] == "A8" {
mutate {
replace => ["utm_owasptop10","Cross-Site Request Forgery (CSRF)"]
}
}
if [utm_owasptop10] == "A9" {
mutate {
replace => ["utm_owasptop10","Using Component with Know Vulnerabilities"]
}
}
if [utm_owasptop10] == "A10" {
mutate {
replace => ["utm_owasptop10","Underprotected APIs (SOAP,REST,RPC,GWT)"]
}
}
} else {
grok {
match => ["message",'.* srcip="%{IP:utm_srcip}" localip="%{IP:utm_localip}" size="%{INT:utm_size}" user="%{DATA:utm_user}" host="%{IP:utm_host}" method="%{DATA:utm_method}" statuscode="%{INT:utm_statuscode}" reason="%{DATA:utm_reason}" extra="%{DATA:utm_extra}" exceptions="%{DATA:utm_exceptions}" time="%{INT:utm_time}" url="%{DATA:utm_url}" server="%{DATA:utm_server}" port="%{DATA:utm_port}" query="%{DATA:utm_query}" referer="%{DATA:utm_referer}"']
}
}
# Find the GeoLite Database here : https://dev.maxmind.com/geoip/geoip2/geolite2/
geoip {
source => "utm_srcip"
target => "geoip"
database => "/etc/logstash/conf.d/data/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
if [process][name] == "httpproxy" {
grok {
#        match => ["message",'.* (?:severity="%{LOGLEVEL:utm_severity}") (?:sys="%{DATA:utm_sys}") (?:sub="%{DATA}") (?:name="%{DATA:utm_name}") (?:action="%{DATA:utm_action}") ?(?:method="%{DATA:utm_method}?")? (?:srcip="%{IP:utm_srcip}?") (?:dstip="%{IP:utm_dstip}?") (?:user="%{DATA:utm_user}?") (?:group="%{DATA:utm_group}?") (?:ad_domain="%{DATA:utm_ad_domain}?") (?:statuscode="%{INT:utm_satuscode}?") (?:cached="%{INT:utm_cached}?") (?:profile="%{DATA:utm_profil}?") (?:filteraction="%{DATA:utm_filteraction}?") (?:size="%{INT:utm_size}?") (?:request="%{DATA:utm_request}?") (?:url="%{DATA:utm_url}?") ?(?:referer="%{DATA:utm_referer}?") ?(?:error="%{DATA:utm_error}?") ?(?:authtime="%{DATA:utm_authtime}?") ?(?:dnstime="%{INT:utm_dnstime}?") ?(?:aptptime="%{INT:utm_aptptime}?") ?(?:cattime="%{INT:utm_cattime}?") ?(?:avscantime="%{INT:utm_avscantime}?")? ?(?:fullreqtime="%{INT:utm_fullreqtime}?")? ?(?:device="%{INT:utm_device}?")? ?(?:auth="%{INT:utm_auth}?")? ?(?:ua="%{DATA:utm_ua}?")? ?(?:exceptions="%{DATA:utm_exceptions}?")? ?(?:application="%{DATA:utm_application}?")? ?(?:app-id="%{INT:utm_app-id}?")? ?(?:category="%{DATA:utm_category}?")? ?(?:reputation="%{DATA:utm_reputation}?")? ?(?:categoryname="%{DATA:utm_categoryname}?")? ?(?:sandbox="%{DATA:utm_sandbox}?")?']
match => ["message",'(?:severity="%{LOGLEVEL:utm_severity}") (?:sys="%{DATA:utm_sys}") (?:sub="%{DATA:utm_sub}") (?:name="%{DATA:utm_name}") (?:action="%{DATA:utm_action}") ?(?:method="%{DATA:utm_method}?")? (?:srcip="%{IP:utm_srcip}?") (?:dstip="%{IP:utm_dstip}?") (?:user="%{DATA:utm_user}?") (?:group="%{DATA:utm_group}?") (?:ad_domain="%{DATA:utm_ad_domain}?") (?:statuscode="%{INT:utm_satuscode}?") (?:cached="%{INT:utm_cached}?") (?:profile="%{DATA:utm_profil}?") (?:filteraction="%{DATA:utm_filteraction}?") (?:size="%{INT:utm_size}?") (?:request="%{DATA:utm_request}?") (?:url="%{URI:utm_url}?") ?(?:referer="%{URI:utm_referer}?") ?(?:error="%{DATA:utm_error}?") ?(?:authtime="%{DATA:utm_authtime}?") ?(?:dnstime="%{INT:utm_dnstime}?") ?(?:aptptime="%{INT:utm_aptptime}?") ?(?:cattime="%{INT:utm_cattime}?") ?(?:avscantime="%{INT:utm_avscantime}?")? ?(?:fullreqtime="%{INT:utm_fullreqtime}?")? ?(?:device="%{INT:utm_device}?")? ?(?:auth="%{INT:utm_auth}?")? ?(?:ua="%{DATA:utm_ua}?")? ?(?:exceptions="%{DATA:utm_exceptions}?")? ?(?:application="%{DATA:utm_application}?")? ?(?:app-id="%{INT:utm_app-id}?")? ?(?:category="%{DATA:utm_category}?")? ?(?:reputation="%{DATA:utm_reputation}?")? ?(?:categoryname="%{DATA:utm_categoryname}?")? ?(?:sandbox="%{DATA:utm_sandbox}?")? ?(?:country="%{DATA:utm_country}?")? ?(?:content-type="%{DATA:utm_content_type}?")?']
match => ['utm_url','.(?<utm_domain>[^.]+.[^.]+)$']
add_tag => [ "Line108" ]
}
if [utm_categoryname] == "Search Engines" {
grok {
match => ["utm_url", '.*q=(?<utm_search>[^$#&]+)(|[$#&].*)']
}
urldecode {
field => "utm_search"
}
mutate {
gsub => ["utm_search","+"," "]
}
}
if "_grokparsefailure" in [tags] {
grok {
add_tag => [ "Line123" ]
}
}      
#dns {
#    reverse => ["utm_srcip"]
#    action => "replace"
#    }
}
if "_grokparsefailure" in [tags] {
grok {
add_tag => [ "Line134" ]
}
}
if [process][name] == "snort" {
grok {
match => ["message", '.* severity="%{LOGLEVEL:utm_severity}" sys="%{DATA:utm_sys}" sub="%{DATA:utm_sub}" name="%{DATA:utm_name}" action="%{DATA:utm_action}" reason="%{DATA:utm_reason}" srcip="%{IP:utm_srcip}" dstip="%{DATA:utm_dstip}" srcport="%{INT:utm_srcport}" dstport="%{INT:utm_dstport}" sid="%{DATA:utm_sid_snort}" class="%{DATA:utm_class}"']
}
geoip {
source => "utm_srcip"
target => "geoip"
database => "/etc/logstash/conf.d/data/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
if [process][name] == "ulogd" {
if [utm_sub] == "ips" {
grok {
match => ["message", '.* severity="%{LOGLEVEL:utm_severity}" sys="%{DATA:utm_sys}" sub="%{DATA:utm_sub}" name="%{DATA:utm_name}" (action="%{DATA:utm_action}") reason="%{DATA:utm_reason}" group="%{DATA:utm_group}" srcip="%{IP:utm_srcip}" dstip="%{DATA:utm_dstip}" proto="%{INT:utm_proto}" srcport="%{INT:utm_srcport}" dstport="%{INT:utm_dstport}" sid="%{DATA:utm_sid_snort}" class="%{DATA:utm_class}"']
match => ["message",'.* severity="%{LOGLEVEL:utm_severity}" sys="%{DATA:utm_sys}" sub="%{DATA:utm_sub}" name="%{DATA:utm_name}" action="%{DATA:utm_action}" fwrule="%{INT:utm_fwrule}" initf="%{DATA:utm_initf}" srcmac="%{MAC:utm_srcmac}" dstmac="%{MAC:utm_dstmac}" srcip="%{IP:utm_srcip}" dstip="%{IP:utm_dstip}".* (srcport="%{INT:utm_srcport}")?.* (dstport="%{INT:utm_dstport}")?']
}
}
if [utm_sub] == "packetfilter" {
grok {
match => ["message", '.* sys="%{DATA:utm_sys}" sub="%{DATA}" name="%{DATA:utm_name}" action="%{DATA:utm_action}" fwrule="%{INT:utm_fwrule}" ?(initf="%{DATA:utm_initf}")? ?(outitf="%{DATA:utm_outif}")? ?(srcmac="%{MAC:utm_srcmac}")? ?(dstmac="%{MAC:utm_dstmac}")? srcip="%{IP:utm_srcip}" dstip="%{IP:utm_dstip}" proto="%{INT:utm_protocol}" length="%{INT:utm_ulogd_pkglength}" tos="%{DATA:utm_ulogd_tos}" prec="%{DATA:utm_ulogd_prec}" ttl="%{INT:utm_ttl}" srcport="%{INT:utm_srcport}" dstport="%{INT:utm_dstport}" ?(tcpflags="%{DATA:utm_tcpflags}")? ?(info="%{DATA:utm_info}")?']
}
}
}
if [process][name] == "awelogger" {
grok {
match => ["message", '.* severity="%{LOGLEVEL:utm_severity}" sys="%{DATA:utm_sys}" sub="%{DATA}" name="%{DATA:utm_name}" ssid="%{DATA:utm_ssid}".* bssid="%{MAC:utm_bssid}"']
}
}
if [process][name] == "awed" {
grok {
match => ["message", '.* [{%DATA:utm_ap}] .* from %{IP:utm_srcip}:%{INT:utm_port}']
}
}
#if [process][name] == "hostapd" {
#  grok {
#   match => ["message", '.*: {%DATA:utm_intf}: .* from %{IP:utm_srcip}:%{INT:utm_port}']
#  }
#}
if [process][name] in ["openvpn", "pppd-l2tp"] {
grok {
match => ["message", '.* severity="%{LOGLEVEL:utm_severity}" sys="%{DATA:utm_sys}" sub="%{DATA}" event="%{DATA:utm_event}" username="%{DATA:utm_username}" variant="%{DATA:utm_variant}" srcip="%{IP:utm_srcip}".* virtual_ip="%{IP:utm_virtual_ip}"']
}
geoip {
source => "utm_srcip"
target => "geoip"
database => "/etc/logstash/conf.d/data/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
if [process][name] == "pluto" {
grok {
match => ["message", '.* severity="%{LOGLEVEL:utm_severity}" sys="%{DATA:utm_sys}" sub="%{DATA}" event="%{DATA:utm_event}" variant="%{DATA:utm_variant}" connection="%{DATA:utm_connection}" address="%{IP:utm_address}" local_net="%{DATA:utm_local_net}" remote_net="%{DATA:utm_remote_net}"']
}
}

if [process][name] == "afcd" {
grok {
match => ["message", '.* severity="%{LOGLEVEL:utm_severity}" sys="%{DATA:utm_sys}" sub="%{DATA}" name="%{DATA:utm_name}" srcip="%{IP:utm_srcip}" dstip="%{IP:utm_dstip}" .* threatname="%{DATA:utm_threatname}" .* host="%{DATA:utm_host}" .* action="%{DATA:utm_action}"']
}
}
mutate {
replace => ["type","sophosutm"]
add_field => ["utm_size_number","%{utm_size}"]
}
if "_grokparsefailure" in [tags] {
grok {
add_tag => [ "Line222" ]
}
}    
mutate {
convert => {"utm_size_number" => "integer"}
}
if "_grokparsefailure" in [tags] {
grok {
add_tag => [ "Line230" ]
}
}        
}
}

我想这就是标签被添加的地方,但我不知道为什么

[DEBUG] 2022-10-24 14:01:31.531 [[main]>worker1] grok - Running grok filter {:event=>{"@version"=>"1", "@timestamp"=>2022-10-24T14:01:31.415658841Z, "message"=>"<30>2022:10:24-15:08:28 utm-1 httpproxy[28052]: id="0003" severity="info" sys="SecureWeb" sub="http" name="http access" action="pass" method="CONNECT" srcip="10.134.240.227" dstip="" user="" group="" ad_domain="" statuscode="407" cached="0" profile="REF_DefaultHTTPProfile (Default Web Filter Profile)" filteraction=" ()" size="2505" request="0x20cc5800" url="https://vcsa.vmware.com/" referer="" error="" authtime="1" dnstime="0" aptptime="0" cattime="0" avscantime="0" fullreqtime="201" device="3" auth="1" ua="Apache-HttpClient/4.5.13 (Java/1.8.0_321)" exceptions="url,ssl,certcheck,certdate"", "host"=>{"hostname"=>"elk-1-test"}, "event"=>{"original"=>"<30>2022:10:24-15:08:28 utm-1 httpproxy[28052]: id="0003" severity="info" sys="SecureWeb" sub="http" name="http access" action="pass" method="CONNECT" srcip="10.134.240.227" dstip="" user="" group="" ad_domain="" statuscode="407" cached="0" profile="REF_DefaultHTTPProfile (Default Web Filter Profile)" filteraction=" ()" size="2505" request="0x20cc5800" url="https://vcsa.vmware.com/" referer="" error="" authtime="1" dnstime="0" aptptime="0" cattime="0" avscantime="0" fullreqtime="201" device="3" auth="1" ua="Apache-HttpClient/4.5.13 (Java/1.8.0_321)" exceptions="url,ssl,certcheck,certdate""}}}
[DEBUG] 2022-10-24 14:01:31.532 [[main]>worker1] grok - Event now:  {:event=>{"@version"=>"1", "message"=>"<30>2022:10:24-15:08:28 utm-1 httpproxy[28052]: id="0003" severity="info" sys="SecureWeb" sub="http" name="http access" action="pass" method="CONNECT" srcip="10.134.240.227" dstip="" user="" group="" ad_domain="" statuscode="407" cached="0" profile="REF_DefaultHTTPProfile (Default Web Filter Profile)" filteraction=" ()" size="2505" request="0x20cc5800" url="https://vcsa.vmware.com/" referer="" error="" authtime="1" dnstime="0" aptptime="0" cattime="0" avscantime="0" fullreqtime="201" device="3" auth="1" ua="Apache-HttpClient/4.5.13 (Java/1.8.0_321)" exceptions="url,ssl,certcheck,certdate"", "@timestamp"=>2022-10-24T14:01:31.415658841Z, "host"=>{"hostname"=>"elk-1-test"}, "tags"=>["_grokparsefailure"], "event"=>{"original"=>"<30>2022:10:24-15:08:28 utm-1 httpproxy[28052]: id="0003" severity="info" sys="SecureWeb" sub="http" name="http access" action="pass" method="CONNECT" srcip="10.134.240.227" dstip="" user="" group="" ad_domain="" statuscode="407" cached="0" profile="REF_DefaultHTTPProfile (Default Web Filter Profile)" filteraction=" ()" size="2505" request="0x20cc5800" url="https://vcsa.vmware.com/" referer="" error="" authtime="1" dnstime="0" aptptime="0" cattime="0" avscantime="0" fullreqtime="201" device="3" auth="1" ua="Apache-HttpClient/4.5.13 (Java/1.8.0_321)" exceptions="url,ssl,certcheck,certdate""}}}
{
"@version" => "1",
"message" => "<30>2022:10:24-15:08:28 utm-1 httpproxy[28052]: id="0003" severity="info" sys="SecureWeb" sub="http" name="http access" action="pass" method="CONNECT" srcip="10.134.240.227" dstip="" user="" group="" ad_domain="" statuscode="407" cached="0" profile="REF_DefaultHTTPProfile (Default Web Filter Profile)" filteraction=" ()" size="2505" request="0x20cc5800" url="https://vcsa.vmware.com/" referer="" error="" authtime="1" dnstime="0" aptptime="0" cattime="0" avscantime="0" fullreqtime="201" device="3" auth="1" ua="Apache-HttpClient/4.5.13 (Java/1.8.0_321)" exceptions="url,ssl,certcheck,certdate"",
"@timestamp" => 2022-10-24T14:01:31.415658841Z,
"host" => {
"hostname" => "elk-1-test"
},
"tags" => [
[0] "_grokparsefailure"
],
"event" => {
"original" => "<30>2022:10:24-15:08:28 utm-1 httpproxy[28052]: id="0003" severity="info" sys="SecureWeb" sub="http" name="http access" action="pass" method="CONNECT" srcip="10.134.240.227" dstip="" user="" group="" ad_domain="" statuscode="407" cached="0" profile="REF_DefaultHTTPProfile (Default Web Filter Profile)" filteraction=" ()" size="2505" request="0x20cc5800" url="https://vcsa.vmware.com/" referer="" error="" authtime="1" dnstime="0" aptptime="0" cattime="0" avscantime="0" fullreqtime="201" device="3" auth="1" ua="Apache-HttpClient/4.5.13 (Java/1.8.0_321)" exceptions="url,ssl,certcheck,certdate""
}
}
[DEBUG] 2022-10-24 14:01:31.671 [[main]-pipeline-manager] javapipeline - Shutdown waiting for worker thread {:pipeline_id=>"main", :thread=>"#<LogStash::WorkerLoopThread:0x57397be3 dead>"}
[DEBUG] 2022-10-24 14:01:31.671 [[main]-pipeline-manager] javapipeline - Shutdown waiting for worker thread {:pipeline_id=>"main", :thread=>"#<LogStash::WorkerLoopThread:0x56f4bfea dead>"}
[DEBUG] 2022-10-24 14:01:31.672 [[main]-pipeline-manager] grok - Closing {:plugin=>"LogStash::Filters::Grok"}
[DEBUG] 2022-10-24 14:01:31.673 [[main]-pipeline-manager] pluginmetadata - Removing metadata for plugin e489c8cb24e095cea22f0d0ea0836e8556029d1f12126d19d9dfbf7ecd8c43d1
[DEBUG] 2022-10-24 14:01:31.673 [[main]-pipeline-manager] grok - Closing {:plugin=>"LogStash::Filters::Grok"}

编辑:谢谢@Badger这对我来说是有效的,添加一个简单的标签来调试

if "_grokparsefailure" in [tags] {
grok {
match => ["message",'%{GREEDYDATA}']
add_tag => [ "Line134-Fail" ]
remove_tag => ["_grokparsefailure"]
}
}

您的事件没有任何标记,因此您的整个配置相当于

grok { add_tag => [ "Line7" ] }

它确实增加了一个"_grokparsefailure";。发生这种情况的原因是match函数默认为失败,如果给定一个空的匹配哈希进行检查,则返回false。

相关内容

  • 没有找到相关文章

最新更新