我们将netscaler设备syslog日志发送到我们的ELK堆栈，并使用从我们端创建的自定义管道对其进行解析。虽然我们能够实现管道配置的目标，但我们注意到摄取节点的性能问题，然后是logstash中的数据排队，这反过来又导致了ELK Stack的整体性能问题。当我们将这个管道配置添加到堆栈中时，摄取节点中的负载变得很高。我们已经停止将netscaler日志加载到系统中。
有没有人能分析一下这个管道配置，并向我们提供一些建议，告诉我们如何在不影响ELK性能的情况下改进它?
POST _ingest/pipeline/_simulate
{
"pipeline": {
"description" : "Netscalaer logs",
"processors" :[
{
"json": {
"field": "message",
"add_to_root": false,
"target_field": "qrapp"
}
},
{
"grok": {
"field": "qrapp.message",
"patterns": [
"%{DATA:qrapp.eventid} %{DATA:qrapp.event.timestamp} %{DATA:qrapp.event.timezone} %{DATA:qrapp.hostname} : %{CISCO_REASON:qrapp.msg1} : Context %{DATA:qrapp.context_user}@%{IP:qrapp.context_ip} - %{GREEDYDATA:qrapp.msg2}",
"%{DATA:qrapp.eventid} %{DATA:qrapp.event.timestamp} %{DATA:qrapp.event.timezone} %{DATA:qrapp.hostname} : %{CISCO_REASON:qrapp.msg1} :%{SPACE}%{GREEDYDATA:qrapp.msg2}",
"%{DATA:qrapp.eventid} %{DATA:qrapp.event.timestamp} %{DATA:qrapp.event.timezone} %{DATA:qrapp.hostname} : %{GREEDYDATA:qrapp.msg}",
"%{GREEDYDATA:qrapp.msg}"
],
"ignore_missing": true
}
},
{
"kv": {
"field": "qrapp.msg2",
"field_split": " - ",
"value_split": " ",
"target_field": "qrapp",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"gsub": {
"field": "qrapp.eventid",
"pattern": "[\<\>]",
"replacement": "",
"ignore_missing": true,
"ignore_failure": true
}
},
{
"rename": {
"field": "qrapp.Total_bytes_recv",
"target_field": "qrapp.total_bytes_recv",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Total_bytes_send",
"target_field": "qrapp.total_bytes_send",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Destination",
"target_field": "qrapp.destination",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Nat_ip",
"target_field": "qrapp.nat_ip",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.User",
"target_field": "qrapp.user",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Vserver",
"target_field": "qrapp.vserver",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Access",
"target_field": "qrapp.access",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Source",
"target_field": "qrapp.source",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Client_ip",
"target_field": "qrapp.client_ip",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Compression_ratio_send",
"target_field": "qrapp.compression_ratio_send",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Total_compressedbytes_send",
"target_field": "qrapp.total_compressedbytes_send",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.End_time",
"target_field": "qrapp.end_time",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Start_time",
"target_field": "qrapp.start_time",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Duration",
"target_field": "qrapp.duration",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Total_compressedbytes_recv",
"target_field": "qrapp.total_compressedbytes_recv",
"ignore_missing": true
}
},
{
"rename": {
"field": "qrapp.Compression_ratio_recv",
"target_field": "qrapp.compression_ratio_recv",
"ignore_missing": true
}
},
{
"date": {
"formats": [
"yyyy-MM-dd_HH:mm:ss.SSS",
"dd-MMM-yy_HH.mm.ss.SSS",
"yyyy-MM-d_HH:mm:ss.SSS",
"yyyy-MM-dd HH:mm:ss,SSS",
"yyyy-MM-d HH:mm:ss,SSS"
],
"ignore_failure": true,
"field": "qrapp.timestamp",
"target_field": "@timestamp",
"timezone": "Asia/Qatar"
}
},
{
"remove": {
"field": ["qrapp.message", "message", "qrapp.msg2", "qrapp.context_ip", "qrapp.context_user"],
"ignore_missing": true
}
}
]
},
"docs": [
{
"_source": {
"message": """{"@version":"1","tags":["_grokparsefailure_sysloginput"],"message":"<134> 11/07/2022:08:02:00 GMT ************* 0-PPE-0 : default SSLVPN ICAEND_CONNSTAT 1314466465 0 : Source *********:*** - Destination *********:*** - customername - username:domainname *****@******: - startTime "11/07/2022:07:57:46 GMT" - endTime "11/07/2022:08:02:00 GMT" - Duration 00:04:14 - Total_bytes_send 73134 - Total_bytes_recv 44465 - Total_compressedbytes_send 0 - Total_compressedbytes_recv 0 - Compression_ratio_send 0.00% - Compression_ratio_recv 0.00% - connectionId 47345284 - Total_bytes_wire_send 4710555505001451848 - Total_bytes_wire_recv 5931570851812355409n","severity_label":"Emergency","@timestamp":"2022-11-07T08:00:07.384Z","host":"********","facility_label":"kernel","severity":0,"priority":0,"facility":0}"""
}
}
]
}
在logstash配置中输入如下:
input {
#Section for Netscaler
syslog {
port => ****
id => "check-syslog"
}
}
# output section
output {
#Section for Netscaler
if [host] == '********'
{
file
{
path => "*************.log"
}
}
}
您的grok正在匹配一组模式，这表明大多数事件将不匹配这些模式中的一个或多个。不匹配模式的成本可能非常高，但是您可以做一些事情来降低不匹配的成本。这篇来自Elastic的博客文章提供了背景信息。
首先，如果可以，锚定你的模式。如果%{DATA:qrapp.eventid}总是出现在qrapp的开始。消息字段，然后更改为^%{DATA:qrapp.eventid}。其次，DATA，尤其是GREEDYDATA，在它们不匹配时是昂贵的。如果您的消息字段是用空格分隔的，那么使用NOTSPACE要比使用DATA便宜得多。