#

# PARSER DEFINITIONS FILE IN YML FORMAT

#

# Please use 'ts' as feild name for dates and time

# RegexTools: https://regex101.com/#javascript

# Date format changes in date-fns:

# DEPRECATED date formats: https://date-fns.org/v2.0.0-alpha.6/docs/parse

# NEW date formats: https://date-fns.org/v2.9.0/docs/parse

# DD -> dd

# YYYY -> yyyy

# ZZ -> X

# Sensitive data can be replaced with a hashcode (sha1)

# it applies to fields matching the field names by a regular expression

# Note: this function is not optimized (yet) and might cause 10-15% performance hit

#autohash: !!js/regexp /user|client_ip|password|email|credit_card_number|payment_info/i

# set the hash function (default sha256), sha256,sha512

#hashFunction: sha512

debug: false

# set originalLine to false when auothash fields

# the original line might include sensitive data!

originalLine: false

# default seperator for multiline logs,

# which don't have a blockStart property

# The default /^\S{2,}/ would match typical stack traces

# All lines that start with a whitespace or contain only one char

# would be attached to previous lines

multiline:

defaultSeparator: ^\S{2,}

# post process all JSON input

json:

enabled: true

# autohashFields:

# _HOSTNAME: true

debug: false

# removeFields:

# - stacktrace

# - msg

# - level

# - time

# mapFields:

# msg: message

# level: severity

# time: '@timestamp'

# transform: !!js/function >

# function (sourceName, parsedObject, config) {

# // map fields

# Object.keys(config.mapFields).forEach(function (f) {

# if (parsedObject[f] !== undefined) {

# parsedObject[config.mapFields[f]] = parsedObject[f]

# if (config.debug) {

# console.log('map ' + f + ' to ' + config.mapFields[f] + ': ' + parsedObject[config.mapFields[f]])

# }

# }

# })

# // remove fields

# for (var i=0; i<config.removeFields.length; i++) {

# if (config.debug) {

# console.log('delete ' + config.removeFields[i])

# }

# delete parsedObject[config.removeFields[i]]

# }

# }

# The global transform function is called for each parsed line.

# A good place to do add global fields, custom date processing, or special parsers.

#globalTransform: !!js/function >

# function (source, parsedObject) {

# // this function is called after parsing

# // regardless of the logging source

# // for pattern specific functions use transform in pattern definitions

# }

# IMPORTANT:

# PATTERNS ARE EVALUATED SEQUENTIALLY FOR EACH LOG EVENT. PUT MORE COMPLEX AND MORE SPECIFIC PATTERNS FIRST.

# As soon as a pattern matches a log event the rest of the patterns are skipped.

#

# To test your pattern: cat myTest.log | logagent -n <pattern name here> -yml -f mypatterns.yml

# A pattern name are things like 'kubernetes hyperkube' or 'Elasticsearch' or 'Apache Solr' below.

patterns:

- # sematext/agent logs

sourceName: !!js/regexp /sematext\/agent|containerd|dockerd/

blockStart: !!js/regexp /^panic\:|^unexpected fault address|^fatal error\:|(^INFO|^ERRO|^WARN|^FAT|^TRAC|^DEB)|^time=|^S{2,}/

match:

- type: sematext_agent_golang

regex: !!js/regexp /time=(\S+)\slevel=(\S+?)\smsg="(.+?)"\ssource="(.+?)"/i

fields: [ts, severity, message, source]

dateFormat: iso

- type: sematext_agent_golang

regex: !!js/regexp /([A-Z]+)\[(.+?)\]\s(.*)/i

fields: [severity, ts, message]

dateFormat: iso

- # Yandex Clickhouse

sourceName: !!js/regexp /clickhouse/

blockStart: !!js/regexp /(\d{4}.\d{2}.\d{2}[\s|T][\d+|\:]+.\d+)\s\[\s(\d+)\s]\s\{(\S*)\}/

match:

- type: clickhouse

regex: !!js/regexp /(\d{4}.\d{2}.\d{2}[\s|T][\d+|\:]+.\d+)\s\[\s(\d+)\s]\s\{(\S*)\}\s<(\S+)>\s((.+?)\:[\s|\S]+)/i

fields: [ts, threadNumber, queryId, severity, message, module]

dateFormat: yyyy.MM.dd HH:mm:ss.SSS

- # kubernetes hyperkube

sourceName: !!js/regexp /hyperkube/

match:

- type: hyperkube

regex: !!js/regexp /^\S+\s(\S+)\s+\S+\s+\S+\s([GET|POST|PUT|DELETE|HEAD|OPTIONS]+)\s+(\/.+)\:\s\(([\d|\.]+)(\S+)\)\s(\d+\s)(.*hyperkube.+)\s(.+)\:(\d+)\]/i

fields: [ts,method,url,duration,duration_unit,status_code,info,ip,port]

dateFormat: HH:mm:ss:S

- # Elasticsearch

blockStart: !!js/regexp /^[\d{4}-\d{2}-\d{2}[\s|T][\d+|\:]+.\d+|log4j\:\S+\s/

sourceName: !!js/regexp /elasticsearch/

match:

- type: elasticsearch_slow_log

regex: !!js/regexp /^\[(\d{4}-\d{2}-\d{2}[\s|T][\d+|\:]+.\d+)\]\[(.+?)\s*\]\[(\S{0,512})\s*\]\s*\[(.+?)\]\s\[(\S+?)\]\[(\d+)\]\s.+took_millis\[(\d+)\].+types\[(.*?)\].+stats\[(.*?)\].*search_type\[(.*?)\].+total_shards\[(.*?)\].+source\[(.*?)\],/i

fields:

- ts

- severity:string

- class_name:string

- node_name:string

- index_name:string

- shard_number:number

- took_millis:number

- types:string

- stats:string

- search_type:string

- total_shards:number

- source:string

- type: elasticsearch

regex: !!js/regexp /^\[(\d{4}-\d{2}-\d{2}[\s|T][\d+|\:]+.\d+)\]\[(.+?)\s*\]\[(\S{0,512})\s*\]\s*\[(.+?)\]\s([\s|\S]+)/

fields: [ts,severity,class_name,node_name,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- type: elasticsearch

regex: !!js/regexp ^\[(\d{4}-\d{2}-\d{2}\s[\d+|\:]+.\d+)\]\[(.+?)\]\[(\S{0,512})\s*\]\s*\s([\s|\S]+)

fields: [ts,severity,class_name,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- # Apache Solr

blockStart: !!js/regexp ^\S*\s*\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3}|^\d+\s+\S{3,5}\s+

sourceName: !!js/regexp /solr/i

match:

- type: apache_solr_7_8_hits

regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s+\((.+?)\)\s\[(.+?)]\s(.+?)\s\[(.+?)\]\s+webapp=(.+?)\spath=(.+?)\sparams={(.*)}\shits=(\d+)\sstatus=(\d+)\sQTime=(\d+)

fields:

- ts

- severity:string

- thread:string

- core:string

- class:string

- shard:string

- webapp:string

- path:string

- params:string

- hits:number

- status:number

- qtime:number

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

transform: !!js/function >

function (p) {

if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')

{

var params = p.params.split('&')

p.parsedParams={}

for(var i=0;i<params.length;i++)

{

var key_value = params[i].split('=')

if (key_value.length>1) {

key = key_value[0]

if (!/[a-zA-Z]/.test(key)) {

key = "field_" + key

}

if (key in p.parsedParams) {

// convert from single to multivalued

if (typeof p.parsedParams[key] == 'string') {

p.parsedParams[key] = [p.parsedParams[key]]

}

p.parsedParams[key].push(key_value[1])

} else {

p.parsedParams[key] = key_value[1]

}

}

}

if (p.parsedParams['NOW']) {

p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)

// if (!p['@timestamp'])

// p['@timestamp'] = p.paramsParsed['NOW']

}

}

var core = p.core.split(' ')

//sometimes core=0, sometimes core="c:test s:shard2...."

if (core.length>1){

p.parsedCore={}

for(var i=0;i<core.length;i++)

{

var key_value = core[i].split(':')

if (key_value.length>1) {

p.parsedCore[key_value[0]] = key_value[1]

}

}

}

}

- type: apache_solr_7_8

regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s+\((.+?)\)\s\[(.+?)]\s(.+?)\s\[(.+?)\]\s+webapp=(.+?)\spath=(.+?)\sparams={(.*)}\sstatus=(\d+)\sQTime=(\d+)

fields:

- ts

- severity:string

- thread:string

- core:string

- class:string

- shard:string

- webapp:string

- path:string

- params:string

- status:number

- qtime:number

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

transform: !!js/function >

function (p) {

if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')

{

var params = p.params.split('&')

p.parsedParams={}

for(var i=0;i<params.length;i++)

{

var key_value = params[i].split('=')

if (key_value.length>1) {

key = key_value[0]

if (!/[a-zA-Z]/.test(key)) {

key = "field_" + key

}

if (key in p.parsedParams) {

// convert from single to multivalued

if (typeof p.parsedParams[key] == 'string') {

p.parsedParams[key] = [p.parsedParams[key]]

}

p.parsedParams[key].push(key_value[1])

} else {

p.parsedParams[key] = key_value[1]

}

}

}

if (p.parsedParams['NOW']) {

p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)

// if (!p['@timestamp'])

// p['@timestamp'] = p.paramsParsed['NOW']

}

}

var core = p.core.split(' ')

//sometimes core=0, sometimes core="c:test s:shard2...."

if (core.length>1){

p.parsedCore={}

for(var i=0;i<core.length;i++)

{

var key_value = core[i].split(':')

if (key_value.length>1) {

p.parsedCore[key_value[0]] = key_value[1]

}

}

}

}

- type: apache_solr_audit_log

regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s*\((.+?)\)\s\[(.+?)]\s(.+?)\stype="(.*?)"\smessage="(.*?)"\smethod="(.*?)"\sstatus="(.*?)"\srequestType="(.*?)"\susername="(.*?)"\sresource="(.*?)"\squeryString="(.*?)"\scollections=(.*)

fields:

- ts

- severity:string

- thread:string

- core:string

- class:string

- type:string

- message:string

- method:string

- status:number

- requestType:string

- username:string

- resource:string

- querystring:string

- collections:string

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: apache_solr_7_8_generic

regex: !!js/regexp ^\S*\s*(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3})\s(.+?)\s+\((.+?)\)\s\[(.+?)]\s(.+?)\s(.*)

fields:

- ts

- severity:string

- thread:string,

- core:string

- class:string

- message:string

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

transform: !!js/function >

function (p) {

var core = p.core.split(' ')

//sometimes core=0, sometimes core="c:test s:shard2...."

if (core.length>1){

p.parsedCore={}

for(var i=0;i<core.length;i++)

{

var key_value = core[i].split(':')

if (key_value.length>1) {

p.parsedCore[key_value[0]] = key_value[1]

}

}

}

}

- type: apache_solr_v4.6

regex: !!js/regexp ^(\S+)\s+-\s(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s(.+?);\s\[(.+?)]\swebapp=(\S+)\spath=(.+?)\sparams={(.*)}.*hits=(\d+)\sstatus=(\d+)\sQTime=(\d+)

fields: [severity,ts,class,application,webapp,path,params,hits,status,qtime]

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

transform: !!js/function >

function (p) {

if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')

{

var params = p.params.split('&')

p.parsedParams={}

for(var i=0;i<params.length;i++)

{

var key_value = params[i].split('=')

if (key_value.length>1) {

p.parsedParams[key_value[0]] = key_value[1]

}

}

if (p.parsedParams['NOW']) {

p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)

// if (!p['@timestamp'])

// p['@timestamp'] = p.paramsParsed['NOW']

}

}

}

- type: apache_solr

regex: !!js/regexp ^(\S+)\s+-\s(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s\[\s*(.+?)]\s(\S+);\s.*.*webapp=(\S+)\spath=(.+?)\sparams={(.*)}.*hits=(\d+)\sstatus=(\d+)\sQTime=(\d+)

fields: [severity,ts,application,class,webapp,path,params,hits,status,qtime]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- type: apache_solr_v5_1

regex: !!js/regexp ^(\d+)\s\[(\S+)]\s(\S+)\s(\S+)\s\[(\S+)\s(\S+)\s(\S+)\s(\S+)\].+?\[(.+?)\]\swebapp=(.+?)\spath=(.+?)\sparams={(.+?)}\sstatus=(\d+)\sQTime=(\d+)

fields: [relative_ts,thread_id,severity,class,collection,shard,core,replica,core_name,webapp,path,params,status,qtime]

transform: !!js/function >

function (p) {

if (process.env.PARSE_SOLR_QUERY_PARAMS === '1')

{

var params = p.params.split('&')

p.parsedParams={}

for(var i=0;i<params.length;i++)

{

var key_value = params[i].split('=')

if (key_value.length>1)

p.parsedParams[key_value[0]] = key_value[1]

}

if (p.parsedParams['NOW']) {

p.parsedParams['NOW'] = new Date(p.parsedParams['NOW']*1)

// if (!p['@timestamp'])

// p['@timestamp'] = p.paramsParsed['NOW']

}

}

}

- type: apache_solr

regex: !!js/regexp ^(\S+)\s+-\s+(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s+(\S+);\s+(.+Exception:\s[\s|\S]+)

fields: [severity,ts,class,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- type: apache_solr

regex: !!js/regexp ^(\S+)\s+-\s+(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s+(\S+);\s([\s|\S]+)

fields: [severity,ts,class,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- type: apache_solr_5_generic

regex: !!js/regexp ^(\S+)\s+-\s+(\d{4}-\d{2}-\d{2}\s[\d|\:+,\d]+\.\d{0,3});\s(.*)

fields: [severity,ts,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- type: apache_solr4

regex: !!js/regexp ^(\d+)\s+(\S+)\s+\((\S+)\)\s+\[(.+?)\]\s(\S+)\s(.+)

fields: [relative_ts,severity,thread,thread_id,class,message]

- # Apache Kafka

sourceName: !!js/regexp /kafka/

match:

- type: apache_kafka

regex: !!js/regexp ^\[(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\]\s(\S+)\s(.+)

fields: [ts,severity,message]

dateFormat: yyyy-MM-dd HH:mm:ss

- # Apache HDFS Data Node

blockStart: !!js/regexp ^\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+\s(\S+)\s/

sourceName: !!js/regexp /hdfs/

match:

- type: apache_hdfs_data_node

regex: !!js/regexp ^(\d{4}-\d{2}-\d{2}\s+[\d|\:]+,\d+)\s+(\S+)\s(\S+):\s([\s|\S]+)

fields: [ts,severity,class,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- # Apache HBase Region Server

blockStart: !!js/regexp ^\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+\s(\S+)\s/

sourceName: !!js/regexp /hbase/

match:

- type: apache_hbase_region_server

regex: !!js/regexp ^(\d{4}-\d{2}-\d{2}\s+[\d|\:]+,\d+)\s+(\S+)\s+\[(.+)\]\s(\S+):\s([\s|\S]+)

fields: [ts,severity,thread,class,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- # Apache YARN

sourceName: !!js/regexp /yarn/

match:

- type: apache_hadoop_yarn_node_manager

regex: !!js/regexp ^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s(\S+)\s(\S+):\s([\S|\s]+)

fields: [ts,severity,class_name,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- # Apache Zookeeper

sourceName: !!js/regexp /zookeeper|zk/

blockStart: !!js/regexp /^\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+\s+/

match:

- type: apache_zookeeper

regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+\[(\S+?):{0,1}\]\s+-\s+(\S+)\s+\[(.+)\]\s-\s+([\S|\s]+[client|from]\s\/(.+?)\:(\d+).*sessionid.+(0x.+).*)/

fields:

- ts

- machineId:string

- severity:string,

- thread_info:string

- message:string

- client_ip:string

- client_port:number

- session_id:string

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- type: apache_zookeeper

regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+\[(\S+?):{0,1}\]\s+-\s+(\S+)\s+\[(.+)\]\s-\s+([\S|\s]+[client|from]\s\/(.+?)\:(\d+).*)/

fields:

- ts

- machineId:string

- severity:string,

- thread_info:string

- message:string

- client_ip:string

- client_port:number

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- type: apache_zookeeper

regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+\[(\S+?):{0,1}\]\s+-\s+(\S+)\s+\[(.+)\]\s-\s+([\S|\s]+)/

fields:

- ts

- machineId:string

- severity:string,

- thread_info:string

- message:string

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- # Apache Cassandra

sourceName: !!js/regexp cassandra

# multi-line, start sequence

blockStart: !!js/regexp ^\S{3,5}\s+\[.+\]\s+\d{4}

match:

- type: apache_cassandra

regex: !!js/regexp ^\S{0,5}(\S*)\s+\[(.+)\]\s(\d{4}-\d{2}-\d{2}\s[\d|\:]+,\d+)\s+(.+\.java)\:(\d+)\s+-\s+([\S|\s]+)

fields: [severity,module,ts,java_file,code_line,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SS

- # MongoDB

# name of the docker image

sourceName: !!js/regexp /mongo/

# 2015-07-28T00:35:46.329+0000 I JOURNAL [initandlisten] journal dir=/data/db/journal

match:

- type: mongodb

regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}T[\d|\.|\:]+\+\d{4})\s(\w+)\s(\S+)\s+\[(\S+)\]\s(.+)/i

fields: [ts,severity, component, context, message]

dateFormat: iso

- # REDIS

# name of the docker image

# example: "1:M 22 Jul 21:58:28.146 # Server started, Redis version 3.0.2"

sourceName: !!js/regexp /redis/i

blockStart: !!js/regexp /^\d+:.\s\d+/

match:

- type: redis

blockStart: !!js/regexp /^\d+:.\s\d+/

fields: [pid,role,ts,level,message]

regex: !!js/regexp /^(\d+):(\w+)\s+(\d\d\s\w+.+)\s+(\S)\s+([\S|\s]+)/

dateFormat: dd MMM HH:mm:ss.SSS

transform: !!js/function >

function transformMessage (p) {

levels = {

'.': 'debug',

'-': 'verbose',

'*': 'notice',

'#': 'warning'

}

roles = {

'X': 'sentinel',

'C': 'RDB/AOF writing child',

'S': 'slave',

'M': 'master'

}

p.role = roles[p.role]

p.severity = levels[p.level] || p.level

// p.message = p.message.replace(/\r/g,'')

delete p.level

}

- # Sonatype Nexus

sourceName: !!js/regexp /nexus/

# yyyy-MM-dd starts a new log entry

blockStart: !!js/regexp ^\d{4}-\d{2}-\d{2}

match:

- type: nexus

regex: !!js/regexp /^([\d|\-|\s|\:|\.|\,|\+]+)\s+([A-Z]+)\s+[^\[]*\[\s*([^\]]+)\]\s(\*?\w+)\s+([\w|\.]+)\W+(.+)/

fields: [ts,severity,thread,user,class,message]

dateFormat: yyyy-MM-dd HH:mm:ss,SSSX

- # NodeBB Forum

sourceName: !!js/regexp /nodebb/i

match:

- type: nodebb_forum

fields: [ts,severity,module,message]

regex: !!js/regexp /^(\d{4}\-\d{2}\-\d{1,2}T\d\d:\d\d:\d\d\.\d+Z)\s-\s(\w+):\s\[(\S+)]\s(.*)/

dateFormat: iso

- type: nodebb_forum

fields: [ts,severity,message]

regex: !!js/regexp /^(\d{4}\-\d{2}\-\d{1,2}T\d\d:\d\d:\d\d\.\d+Z)\s-\s(\w+):\s(.*)/

- # mysql

# 2015-07-25 14:11:35 0 [Note] mysqld (mysqld 5.6.26) starting as process 1 ...

sourceName: !!js/regexp /mysql/

match:

- regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s[\d|\:]+)\s(\d+)\s\[(.+?)\]\s+(.*)/

fields: [ts,pid,level,message]

dateFormat: yyyy-MM-dd HH:mm:ss

type: mysql

- # nsq.io

sourceName: !!js/regexp nsqio\/nsq

match:

- type: nsq

regex: !!js/regexp (^\d{4}\/\d{2}\/\d{2}\s[\d|\:]+)\s(\S+)\s+(\d+)\s+\[(\S+)\]\s+(.+)

fields: [ts, level, pid, module, message]

dateFormat: yyyy/MM/dd HH:mm:ss

- # Web Logs

sourceName: !!js/regexp /httpd|access_log|apache2|nginx|sematext\/frontend-app/

match:

- type: access_log_combined

regex: !!js/regexp ^([0-9a-f.:]+)\s(-|\S+)\s(-|\S+)\s\[(.*)\]\s\"(\w+)\s(\S+)\s{0,1}(.*)\" ([0-9|\-]+) ([0-9|\-]+) \"([^\"]+)\" \"([^\"]+)\"

fields:

- client_ip:string

- remote_id:string

- user:string

- ts

- method:string

- path:string

- protocol:string

- status_code:number

- size:number

- referer:string

- user_agent:string

dateFormat: dd/MMM/yyyy:HH:mm:ss X

transform: !!js/function >

function transformMessage (p) {

p.message = p.method + ' ' + p.path

if(p.status_code === '-') {

p.status_code = 0

}

if(p.size === '-') {

p.size = 0

}

}

# nginx proxy jwilder/nginx-proxy

- regex: !!js/regexp /^(\S+)\s+(-|.+?)\s+(-|.+?)\s+\[(.*)\]\s\"(\S+)\s(\S+)\s(\S+)\"\s(\d+)\s(\d+|\"-\"|-)\s+"{0,1}(.+?)"{0,1}\s+"{0,1}([\S|\s]+)"{0,1}/i

type: access_common

fields:

- proxy_service:string

- virtual_host:string

- client_ip:string

- remote_id:string

- user:String

- ts

- method:string

- path:string

- http_version:string

- status_code:number

- size:number

- url:string

- user_agent:string

dateFormat: dd/MMM/yyyy:HH:mm:ss X

#transform: !!js/function >

# function transformMessage (p) {

# p.message = p.method + ' ' + p.path

# if(p.status_code === '-') {

# p.status_code = 0

# }

# if(p.size === '-') {

# p.size = 0

# }

# }

- regex: !!js/regexp ^(\S+)\s+(-|.+?)\s+(-|.+?)\s+\[(.*)\]\s\"(\S+)\s(\S+)\s(\S+)\"\s(\d+)\s(\d+|\"-\"|-)

type: access_common

fields:

- client_ip:string

- remote_id:string

- user:string

- ts

- method:string

- path:string

- http_version:string

- status_code:number

- size:number

dateFormat: dd/MMM/yyyy:HH:mm:ss X

#transform: !!js/function >

# function transformMessage (p) {

# p.message = p.method + ' ' + p.path

# if(p.status_code === '-') {

# p.status_code = 0

# }

# if(p.size === '-') {

# p.size = 0

# }

# }

- type: nginx_error_log

regex: !!js/regexp /^(\d{4}\/\d{2}\/\d{2}\s[\d|\:]+)\s\[(.+?)]\s(\d+)#(\d+)\:\s(.*)/

fields: [ts,level,pid,tid,message]

dateformat: yyyy/MM/dd HH:mm:ss

- type: apache_error_log

regex: !!js/regexp /^\[(\w{3} \w{3} \d{2} [\d|\:]+\s\d+)\] \[(.+?)\] \[client ([\d|\.]+)\] (.+)/

fields: [ts,level,client_ip,message]

dateformat: ddd MMM dd hh:mm:ss.SSS yyyy

# Apache MPM events

- regex: !!js/regexp /^\[(.+?)\]\s+\[(.+?)\]\s+\[(.+?)\]\s+(.+)/

fields: [ts,event_type,processInfo,message]

type: apache_mpm

dateformat: ddd MMM dd hh:mm:ss.SSS yyyy

- # Apache Flink jobmanager and taskmanager

sourceName: !!js/regexp /flink/

blockStart: !!js/regexp /^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2},\d{3}\s/

match:

- type: flink[job|task]manager

regex: !!js/regexp /^(\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2},\d{3})\s(INFO|info|WARN|ERROR|FATAL)\s+([\S]+)\s+-\s([\S|\s|\n]+)/

fields: [ts, severity, processid, message]

dateFormat: yyyy-MM-dd HH:mm:ss:l

- # Apache Flink jobmanager and taskmanager

sourceName: !!js/regexp /flink/

blockStart: !!js/regexp /^\w{3}\s[\d|,|\s|\:]+/

match:

- type: flink[job|task]manager

regex: !!js/regexp /^(\w{3}\s[\d|,|\s|\:]+\w{3}\s[\d|,|\s|\:]+[AP]M)\s([\S|\s]+)\s(INFO|WARN|ERROR|SEVERE|FATAL):\s([\S|\s|\n]+)/

fields: [ts, processid, severity, message]

dateFormat: MMM dd, yyyy H:mm:ss

- # Traefik access_log

sourceName: !!js/regexp /traefik/

match:

- type: traefik_access_log

regex: !!js/regexp ^([0-9a-f.:]+)\s(-|\S+)\s(-|\S+)\s\[(.*)\]\s\"(\w+)\s(\S+)\s{0,1}(.*)\"\s([0-9|\-]+)\s([0-9|\-]+)\s\"([^\"]+)\"\s\"([^\"]+)\"\s([0-9|\-]+)\s\"(.+)\"\s\"(.+)\"\s([0-9]+)ms

fields:

- client_ip:string

- remote_id:string

- user:string

- ts

- method:string

- path:string

- protocol:string

- status_code:number

- size:number

- referer:string

- user_agent:string

- req_count:string # https://github.com/containous/traefik/blob/master/middlewares/accesslog/logger_formatters.go#L45

- frontend_name:string

- backend_url:string

- response_time:number

dateFormat: dd/MMM/yyyy:HH:mm:ss X

transform: !!js/function >

function transformMessage (p) {

p.message = p.method + ' ' + p.path

if(p.status_code === '-') {

p.status_code = 0

}

if(p.size === '-') {

p.size = 0

}

}

- # Tutum Logs

sourceName: !!js/regexp /tutum\/cleanup/

match:

- type: tutum_cleanup

regex: !!js/regexp /^(\d+\/\d+\/\d+\s\d+:\d+:\d+)\s(.*)/

fields: [ts,message]

dateFormat: yyyy/MM/dd hh:mm:ss

- # RabbitMQ

sourceName: !!js/regexp /rabbitmq/

blockStart: !!js/regexp /^\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d/

match:

- type: rabbitmq_startup

# regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s*\n\s*(Starting\sRabbitMQ\s(\S+)\son\sErlang\s(.+)\n[\s\S]*)/

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s*\n\s*(Starting\sRabbitMQ\s(\S+)\son\sErlang\s(.+)\s*\n\s*(Copyright\s\(c\)\s\d{4}-\d{4}\s[^\n]*)\s*\n\s*Licensed\sunder\sthe\s(\S*\s?\d*\.?\d?)[\s\S]*)/

fields:

- ts

- severity

- connection:string

- message

- rabbitmq_version

- erlang_version

- copyright

- license

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: rabbitmq_wal_recovery

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(WAL:\srecovering\s\[(.*)\])/

fields:

- ts

- severity

- connection:string

- message

- wal_path:string

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: rabbitmq_access_denied

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(.+access\sdenied:\suser\s'(\S+)'\s-\s(.+))/

fields:

- ts

- severity

- connection:string

- message

- user:string

- reason:string

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: rabbitmq_login_refused

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\sError\son\sAMQP\sconnection\s<(\S+)>\s\((\S+):(\d+)\s->\s(\S+):(\d+),\sstate:\s(\S+)\):\n((\S+)\slogin\srefused:\suser\s'(\S+)'\s-\s(.+))/

fields:

- ts

- severity

- connection:string

- refused_connection:string

- source_address:string

- source_port:number

- destination_address:string

- destination_port:number

- state:string

- message

- auth_method:string

- user:string

- reason:string

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: rabbitmq_connections_authenticated

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\sconnection\s<\S+>\s\((\S+):(\d+)\s->\s(\S+):(\d+)\):\s(user\s'(\S+)'\sauthenticated\sand\sgranted\saccess\sto\svhost\s'(\S+)\')/

fields:

- ts

- severity

- connection:string

- source_address:string

- source_port:number

- destination_address:string

- destination_port:number

- message

- user:string

- vhost:string

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: rabbitmq_connections_vhost_user

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(accepting|closing)\sAMQP\sconnection\s<\S+>\s\((\S+):(\d+)\s->\s(\S+):(\d+),\svhost:\s'(.+)',\suser:\s'(.+)'\):?([\s\S]*)/

fields:

- ts

- severity

- connection:string

- connection_event

- source_address:string

- source_port:number

- destination_address:string

- destination_port:number

- vhost:string

- user:string

- message

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: rabbitmq_connections

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s<(\S+)>\s(accepting|closing)\sAMQP\sconnection\s<\S+>\s\((\S+):(\d+)\s->\s(\S+):(\d+)\)([\s\S]*)/

fields:

- ts

- severity

- connection:string

- connection_event

- source_address:string

- source_port:number

- destination_address:string

- destination_port:number

- message

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: rabbitmq

regex: !!js/regexp /^(\S+\s\S+)\s\[(\S+)\]\s\<(\S+)\>\s([\s\S]+)/

fields:

- ts

- severity

- connection:string

- message

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- # Postgres

sourceName: !!js/regexp /postgres/

blockStart: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]/

match:

- type: postgres_slowlog

regex: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]\s(\S+)@(\S+)\s(\S+):\s+duration:\s(\S+)\sms\s+(\S+).*:\s+([\S|\s]+)/

fields:

- ts

- timezone:string

- pid:number

- user:string

- database:string

- severity:string

- duration_ms:number

- operation:string

- statement:string

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: postgres_with_user

regex: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]\s(\S+)@(\S+)\s(\S+):\s+([\S|\s]+)/

fields:

- ts

- timezone:string # assuming timezone is always "UTC", but parsing it just in case

- pid:number

- user:string

- database:string

- severity:string

- message

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- type: postgres

regex: !!js/regexp /^(\S+\s\S+)\s(\S+)\s\[(\d+)\]\s(\S+):\s+([\S|\s]+)/

fields:

- ts

- timezone:string # assuming timezone is always "UTC", but parsing it just in case

- pid:number

- severity:string

- message

dateFormat: yyyy-MM-dd HH:mm:ss.SSS

- # CouchDB

sourceName: !!js/regexp /couchdb/

match:

- type: couchdb

regex: !!js/regexp /^\[(\S+)\]\s(\S+)\s(\S+)\s(\S+)\s(\S{8})\s([\s|\S]+)/

fields:

- severity:string

- ts

- node:string

- module:string

- code:string

- message:string

dateFormat: iso

transform: !!js/function >

function (p) {

p.os = {host: p.node}

}

- type: couchdb_http

regex: !!js/regexp /^\[(\S+)\]\s(\S+)\s(\S+)\s(\S+)\s(\S{10})\s(\S+):(\d+)\s(\S+)\s(\S+)\s(GET|PUT|POST|DELETE|HEAD)\s(\S+)\s(\d+)\s(\S|\s+)/

fields:

- severity:string

- ts

- node:string

- module:string

- code:string

- server_ip:string

- server_port:number

- client_ip:string

- user:string

- method:string

- url:string

- status_code:number

- response:string

transform: !!js/function >

function (p) {

p.os = {host: p.node}

}

dateFormat: iso

- # Heroku Syslog Messages

sourceName: !!js/regexp /syslog_framed|heroku/

match:

-

type: heroku

# blockStart: !!js/regexp \^+\s<(\d+)>(\d+)\s/

regex: !!js/regexp ^\d*\s{0,1}<(\d+)>(\d+)\s(\S+)\s(\S+)\s(\S+)\s(\S+)\.{0,1}(\d*)\s+-\s(.*)

fields: [prio,version,ts,host,app,process_type,dyno,message]

dateFormat: iso

transform: !!js/function >

function (p) {

p.os = {host: p.host}

const SEVERITY = [

'emerg',

'alert',

'crit',

'err',

'warning',

'notice',

'info',

'debug'

]

const FACILITY = [

'kern',

'user',

'mail',

'daemon',

'auth',

'syslog',

'lpr',

'news',

'uucp',

'cron',

'authpriv',

'ftp',

'ntp',

'logaudit',

'logalert',

'clock',

'local0',

'local1',

'local2',

'local3',

'local4',

'local5',

'local6',

'local7'

]

p.facility = FACILITY[p.prio>>3] || String(p.prio>>3)

p.severity = SEVERITY[p.prio&7] || String(p.prio&7)

if (p.process_type === 'router')

{

try {

var keyValue = p.message.trim().split(' ')

keyValue.forEach (function (kv) {

var kvs = kv.split ('=')

p[kvs[0].trim()] = kvs[1].trim()

})

} catch (err) {

// ignore

}

}

}

- # CloudFoundry Syslog Messages

sourceName: !!js/regexp /cloudfoundry.*|syslog_raw/

match:

-

type: cloudfoundry

regex: !!js/regexp ^\d*\s{0,1}<(\d+)>(\d+)\s([\d|-]+T[\d|\:|.|\+]+)\s([a-zA-Z0-9\-\.\_\s]+)\s(.+?)\s\[(.+)\]\s-\s\[(.*)\]\s([a-zA-Z0-9\.\_]+)\s-\s\[[\d|-]+T[\d|\:|.]+Z\]\s(.*)

fields: [prio,version,ts,host,processID,applicationID,tags,domain,message]

dateFormat: iso

transform: !!js/function >

function (p) {

const SEVERITY = [

'emerg',

'alert',

'crit',

'err',

'warning',

'notice',

'info',

'debug'

]

const FACILITY = [

'kern',

'user',

'mail',

'daemon',

'auth',

'syslog',

'lpr',

'news',

'uucp',

'cron',

'authpriv',

'ftp',

'ntp',

'logaudit',

'logalert',

'clock',

'local0',

'local1',

'local2',

'local3',

'local4',

'local5',

'local6',

'local7'

]

p.facility = FACILITY[p.prio>>3] || String(p.prio>>3)

p.severity = SEVERITY[p.prio&7] || String(p.prio&7)

delete p.prio

p.os = {host: p.host}

}

-

type: cloudfoundry

regex: !!js/regexp ^\d*\s{0,1}<(\d+)>(\d+)\s([\d|-]+T[\d|\:|.|\+]+)\s([a-zA-Z0-9\-\.\_\s]+)\s(.+?)\s\[(.+)\]\s-\s\[(.*)\]\s(.*)

fields: [prio,version,ts,host,processID,applicationID,tags,message]

dateFormat: iso

transform: !!js/function >

function (p) {

const SEVERITY = [

'emerg',

'alert',