Geek Answers Handbook

Parse multiline JSON with grok in logstash

I have a format JSON:

{
    "SOURCE":"Source A",
    "Model":"ModelABC",
    "Qty":"3"
}

I am trying to parse this JSON using logstash. Basically, I want the output of logstash to be a list of key: value pairs that I can parse with kibana. I thought it could be done out of the box. From a lot of reading, I understand that I should use the grok plugin (I'm still not sure what the json plugin is for). But I cannot receive an event with all fields. I get multiple events (one even for every attribute of my JSON). For instance:

{
       "message" => "  \"SOURCE\": \"Source A\",",
      "@version" => "1",
    "@timestamp" => "2014-08-31T01:26:23.432Z",
          "type" => "my-json",
          "tags" => [
        [0] "tag-json"
    ],
          "host" => "myserver.example.com",
          "path" => "/opt/mount/ELK/json/mytestjson.json"
}
{
       "message" => "  \"Model\": \"ModelABC\",",
      "@version" => "1",
    "@timestamp" => "2014-08-31T01:26:23.438Z",
          "type" => "my-json",
          "tags" => [
        [0] "tag-json"
    ],
          "host" => "myserver.example.com",
          "path" => "/opt/mount/ELK/json/mytestjson.json"
}
{
       "message" => "  \"Qty\": \"3\",",
      "@version" => "1",
    "@timestamp" => "2014-08-31T01:26:23.438Z",
          "type" => "my-json",
          "tags" => [
        [0] "tag-json"
    ],
          "host" => "myserver.example.com",
          "path" => "/opt/mount/ELK/json/mytestjson.json"
}

json_lines? , ? grok - JSON, : , ? , . . conf :

input
{
        file
        {
                type => "my-json"
                path => ["/opt/mount/ELK/json/mytestjson.json"]
                codec => json
                tags => "tag-json"
        }
}

filter
{
   if [type] == "my-json"
   {
        date { locale => "en"  match => [ "RECEIVE-TIMESTAMP", "yyyy-mm-dd HH:mm:ss" ] }
   }
}

output
{
        elasticsearch
        {
                host => localhost
        }
        stdout { codec => rubydebug }
}
+4
2

, . , , JSON .

input 
{   
    file 
    {
        codec => multiline
        {
            pattern => '^\{'
            negate => true
            what => previous                
        }
        path => ["/opt/mount/ELK/json/*.json"]
        start_position => "beginning"
        sincedb_path => "/dev/null"
        exclude => "*.gz"
    }
}

filter 
{
    mutate
    {
        replace => [ "message", "%{message}}" ]
        gsub => [ 'message','\n','']
    }
    if [message] =~ /^{.*}$/ 
    {
        json { source => message }
    }

}

output 
{ 
    stdout { codec => rubydebug }
}

mutliline , JSON json { source => message }. , :

replace => [ "message", "%{message}}" ]

.

gsub => [ 'message','\n','']

\n. , JSON, json { source => message }

/ JSON JSON, , POST, , .

+5

multiline.

input {
  file {
    codec => multiline {
        pattern => '^{'
        negate => true
        what => previous
    }
    path => ['/opt/mount/ELK/json/mytestjson.json']
  }
}
filter {
  json {
    source => message
    remove_field => message
  }
}

, , . , ( ) - { , , .

+4

More articles:


All Articles