Some Icinga2 notes

Currently in order to get icinga2 performance data output, you can use a couple
of plugins - graphite is one of the options.
Logstash has a graphite input plugin, so you can set that up to import things 
from icinga2 and export to elasticsearch.
The downside to this is that you end up with a whole lot of field names - this
grows ridiclously with the number of hosts you add. The default max number of 
fields in an elasticsearch doc is 1000, so you might quickly run up against the
wall.

In order to fix this, you need to write a grok filter to split out the icinga
data into fields, and remove the original metric from the doc.
 
I came up with the following.
Note: the template fields here, you will probably have to install that manually.
I just keep them in there for documentation purposes really. I'll include the
json template underneath the config as that's quite useful as well.

You can extend the outputs to multiple indexes based on the plugin names.
The ones provided here are just my first cut at splitting them up.

The reason for setting the fields to not_analyzed is so that elasticsearch
doesn't split them up into tokens for searching - things like hostnames will
have - and . and other separator fields which you want to keep intact and not
search partials. This will aid you greatly when setting up Grafana for graphing
the metrics.

Also see my earlier stackoverflow question/answer - stackoverflow


---
graphite-filter.conf
---

input {
  graphite {
    type => graphite
    port => 2003
    id => "graphite_input"
  }
}


filter {

        if [type] == "graphite" {
                grok {
                        match => [ "message", "\Aicinga2\.%{MONGO_WORDDASH:host_name}\.%{WORD:metric_type}\.%{NOTSPACE:metric_name}\.value%{SPACE}%{NUMBER:metric_value:float}%{SPACE}%{POSINT:timestamp:date}" ]
                }
                mutate {
                        remove_field => [ "icinga2.%{host_name}.%{metric_type}.%{metric_name}.value" ]
                }
        }

}


output {

        if [type] == "graphite" {
                if [metric_name] =~ "hostalive.perfdata" {
                        elasticsearch {
                                index => "graphite-hostalive-%{+YYYY.MM}"
                                hosts => ["localhost"]
                                template => "/etc/logstash/graphite-new.json"
                        }
                } else if [metric_name] =~ "snmp-interface" {
                        elasticsearch {
                                index => "graphite-snmp-%{+YYYY.MM}"
                                hosts => ["localhost"]
                                template => "/etc/logstash/graphite-new.json"
                        }


                } else {
                        elasticsearch {
                                index => "graphite-%{+YYYY.MM}"
                                hosts => ["localhost"]
                                template => "/etc/logstash/graphite-new.json"
                        }
                }
        }

}

---
graphite-new.json
---

{
    "template" : "graphite-*",
    "settings" : { "index.refresh_interval" : "60s",
                   "index.mapping.total_fields.limit" : 1000000000},
    "mappings" : {
        "_default_" : {
            "_all" : { "enabled" : false },
            "dynamic_templates" : [{
              "message_field" : {
                "match" : "message",
                "match_mapping_type" : "string",
                "mapping" : { "type" : "string", "index" : "not_analyzed" }
              }
            }, {
              "string_fields" : {
                "match" : "*",
                "match_mapping_type" : "string",
                "mapping" : { "type" : "string", "index" : "not_analyzed" }
              }
            }],
            "properties" : {
                "@timestamp" : { "type" : "date", "format" : "dateOptionalTime" },
                "@version" : { "type" : "integer", "index" : "not_analyzed" },
                "metric_name" : { "type" : "string", "index" : "not_analyzed" },
                "host" : { "type" : "string", "index" : "not_analyzed" },
                "host_name" : { "type" : "string", "index" : "not_analyzed" },
                "metric_type" : { "type" : "string", "index" : "not_analyzed" }
            }
        }
    }
}