Commit 00041559 authored by Mathias Ettinger's avatar Mathias Ettinger

Removed superfluous data

parent 36d45121
input {
tcp {
port => {{ logstash_stats_port }}
add_field => { "[@metadata][type]" => "stats" }
}
udp {
port => {{ logstash_stats_port }}
add_field => { "[@metadata][type]" => "stats" }
}
syslog {
port => {{ logstash_logs_port }}
add_field => { "[@metadata][type]" => "logs" }
}
}
filter {
# If the type is 'stats', we parse the message and delete useless field
if [@metadata][type] == "stats" {
mutate {
# Remove these fields first so stats can override them
remove_field => [ "@version", "host", "tags", "type", "port" ]
}
json {
source => "message"
}
# If no flag provided or flag is 0 or the json parser has failed, drop the message
if !([_metadata][flag]) or [_metadata][flag] == 0 or "_jsonparsefailure" in [tags] {
drop {}
}
mutate {
remove_field => "message"
rename => { "_metadata" => "@metadata" }
# Put back the field we just overwrite & Save important fields for export (InfluxDB + broadcast)
add_field => {
"[@metadata][type]" => "stats"
"@owner_scenario_instance_id" => "%{[@metadata][owner_scenario_instance_id]}"
"@scenario_instance_id" => "%{[@metadata][scenario_instance_id]}"
"@job_instance_id" => "%{[@metadata][job_instance_id]}"
"@agent_name" => "%{[@metadata][agent_name]}"
"@job_name" => "%{[@metadata][job_name]}"
"@stored_file" => "%{[@metadata][is_file]}"
}
}
date {
match => [ "[@metadata][time]", "UNIX_MS" ]
}
if ([@metadata][suffix]) {
mutate {
add_field => { "@suffix" => "%{[@metadata][suffix]}" }
}
}
}
# If the type is 'logs', we add the flag (broadcast on severity ERROR or more)
if [@metadata][type] == "logs" {
if [severity] <= 3 {
mutate {
add_field => { "[@metadata][flag]" => 3 }
}
} else {
mutate {
add_field => { "[@metadata][flag]" => 1 }
}
}
grok {
patterns_dir => ["/etc/logstash/conf.d/pattern/"]
match => { "message" => "OWNER_SCENARIO_INSTANCE_ID %{NUMBER:owner_scenario_instance_id}, SCENARIO_INSTANCE_ID %{NUMBER:scenario_instance_id}, JOB_INSTANCE_ID %{NUMBER:job_instance_id}, AGENT_NAME %{HOSTNAME:agent_name}, %{GREEDYDATA:syslog_message}" }
}
mutate {
convert => { "[@metadata][flag]" => "integer" }
remove_field => [ "tags", "type", "port" ]
}
if "_grokparsefailure" in [tags] {
mutate {
remove_tag => "_grokparsefailure"
}
} else {
mutate {
rename => { "syslog_message" => "message" }
}
}
}
}
output {
# Flag & 0x01 => Storage
# Flag & 0x10 => Broadcast
# If the flag is 1 or 3, we have to store the data in the collector database
if [@metadata][flag] == 1 or [@metadata][flag] == 3 {
# If the type is 'logs', send the data to elasticsearch
if [@metadata][type] == "logs" {
elasticsearch {
hosts => "{{ ansible_default_ipv4.address }}"
{% if openbach_ldap_auth is defined and openbach_ldap_auth %}
user => "{{ openbach_backend_admin_name }}"
password => "{{ openbach_backend_admin_password }}"
{% else %}
user => "{{ elasticsearch_cluster_name }}"
{% endif %}
}
}
# If type is 'stats', send the data to influxdb
if [@metadata][type] == "stats" {
influxdb {
measurement => "%{@job_name}"
use_event_fields_for_data_points => true
exclude_fields => [ "@job_name", "@timestamp" ]
send_as_tags => [ "@owner_scenario_instance_id", "@scenario_instance_id", "@job_instance_id", "@agent_name", "@stored_file", "@suffix" ]
host => "{{ ansible_default_ipv4.address }}"
port => {{ influxdb_port }}
db => "{{ influxdb_database_name }}"
time_precision => "{{ influxdb_database_precision }}"
retention_policy => "{{ influxdb_database_name }}"
}
}
}
# Broadcast the message to the Auditorium
if [@metadata][flag] == 2 or [@metadata][flag] == 3 {
{{ auditorium_broadcast_mode }} {
host => "{{ auditorium_ip }}"
port => {{ auditorium_broadcast_port }}
}
}
}
......@@ -13,29 +13,18 @@ configure(){
while true; do
case "$STATE" in
1)
db_beginblock
db_input high opensand-collector/manager/ip || true
;;
2)
db_input high opensand-collector/manager/stats/port || true
;;
3)
db_input high opensand-collector/manager/logs/port || true
db_endblock
;;
4)
db_beginblock
db_input high opensand-collector/collector/stats/port || true
;;
5)
2)
db_input high opensand-collector/collector/logs/port || true
db_endblock
;;
6)
3)
db_beginblock
db_input high opensand-collector/elasticsearch/port || true
;;
7)
4)
db_input high opensand-collector/influxdb/port || true
db_endblock
;;
......@@ -69,9 +58,6 @@ configure(){
}
unseen(){
db_fset opensand-collector/manager/ip seen false
db_fset opensand-collector/manager/stats/port seen false
db_fset opensand-collector/manager/logs/port seen false
db_fset opensand-collector/collector/stats/port seen false
db_fset opensand-collector/collector/logs/port seen false
db_fset opensand-collector/elasticsearch/port seen false
......
......@@ -34,6 +34,14 @@ create_opensand_db(){
curl -i -XPOST http://localhost:${INFLUXDB_PORT}/query --data-urlencode "q=CREATE DATABASE opensand"
}
create_influxdb_conf(){
db_get opensand-collector/influxdb/port
INFLUXDB_PORT="${RET}"
CONFIG_FILE="/etc/influxdb/influxdb.conf"
sed -i "/bind-address = \":8086\"/c\ bind-address = \"${INFLUXDB_PORT}\"" "${CONFIG_FILE}"
}
create_elasticsearch_conf(){
db_get opensand-collector/elasticsearch/port
ELASTICSEARCH_PORT="${RET}"
......@@ -62,12 +70,6 @@ create_logstash_conf(){
STATS_PORT="${RET}"
db_get opensand-collector/collector/logs/port
LOGS_PORT="${RET}"
db_get opensand-collector/manager/ip
MANAGER_IP="${RET}"
db_get opensand-collector/manager/stats/port
MANAGER_STATS_PORT="${RET}"
db_get opensand-collector/manager/logs/port
MANAGER_LOGS_PORT="${RET}"
CONFIG_FILE="/etc/logstash/conf.d/collector.conf"
cat << EOF > "${CONFIG_FILE}"
......@@ -162,6 +164,7 @@ restart_services(){
case "$1" in
configure)
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch
create_influxdb_conf
create_elasticsearch_conf
create_logstash_conf
install_logstash_plugin
......
......@@ -21,10 +21,10 @@ set -e
remove_config_files(){
EXE_RM="/bin/rm"
ELASTIC_CONF="/etc/elasticsearch/elasticsearch.yml"
INFLUX_CONF="/etc/logstash/conf.d/collector.conf"
LOGSTASH_CONF="/etc/logstash/conf.d/collector.conf"
${EXE_RM} -f ${ELASTIC_CONF} || true
${EXE_RM} -f ${INFLUX_CONF} || true
${EXE_RM} -f ${LOGSTASH_CONF} || true
}
......
Template: opensand-collector/manager/ip
Type: string
Default: localhost
Description: Manager IP
Address of the manager so Logstash can forward logs
and statistics.
Template: opensand-collector/manager/stats/port
Type: string
Default: 5359
Description: Manager Statistics Port
Port that the manager uses to listen to incomming statistics.
Template: opensand-collector/manager/logs/port
Type: string
Default: 5360
Description: Manager Logs Port
Port that the manager uses to listen to incomming logs.
Template: opensand-collector/collector/stats/port
Type: string
Default: 5361
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment