Commit 47a89799 authored by Mathias Ettinger's avatar Mathias Ettinger

[WIP] Adapt collector packaging to use standalone libs

parent 6ab05ab7
opensand-collector (5.2.0) xenial; urgency=low
* Release 5.2.0
-- mettinger <mathias.ettinger@viveris.fr> Tue, 19 Mar 2020 15:00:00 +0200
opensand-collector (5.1.2) xenial; urgency=low
* Release 5.1.2
......
input {
tcp {
port => {{ logstash_stats_port }}
add_field => { "[@metadata][type]" => "stats" }
}
udp {
port => {{ logstash_stats_port }}
add_field => { "[@metadata][type]" => "stats" }
}
syslog {
port => {{ logstash_logs_port }}
add_field => { "[@metadata][type]" => "logs" }
}
}
filter {
# If the type is 'stats', we parse the message and delete useless field
if [@metadata][type] == "stats" {
mutate {
# Remove these fields first so stats can override them
remove_field => [ "@version", "host", "tags", "type", "port" ]
}
json {
source => "message"
}
# If no flag provided or flag is 0 or the json parser has failed, drop the message
if !([_metadata][flag]) or [_metadata][flag] == 0 or "_jsonparsefailure" in [tags] {
drop {}
}
mutate {
remove_field => "message"
rename => { "_metadata" => "@metadata" }
# Put back the field we just overwrite & Save important fields for export (InfluxDB + broadcast)
add_field => {
"[@metadata][type]" => "stats"
"@owner_scenario_instance_id" => "%{[@metadata][owner_scenario_instance_id]}"
"@scenario_instance_id" => "%{[@metadata][scenario_instance_id]}"
"@job_instance_id" => "%{[@metadata][job_instance_id]}"
"@agent_name" => "%{[@metadata][agent_name]}"
"@job_name" => "%{[@metadata][job_name]}"
"@stored_file" => "%{[@metadata][is_file]}"
}
}
date {
match => [ "[@metadata][time]", "UNIX_MS" ]
}
if ([@metadata][suffix]) {
mutate {
add_field => { "@suffix" => "%{[@metadata][suffix]}" }
}
}
}
# If the type is 'logs', we add the flag (broadcast on severity ERROR or more)
if [@metadata][type] == "logs" {
if [severity] <= 3 {
mutate {
add_field => { "[@metadata][flag]" => 3 }
}
} else {
mutate {
add_field => { "[@metadata][flag]" => 1 }
}
}
grok {
patterns_dir => ["/etc/logstash/conf.d/pattern/"]
match => { "message" => "OWNER_SCENARIO_INSTANCE_ID %{NUMBER:owner_scenario_instance_id}, SCENARIO_INSTANCE_ID %{NUMBER:scenario_instance_id}, JOB_INSTANCE_ID %{NUMBER:job_instance_id}, AGENT_NAME %{HOSTNAME:agent_name}, %{GREEDYDATA:syslog_message}" }
}
mutate {
convert => { "[@metadata][flag]" => "integer" }
remove_field => [ "tags", "type", "port" ]
}
if "_grokparsefailure" in [tags] {
mutate {
remove_tag => "_grokparsefailure"
}
} else {
mutate {
rename => { "syslog_message" => "message" }
}
}
}
}
output {
# Flag & 0x01 => Storage
# Flag & 0x10 => Broadcast
# If the flag is 1 or 3, we have to store the data in the collector database
if [@metadata][flag] == 1 or [@metadata][flag] == 3 {
# If the type is 'logs', send the data to elasticsearch
if [@metadata][type] == "logs" {
elasticsearch {
hosts => "{{ ansible_default_ipv4.address }}"
{% if openbach_ldap_auth is defined and openbach_ldap_auth %}
user => "{{ openbach_backend_admin_name }}"
password => "{{ openbach_backend_admin_password }}"
{% else %}
user => "{{ elasticsearch_cluster_name }}"
{% endif %}
}
}
# If type is 'stats', send the data to influxdb
if [@metadata][type] == "stats" {
influxdb {
measurement => "%{@job_name}"
use_event_fields_for_data_points => true
exclude_fields => [ "@job_name", "@timestamp" ]
send_as_tags => [ "@owner_scenario_instance_id", "@scenario_instance_id", "@job_instance_id", "@agent_name", "@stored_file", "@suffix" ]
host => "{{ ansible_default_ipv4.address }}"
port => {{ influxdb_port }}
db => "{{ influxdb_database_name }}"
time_precision => "{{ influxdb_database_precision }}"
retention_policy => "{{ influxdb_database_name }}"
}
}
}
# Broadcast the message to the Auditorium
if [@metadata][flag] == 2 or [@metadata][flag] == 3 {
{{ auditorium_broadcast_mode }} {
host => "{{ auditorium_ip }}"
port => {{ auditorium_broadcast_port }}
}
}
}
......@@ -3,13 +3,6 @@
# Source debconf library.
. /usr/share/debconf/confmodule
IFACES_LIST=$(ip a | sed -rn "s/^[0-9]+: ([^:]+): .*/\1/p" | paste -d',' -s)
IFACES_LIST=$(echo ${IFACES_LIST} | sed 's/^lo,//' | sed 's/,lo$//' | sed 's/,lo,/,/')
IFACES_LIST=$(echo ${IFACES_LIST} | sed 's/opensand_tun//')
IFACES_LIST=$(echo ${IFACES_LIST} | sed 's/opensand_tap//')
IFACES_LIST=$(echo ${IFACES_LIST} | sed 's/opensand_br//')
IFACES_LIST=$(echo ${IFACES_LIST} | sed 's/,\+/,/g' | sed 's/^,//' | sed 's/,$//')
IFACES_LIST=$(echo ${IFACES_LIST} | sed 's/,/,\ /g')
usage(){
/bin/echo -e "Usage: $0 command\n\twith command: configure or reconfigure"
......@@ -21,11 +14,29 @@ configure(){
case "$STATE" in
1)
db_beginblock
db_input high opensand-collector/service/type || true
db_input high opensand-collector/manager/ip || true
;;
2)
db_subst opensand-collector/service/interface ifaces_list "${IFACES_LIST}"
db_input high opensand-collector/service/interface || true
db_input high opensand-collector/manager/stats/port || true
;;
3)
db_input high opensand-collector/manager/logs/port || true
db_endblock
;;
4)
db_beginblock
db_input high opensand-collector/collector/stats/port || true
;;
5)
db_input high opensand-collector/collector/logs/port || true
db_endblock
;;
6)
db_beginblock
db_input high opensand-collector/elasticsearch/port || true
;;
7)
db_input high opensand-collector/influxdb/port || true
db_endblock
;;
*)
......@@ -58,8 +69,13 @@ configure(){
}
unseen(){
db_fset opensand-collector/service/type seen false
db_fset opensand-collector/service/interface seen false
db_fset opensand-collector/manager/ip seen false
db_fset opensand-collector/manager/stats/port seen false
db_fset opensand-collector/manager/logs/port seen false
db_fset opensand-collector/collector/stats/port seen false
db_fset opensand-collector/collector/logs/port seen false
db_fset opensand-collector/elasticsearch/port seen false
db_fset opensand-collector/influxdb/port seen false
}
db_capb backup
......
......@@ -5,6 +5,9 @@
set -e
# Source debconf library.
. /usr/share/debconf/confmodule
# summary of how this script can be called:
# * <postinst> `configure' <most-recently-configured-version>
# * <old-postinst> `abort-upgrade' <new version>
......@@ -17,23 +20,134 @@ set -e
# for details, see http://www.debian.org/doc/debian-policy/ or
# the debian-policy package
set_owner(){
# Set owner to files and directories
EXE_CHOWN="/bin/chown"
USER_NAME="opensand"
create_elasticsearch_conf(){
db_get opensand-collector/elasticsearch/port
ELASTICSEARCH_PORT="${RET}"
CONFIG_FILE="/etc/elasticsearch/elasticsearch.yml"
cat << EOF > "${CONFIG_FILE}"
path.data: /var/lib/elasticsearch
path.log: /var/log/elasticsearch
cluster.name: "opensand"
node.name: "collector"
network.host: 0.0.0.0
http.port: ${ELASTICSEARCH_PORT}
path.repo: ["/tmp"]
EOF
chown root:elasticsearch "${CONFIG_FILE}"
}
create_logstash_conf(){
db_get opensand-collector/elasticsearch/port
ELASTICSEARCH_PORT="${RET}"
db_get opensand-collector/influxdb/port
INFLUXDB_PORT="${RET}"
db_get opensand-collector/collector/stats/port
STATS_PORT="${RET}"
db_get opensand-collector/collector/logs/port
LOGS_PORT="${RET}"
db_get opensand-collector/manager/ip
MANAGER_IP="${RET}"
db_get opensand-collector/manager/stats/port
MANAGER_STATS_PORT="${RET}"
db_get opensand-collector/manager/logs/port
MANAGER_LOGS_PORT="${RET}"
CONFIG_FILE="/etc/logstash/conf.d/collector.conf"
cat << EOF > "${CONFIG_FILE}"
input {
udp { port => ${STATS_PORT} }
tcp { port => ${STATS_PORT} }
udp { port => ${LOGS_PORT} }
tcp { port => ${LOGS_PORT} }
}
filter {
mutate {
remove_field => [ "@version", "host", "tags", "type", "port" ]
}
if ("[DEBUG]" in [message] or "[INFO]" in [message] or "[NOTICE]" in [message] or "[WARNING]" in [message] or "[ERROR]" in [message] or "[CRITICAL]" in [message]) {
dissect {
mapping => { "message" => "[%{timestamp}][%{log_level}][%{log_name}]%{log_message}" }
convert_datatype => { "timestamp" => "int" }
}
mutate {
rename => { "log_message" => "message" }
}
} else {
grok {
match => { "message" => "%{NUMBER:timestamp:int} %{GREEDYDATA:message}" }
overwrite => [ "message" ]
}
kv {
allow_duplicate_values => false
field_split => " "
value_split => " "
remove_field => [ "message" ]
target => "kv"
}
ruby {
code => '
event.get("kv").each do |key, value|
event.set(key, value.to_f)
end'
remove_field => [ "kv" ]
}
}
date {
match => [ "timestamp", "UNIX_MS" ]
remove_field => [ "timestamp" ]
}
}
output {
if "message" in event.fields {
elasticserach {
hosts => "localhost:${ELASTICSEARCH_PORT}"
}
} else {
influxdb {
measurement => "opensand"
use_event_fields_for_data_points => true
host => "localhost"
port => ${INFLUXDB_PORT}
db => "opensand"
time_precision => "ms"
retention_policy => "opensand"
}
}
}
EOF
}
${EXE_CHOWN} ${USER_NAME} /var/run/sand-collector
install_logstash_plugin(){
cd /usr/share/logstash
bin/logstash-plugin install logstash-output-influxdb
}
restart_syslog(){
service rsyslog restart
restart_services(){
systemctl daemon-reload
systemctl enable elasticsearch.service
systemctl enable influxdb.service
systemctl enable logstash.service
systemctl restart elasticsearch.service
systemctl restart influxdb.service
systemctl restart logstash.service
}
case "$1" in
configure)
restart_syslog
set_owner
chown -R elasticsearch:elasticsearch /usr/share/elasticsearch
create_elasticsearch_conf
create_logstash_conf
install_logstash_plugin
restart_services
;;
abort-upgrade|abort-remove|abort-deconfigure)
......
......@@ -18,21 +18,33 @@ set -e
# for details, see http://www.debian.org/doc/debian-policy/ or
# the debian-policy package
remove_syslog_conf(){
remove_config_files(){
EXE_RM="/bin/rm"
SYSLOG_CONF="/etc/rsyslog.d/opensand-collector.conf"
LOGROTATE_CONF="/etc/logrotate.d/opensand-collector.conf"
ELASTIC_CONF="/etc/elasticsearch/elasticsearch.yml"
INFLUX_CONF="/etc/logstash/conf.d/collector.conf"
${EXE_RM} -f ${SYSLOG_CONF} || true
${EXE_RM} -f ${LOGROTATE_CONF} || true
${EXE_RM} -f ${ELASTIC_CONF} || true
${EXE_RM} -f ${INFLUX_CONF} || true
}
stop_services(){
systemctl disable elasticsearch.service
systemctl disable influxdb.service
systemctl disable logstash.service
systemctl stop elasticsearch.service
systemctl stop influxdb.service
systemctl stop logstash.service
}
case "$1" in
purge)
stop_services
remove_syslog_conf
;;
remove)
stop_services
;;
upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
......
#!/bin/sh
# preinst script for opensand-core
#
# see: dh_installdeb(1)
set -e
# summary of how this script can be called:
# * <new-preinst> `install'
# * <new-preinst> `install' <old-version>
# * <new-preinst> `upgrade' <old-version>
# * <old-preinst> `abort-upgrade' <new-version>
# for details, see http://www.debian.org/doc/debian-policy/ or
# the debian-policy package
add_user(){
# add user opensand
NEW_USER="opensand"
EXE_GREP="/bin/grep"
EXE_ADDUSER="/usr/sbin/adduser"
EXE_NOLOGIN="/usr/sbin/nologin"
# Check that user does not exist yet
if [ -z "$(${EXE_GREP} ${NEW_USER} /etc/passwd)" ]
then
# Create or modify user
${EXE_ADDUSER} --system --no-create-home --quiet --shell ${EXE_NOLOGIN} ${NEW_USER}
fi
}
case "$1" in
install)
add_user
;;
upgrade)
;;
abort-upgrade)
;;
*)
echo "preinst called with unknown argument \`$1'" >&2
exit 1
;;
esac
# dh_installdeb will replace this with shell code automatically
# generated by other debhelper scripts.
#DEBHELPER#
exit 0
......@@ -9,19 +9,5 @@
# Uncomment this to turn on verbose mode.
#export DH_VERBOSE=1
override_dh_auto_install:
python setup.py install --root=$(CURDIR)/debian/opensand-collector --install-layout=deb
override_dh_install:
dh_install --with=systemd
# Configuration for syslog/logrotate
install -D -m 0644 conf/syslog.conf $(CURDIR)/debian/opensand-collector/etc/rsyslog.d/opensand-collector.conf
install -D -m 0644 conf/logrotate.conf $(CURDIR)/debian/opensand-collector/etc/logrotate.d/opensand-collector.conf
override_dh_pysupport:
%:
dh $@ --with=systemd
Template: opensand-collector/service/type
Template: opensand-collector/manager/ip
Type: string
Description: Platform ID
Select the ID of the platform (leave empty by default). This name will be
used to form the zeroconf service type (_{platform_id}_opensand._tcp).
No spaces or special characters are admitted.
All the hosts from the same platform MUST have the same platform ID.
Default: localhost
Description: Manager IP
Address of the manager so Logstash can forward logs
and statistics.
Template: opensand-collector/service/interface
Type: select
Choices: ${ifaces_list}
Description: Name of the interface for service publishing
The name of the interface where the service should be published,
empty for all
Template: opensand-collector/manager/stats/port
Type: string
Default: 5359
Description: Manager Statistics Port
Port that the manager uses to listen to incomming statistics.
Template: opensand-collector/manager/logs/port
Type: string
Default: 5360
Description: Manager Logs Port
Port that the manager uses to listen to incomming logs.
Template: opensand-collector/collector/stats/port
Type: string
Default: 5361
Description: Statistics Port
Port that Logstash will use to listen to incomming
statistics from OpenSAND binaries.
Template: opensand-collector/collector/logs/port
Type: string
Default: 5362
Description: Logs Port
Port that Logstash will use to listen to incomming
logs from OpenSAND binaries.
Template: opensand-collector/elasticsearch/port
Type: string
Default: 9200
Description: ElasticSearch Port
Port that ElasticSearch will use to listen to incomming
logs from Logstash.
Template: opensand-collector/influxdb/port
Type: string
Default: 8086
Description: InfluxDB Port
Port that InfluxDB will use to listen to incomming
statistics from Logstash.
#!/bin/bash
THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
usage()
{
echo 'Usage : $0 -s /path/to/sources -d /path/to/build'
}
while getopts ":s:d:h" o; do
case "${o}" in
s)
SRC_DIR=`echo $OPTARG | sed 's/\/$//'`
;;
d)
DST_DIR=`echo $OPTARG | sed 's/\/$//'`
;;
h)
usage
exit 0
;;
*)
usage
exit 1
;;
esac
done
shift $((OPTIND-1))
# Check if valid source dir
if [ ! -d "$SRC_DIR" ]
then
echo "Invalid source dir \"$SRC_DIR\"" && usage
fi
# Check if valid destination dir
if [ ! -d "$DST_DIR" ]
then
echo "Invalid destination dir \"$DST_DIR\"" && usage
fi
# Create symbolic link to service file
cd $THIS_DIR
ln -s ../init/opensand-collector.service $DST_DIR/debian/opensand-collector.service
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment