Setting up ELK using Rexify

During this meeting Ferenc Erki, the lead developer of Rex is going to lead us setting up a system using several different Linux distributions and handling the differences.

We will also attempt to set up a full ELK stack.

We will start where we left of las time when FErki gave us an introduction to Rex

Plan

ELK

Most of this will be probably delayed to a future meeting:

• Download the rpm file of ElasticSearch and install on a CentOS based box.
• Configure the /etc/elasticsearch/elasticsearch.yml file
• Make sure we can access it locally with curl
• Install Kibana on another machine
• Configure ElastiSearch and Kibana so Kibana can access ElasticSearch
• Configure Nginx on the Kibana machine as a reverse proxy and add Basic Authentication.
• Install Metricbeat on all the machines and make them send the data to the ElasticSearch
• Add different tags to the instances.
• Install our log generator application on one of the boxes.
• Install Logstash on one of the boxes
• Install Filebeat on the box with the log generator application
• Configure filebeat to process the logfile and send it to the Logstash
• Configure Logstash to accept the data from filebeat and send it to ElasTicsearch

• ELK with Ansible
• Rex::Group::Lookup::YAML for inventory

examples/ansible/elk/Rexfile

use Rex -feature => [qw( 1.4 exec_autodie)];
use Rex::Group::Lookup::YAML;
use Rex::CodeMaven::Nginx;
use Rex::CodeMaven::Elasticsearch;
# Module::Pluggable

groups_yaml('rex_inventory.yml');

desc 'Just printing hostname';
task 'print_hostname', sub {
    say run('hostname');
};


# vim: syntax=perl

examples/ansible/elk/rex_inventory.yml

demo:
 - 159.203.85.198

examples/ansible/elk/lib/Rex/CodeMaven/Elasticsearch.pm

package Rex::CodeMaven::Elasticsearch;
use strict;
use warnings;

use Rex -base;

desc 'Base Setup';
task base_setup => sub {
    update_package_db;
    pkg 'wget', ensure => 'present';
};


desc 'Setup Elasticsearch';
task setup => sub {
    needs 'base_setup';

    my $elastic = 'elasticsearch-7.11.2-x86_64.rpm';
    my $project_root = '/root';
    my $url = "https://artifacts.elastic.co/downloads/elasticsearch/$elastic";
    my $dest = "$project_root/$elastic";
    file $project_root, ensure => 'directory';
    run("wget $url -O $dest", unless => "test -e $dest");
    run("rpm -vi $dest", unless => "rpm -q elasticsearch");

    needs 'config';
    service 'elasticsearch', ensure => 'started';
};

desc 'Config Elasticsearch';
task config => sub {
    file '/etc/elasticsearch/elasticsearch.yml',
        source => 'files/etc/elasticsearch/elasticsearch.yml',
        on_change => sub {
            service 'elasticsearch' => 'restart';
        };
};

task verify => sub {
    say run("curl http://localhost:9200");
};


1;

examples/ansible/elk/lib/Rex/CodeMaven/Nginx.pm

package Rex::CodeMaven::Nginx;
use strict;
use warnings;

use Rex -base;

desc 'Setup nginx';
task setup => sub {
    update_package_db;
    pkg 'nginx', ensure => 'present';
    service 'nginx', ensure => 'started';
};

desc 'Configure Nginx';
task configure => sub {
    my $reload_needed = FALSE;
    # We only checked this on CentOs
    my $nginx_root_dir ="/etc/nginx";
    file "$nginx_root_dir/nginx.conf",
        source => 'files/etc/nginx/nginx.conf',
        on_change => sub {
            $reload_needed = TRUE;
        };


    my $nginx_conf_dir = case operating_system, {
                qr{Debian|Ubuntu}i  => "$nginx_root_dir/sites-enabled",
                qr{Fedora|Centos}i  => "$nginx_root_dir/conf.d",
              };

    file "$nginx_conf_dir/default", ensure => 'absent';

    file "$nginx_conf_dir/nginx-elk.conf",
        source => 'files/etc/nginx/conf.d/nginx-elk.conf',
        on_change => sub {
            $reload_needed = TRUE;
        };

    file "/usr/share/nginx/html/.htpasswd",
        source => 'files/usr/share/nginx/html/.htpasswd',
        on_change => sub {
            $reload_needed = TRUE;
        };

    run("setsebool httpd_can_network_connect on -P");

    service 'nginx' => 'reload' if $reload_needed;
};

1;

rex -u root -d -g demo CodeMaven:Elasticsearch:setup
rex -u root -d -g demo CodeMaven:Elasticsearch:config
rex -u root -d -g demo CodeMaven:Elasticsearch:verify
rex -u root -d -g demo CodeMaven:Nginx:setup
rex -u root -d -g demo CodeMaven:Nginx:configure