diff --git a/pillar/logstash/manager.sls b/pillar/logstash/manager.sls index cfeb0a6ae..41a2197fd 100644 --- a/pillar/logstash/manager.sls +++ b/pillar/logstash/manager.sls @@ -2,9 +2,7 @@ logstash: pipelines: manager: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/receiver.sls b/pillar/logstash/receiver.sls index 09c2549e6..4d0637dde 100644 --- a/pillar/logstash/receiver.sls +++ b/pillar/logstash/receiver.sls @@ -2,8 +2,7 @@ logstash: pipelines: receiver: config: - - so/0009_input_beats.conf - - so/0010_input_hhbeats.conf - so/0011_input_endgame.conf + - so/0012_input_elastic_agent.conf - so/9999_output_redis.conf.jinja - \ No newline at end of file + diff --git a/pillar/logstash/search.sls b/pillar/logstash/search.sls index fb10d18e7..0b660b7ef 100644 --- a/pillar/logstash/search.sls +++ b/pillar/logstash/search.sls @@ -3,16 +3,5 @@ logstash: search: config: - so/0900_input_redis.conf.jinja - - so/9000_output_zeek.conf.jinja - - so/9002_output_import.conf.jinja - - so/9034_output_syslog.conf.jinja - - so/9050_output_filebeatmodules.conf.jinja - - so/9100_output_osquery.conf.jinja - - so/9400_output_suricata.conf.jinja - - so/9500_output_beats.conf.jinja - - so/9600_output_ossec.conf.jinja - - so/9700_output_strelka.conf.jinja - - so/9800_output_logscan.conf.jinja - - so/9801_output_rita.conf.jinja - so/9805_output_elastic_agent.conf.jinja - so/9900_output_endgame.conf.jinja diff --git a/salt/allowed_states.map.jinja b/salt/allowed_states.map.jinja index 2f6cc60a0..823b7b647 100644 --- a/salt/allowed_states.map.jinja +++ b/salt/allowed_states.map.jinja @@ -107,7 +107,8 @@ 'zeek', 'schedule', 'tcpreplay', - 'docker_clean' + 'docker_clean', + 'elastic-fleet' ], 'so-manager': [ 'salt.master', diff --git a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load index b3148f7aa..f409d9a17 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load +++ b/salt/common/tools/sbin/so-elastic-fleet-integration-policy-load @@ -66,10 +66,10 @@ curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POS echo # RITA Logs -echo -echo "Setting up RITA package policy..." -curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' -echo +#echo +#echo "Setting up RITA package policy..." +#curl -K /opt/so/conf/elasticsearch/curl.config -b "sid=$SESSIONCOOKIE" -L -X POST "localhost:5601/api/fleet/package_policies" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d'{ "policy_id": "so-grid-nodes", "package": { "name": "log", "version": "1.1.0" }, "id": "rita-logs", "name": "rita-logs", "description": "RITA Beacon logs", "namespace": "so", "inputs": { "logs-logfile": { "enabled": true, "streams": { "log.log": { "enabled": true, "vars": { "paths": [ "/nsm/rita/beacons.csv", "/nsm/rita/long-connections.csv", "/nsm/rita/short-connections.csv", "/nsm/rita/exploded-dns.csv" ], "data_stream.dataset": "rita", "tags": [], "processors": "- add_fields:\n target: event\n fields:\n category: network\n module: rita\n- if:\n log.file.path: beacons.csv\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.beacon\n- if:\n regexp:\n log.file.path: \"*connections.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.connection\n- if:\n log.file.path: \"exploded-dns.csv\"\n then: \n - add_fields:\n target: \"@metadata\"\n fields:\n pipeline: rita.dns" }}}}}}' +#echo # Elasticsearch logs echo diff --git a/salt/common/tools/sbin/so-elastic-fleet-setup b/salt/common/tools/sbin/so-elastic-fleet-setup index 4e3d7639c..c945aeec3 100755 --- a/salt/common/tools/sbin/so-elastic-fleet-setup +++ b/salt/common/tools/sbin/so-elastic-fleet-setup @@ -19,10 +19,22 @@ printf "\n" curl -K /opt/so/conf/elasticsearch/curl.config -L -X PUT "localhost:5601/api/fleet/settings" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{"fleet_server_hosts":["https://{{ GLOBALS.manager_ip }}:8220"]}' printf "\n\n" -# Create Logstash Output payload +# Configure certificates mkdir -p /opt/so/conf/elastic-fleet/certs cp /etc/ssl/certs/intca.crt /opt/so/conf/elastic-fleet/certs cp /etc/pki/elasticfleet* /opt/so/conf/elastic-fleet/certs + +{% if grains.role == 'so-import' %} +# Add SO-Manager Elasticsearch Ouput +ESCACRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) +JSON_STRING=$( jq -n \ + --arg ESCACRT "$ESCACRT" \ + '{"name":"so-manager_elasticsearch2","id":"so-manager_elasticsearch2","type":"elasticsearch","hosts":["https://{{ GLOBALS.manager_ip }}:9200"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate_authorities": [$ESCACRT]}}' ) +curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" +printf "\n\n" + +{% else %} +# Create Logstash Output payload LOGSTASHCRT=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/elasticfleet.crt) LOGSTASHKEY=$(openssl rsa -in /opt/so/conf/elastic-fleet/certs/elasticfleet.key) LOGSTASHCA=$(openssl x509 -in /opt/so/conf/elastic-fleet/certs/intca.crt) @@ -30,12 +42,13 @@ JSON_STRING=$( jq -n \ --arg LOGSTASHCRT "$LOGSTASHCRT" \ --arg LOGSTASHKEY "$LOGSTASHKEY" \ --arg LOGSTASHCA "$LOGSTASHCA" \ - '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' + '{"name":"so-manager_logstash","id":"so-manager_logstash","type":"logstash","hosts":["{{ GLOBALS.manager_ip }}:5055"],"is_default":true,"is_default_monitoring":true,"config_yaml":"","ssl":{"certificate": $LOGSTASHCRT,"key": $LOGSTASHKEY,"certificate_authorities":[ $LOGSTASHCA ]}}' ) # Add SO-Manager Logstash Ouput curl -K /opt/so/conf/elasticsearch/curl.config -L -X POST "localhost:5601/api/fleet/outputs" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d "$JSON_STRING" printf "\n\n" +{%- endif %} # Add Elastic Fleet Integrations diff --git a/salt/common/tools/sbin/so-image-common b/salt/common/tools/sbin/so-image-common index 3851d8b4a..945d3f4ed 100755 --- a/salt/common/tools/sbin/so-image-common +++ b/salt/common/tools/sbin/so-image-common @@ -36,7 +36,9 @@ container_list() { "so-steno" "so-suricata" "so-telegraf" - "so-zeek" + "so-zeek" + "so-elastic-agent" + "so-elastic-agent-builder" ) elif [ $MANAGERCHECK != 'so-helix' ]; then TRUSTED_CONTAINERS=( diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 11b445c62..3d1182255 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -2677,6 +2677,9 @@ elasticsearch: delete: 365 index_sorting: False index_template: + data_stream: + hidden: false + allow_custom_routing: false index_patterns: - logs-kratos-so* template: diff --git a/salt/firewall/assigned_hostgroups.map.yaml b/salt/firewall/assigned_hostgroups.map.yaml index 2950dc8be..fc0a629e7 100644 --- a/salt/firewall/assigned_hostgroups.map.yaml +++ b/salt/firewall/assigned_hostgroups.map.yaml @@ -454,6 +454,7 @@ role: - {{ portgroups.influxdb }} - {{ portgroups.elasticsearch_rest }} - {{ portgroups.elasticsearch_node }} + - {{ portgroups.elastic_agent_control }} sensors: portgroups: - {{ portgroups.beats_5044 }} @@ -471,6 +472,10 @@ role: elasticsearch_rest: portgroups: - {{ portgroups.elasticsearch_rest }} + elastic_agent_endpoint: + portgroups: + - {{ portgroups.elastic_agent_control }} + - {{ portgroups.elastic_agent_data }} analyst: portgroups: - {{ portgroups.nginx }} diff --git a/salt/logstash/pipelines/config/so/0009_input_beats.conf b/salt/logstash/pipelines/config/so/0009_input_beats.conf deleted file mode 100644 index 8643a64b4..000000000 --- a/salt/logstash/pipelines/config/so/0009_input_beats.conf +++ /dev/null @@ -1,11 +0,0 @@ -input { - beats { - port => "5044" - tags => [ "beat-ext" ] - } -} -filter { - mutate { - rename => {"@metadata" => "metadata"} - } -} diff --git a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf b/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf deleted file mode 100644 index 050d01d73..000000000 --- a/salt/logstash/pipelines/config/so/0010_input_hhbeats.conf +++ /dev/null @@ -1,40 +0,0 @@ -input { - beats { - port => "5644" - ssl => true - ssl_certificate_authorities => ["/usr/share/filebeat/ca.crt"] - ssl_certificate => "/usr/share/logstash/filebeat.crt" - ssl_key => "/usr/share/logstash/filebeat.key" - #tags => [ "beat" ] - } -} -filter { - if [type] == "ids" or [type] =~ "bro" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "sensor_name" => "%{[beat][name]}" } - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] =~ "ossec" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_field => { "syslog-host_from" => "%{[beat][name]}" } - remove_field => [ "beat", "prospector", "input", "offset" ] - } - } - if [type] == "osquery" { - mutate { - rename => { "host" => "beat_host" } - remove_tag => ["beat"] - add_tag => ["osquery"] - } - json { - source => "message" - target => "osquery" - } - } -} diff --git a/salt/logstash/pipelines/config/so/0800_input_eval.conf b/salt/logstash/pipelines/config/so/0800_input_eval.conf deleted file mode 100644 index 35a977d04..000000000 --- a/salt/logstash/pipelines/config/so/0800_input_eval.conf +++ /dev/null @@ -1,204 +0,0 @@ -# Updated by: Mike Reeves -# Last Update: 11/1/2018 - -input { - file { - path => "/suricata/eve.json" - type => "ids" - add_field => { "engine" => "suricata" } - } - file { - path => "/nsm/zeek/logs/current/conn*.log" - type => "zeek.conn" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dce_rpc*.log" - type => "zeek.dce_rpc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dhcp*.log" - type => "zeek.dhcp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dnp3*.log" - type => "zeek.dnp3" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dns*.log" - type => "zeek.dns" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/dpd*.log" - type => "zeek.dpd" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/files*.log" - type => "zeek.files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ftp*.log" - type => "zeek.ftp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/http*.log" - type => "zeek.http" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/intel*.log" - type => "zeek.intel" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/irc*.log" - type => "zeek.irc" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/kerberos*.log" - type => "zeek.kerberos" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/modbus*.log" - type => "zeek.modbus" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/mysql*.log" - type => "zeek.mysql" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/notice*.log" - type => "zeek.notice" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ntlm*.log" - type => "zeek.ntlm" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/pe*.log" - type => "zeek.pe" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/radius*.log" - type => "zeek.radius" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rdp*.log" - type => "zeek.rdp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/rfb*.log" - type => "zeek.rfb" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/signatures*.log" - type => "zeek.signatures" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/sip*.log" - type => "zeek.sip" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_files*.log" - type => "zeek.smb_files" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smb_mapping*.log" - type => "zeek.smb_mapping" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/smtp*.log" - type => "zeek.smtp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/snmp*.log" - type => "zeek.snmp" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/socks*.log" - type => "zeek.socks" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/software*.log" - type => "zeek.software" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssh*.log" - type => "zeek.ssh" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/ssl*.log" - type => "zeek.ssl" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/syslog*.log" - type => "zeek.syslog" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/tunnel*.log" - type => "zeek.tunnels" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/weird*.log" - type => "zeek.weird" - tags => ["zeek"] - } - file { - path => "/nsm/zeek/logs/current/x509*.log" - type => "zeek.x509" - tags => ["zeek"] - } - file { - path => "/wazuh/alerts/alerts.json" - type => "ossec" - } -# file { -# path => "/wazuh/archives/archives.json" -# type => "ossec_archive" -# } - file { - path => "/osquery/logs/result.log" - type => "osquery" - tags => ["osquery"] - } - file { - path => "/strelka/strelka.log" - type => "strelka" - } -} -filter { - if "import" in [tags] { - mutate { - #add_tag => [ "conf_file_0007"] - } - } -} diff --git a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja b/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja deleted file mode 100644 index 7a0848b39..000000000 --- a/salt/logstash/pipelines/config/so/0899_input_minio.conf.jinja +++ /dev/null @@ -1,23 +0,0 @@ -{%- if grains.role == 'so-heavynode' %} -{%- set HOST = GLOBALS.hostname %} -{%- else %} -{%- set HOST = GLOBALS.manager %} -{% endif -%} -{%- set THREADS = salt['pillar.get']('logstash_settings:ls_input_threads', '') %} -{%- set access_key = salt['pillar.get']('minio:access_key', '') %} -{%- set access_secret = salt['pillar.get']('minio:access_secret', '') %} -{%- set INTERVAL = salt['pillar.get']('s3_settings:interval', 5) %} -input { - s3 { - access_key_id => "{{ access_key }}" - secret_access_key => "{{ access_secret }}" - endpoint => "https://{{ HOST }}:9595" - bucket => "logstash" - delete => true - interval => {{ INTERVAL }} - codec => json - additional_settings => { - "force_path_style" => true - } - } -} diff --git a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja b/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja deleted file mode 100644 index 7b8c03f45..000000000 --- a/salt/logstash/pipelines/config/so/9000_output_zeek.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "zeek" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-zeek" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja b/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja deleted file mode 100644 index a57830229..000000000 --- a/salt/logstash/pipelines/config/so/9002_output_import.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if "import" in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-import" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja b/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja deleted file mode 100644 index 4c49c61ea..000000000 --- a/salt/logstash/pipelines/config/so/9034_output_syslog.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "syslog" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-syslog" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja b/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja deleted file mode 100644 index 672a83876..000000000 --- a/salt/logstash/pipelines/config/so/9050_output_filebeatmodules.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if "filebeat" in [metadata][pipeline] { - elasticsearch { - id => "filebeat_modules_metadata_pipeline" - pipeline => "%{[metadata][pipeline]}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-%{[event][module]}-%{+YYYY.MM.dd}" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja b/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja deleted file mode 100644 index 8dbea872e..000000000 --- a/salt/logstash/pipelines/config/so/9100_output_osquery.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "osquery" and "live_query" not in [dataset] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-osquery" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja b/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja deleted file mode 100644 index 7942aa50c..000000000 --- a/salt/logstash/pipelines/config/so/9200_output_firewall.conf.jinja +++ /dev/null @@ -1,12 +0,0 @@ -output { - if [dataset] =~ "firewall" { - elasticsearch { - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-firewall" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja b/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja deleted file mode 100644 index 13df33e16..000000000 --- a/salt/logstash/pipelines/config/so/9400_output_suricata.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "suricata" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ids" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja b/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja deleted file mode 100644 index b4aafecad..000000000 --- a/salt/logstash/pipelines/config/so/9500_output_beats.conf.jinja +++ /dev/null @@ -1,26 +0,0 @@ -output { - if "beat-ext" in [tags] and "import" not in [tags] and "filebeat" not in [metadata][pipeline] { - if [metadata][_id] { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - document_id => "%{[metadata][_id]}" - } - } else { - elasticsearch { - pipeline => "beats.common" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-beats" - ssl => true - ssl_certificate_verification => false - } - } - } -} diff --git a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja b/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja deleted file mode 100644 index ca3eeb6c1..000000000 --- a/salt/logstash/pipelines/config/so/9600_output_ossec.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "ossec" { - elasticsearch { - pipeline => "%{module}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-ossec" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja b/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja deleted file mode 100644 index 281cdda5b..000000000 --- a/salt/logstash/pipelines/config/so/9700_output_strelka.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "strelka" { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-strelka" - ssl => true - ssl_certificate_verification => false - } - } -} - diff --git a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja b/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja deleted file mode 100644 index 8127de23a..000000000 --- a/salt/logstash/pipelines/config/so/9800_output_logscan.conf.jinja +++ /dev/null @@ -1,14 +0,0 @@ -output { - if [module] =~ "logscan" { - elasticsearch { - id => "logscan_pipeline" - pipeline => "logscan.alert" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-logscan" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja b/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja deleted file mode 100644 index 7f9d795e6..000000000 --- a/salt/logstash/pipelines/config/so/9801_output_rita.conf.jinja +++ /dev/null @@ -1,13 +0,0 @@ -output { - if [module] =~ "rita" and "import" not in [tags] { - elasticsearch { - pipeline => "%{module}.%{dataset}" - hosts => "{{ GLOBALS.manager }}" - user => "{{ ES_USER }}" - password => "{{ ES_PASS }}" - index => "so-rita" - ssl => true - ssl_certificate_verification => false - } - } -} diff --git a/salt/top.sls b/salt/top.sls index e29d3b081..4b8531f4d 100644 --- a/salt/top.sls +++ b/salt/top.sls @@ -344,6 +344,7 @@ base: - zeek - schedule - docker_clean + - elastic-fleet '*_receiver and G@saltversion:{{saltversion}}': - match: compound diff --git a/setup/so-functions b/setup/so-functions index 8da453cf9..a76126519 100755 --- a/setup/so-functions +++ b/setup/so-functions @@ -383,16 +383,11 @@ collect_mngr_hostname() { fi } -collect_net_method() { +collect_net_method() { whiptail_net_method - - if [[ "$network_traffic" == *"_MANAGER" ]]; then - whiptail_manager_updates_warning - MANAGERUPDATES=1 - fi - if [[ "$network_traffic" == "PROXY"* ]]; then collect_proxy no_ask + needs_proxy=true fi } diff --git a/setup/so-setup b/setup/so-setup index ec0bc69ed..9ecbed08c 100755 --- a/setup/so-setup +++ b/setup/so-setup @@ -142,6 +142,7 @@ if [[ -f /root/accept_changes ]]; then mv "$setup_log" "$setup_log.bak" [ -f "$error_log" ] && mv "$error_log" "$error_log.bak" reinstall_init + reset_proxy fi title "Parsing Username for Install" @@ -319,6 +320,7 @@ if ! [[ -f $install_opt_file ]]; then check_elastic_license check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -342,6 +344,7 @@ if ! [[ -f $install_opt_file ]]; then check_elastic_license check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -364,6 +367,7 @@ if ! [[ -f $install_opt_file ]]; then ubuntu_check check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -385,6 +389,7 @@ if ! [[ -f $install_opt_file ]]; then ubuntu_check check_requirements "manager" networking_needful + collect_net_method collect_dockernet if [[ $is_iso ]]; then whiptail_airgap @@ -490,6 +495,9 @@ if ! [[ -f $install_opt_file ]]; then percentage=0 es_heapsize ls_heapsize + if [[ $needs_proxy ]]; then + set_proxy + fi set_redirect # Generate Interface Vars generate_interface_vars @@ -595,9 +603,9 @@ if ! [[ -f $install_opt_file ]]; then add_web_user info "Restarting SOC to pick up initial user" logCmd "so-soc-restart" + title "Setting up Elastic Fleet" + logCmd "so-elastic-fleet-setup" if [[ ! $is_import ]]; then - title "Setting up Elastic Fleet" - logCmd "so-elastic-fleet-setup" title "Setting up Playbook" logCmd "so-playbook-reset" fi diff --git a/setup/so-verify b/setup/so-verify index 77cdf7c8e..44c0465ff 100755 --- a/setup/so-verify +++ b/setup/so-verify @@ -36,6 +36,7 @@ log_has_errors() { grep -vE "The Salt Master has cached the public key for this node" | \ grep -vE "Minion failed to authenticate with the master" | \ grep -vE "Failed to connect to ::1" | \ + grep -vE "Failed to set locale" | \ grep -vE "perl-Error-" | \ grep -vE "Failed:\s*?[0-9]+" | \ grep -vE "Status .* was not found" | \ diff --git a/setup/so-whiptail b/setup/so-whiptail index 5615edf41..e5de2cf5b 100755 --- a/setup/so-whiptail +++ b/setup/so-whiptail @@ -816,34 +816,15 @@ whiptail_net_method() { ) local proxy_desc="proxy the traffic for git, docker client, wget, curl, ${pkg_mngr}, and various other SO components through a separate server in your environment." - if [[ $is_minion ]]; then - read -r -d '' options_msg <<- EOM + read -r -d '' options_msg <<- EOM ${options_msg} - - "Direct + Manager" - all traffic passes to the Internet normally, but ${pkg_mngr} updates will instead be pulled from ${mngr_article} manager. - "Proxy" - ${proxy_desc} + EOM + options+=( + " Proxy " "" + ) - "Proxy + Manager" - proxy all traffic from the "Proxy" option except ${pkg_mngr} updates, which will instead pull from the manager. - EOM - - options+=( - " Direct + Manager " "" - " Proxy " "" - " Proxy + Manager " "" - ) - local height=25 - else - read -r -d '' options_msg <<- EOM - ${options_msg} - - "Proxy" - ${proxy_desc} - EOM - options+=( - " Proxy " "" - ) - local height=17 - fi + local height=17 local msg read -r -d '' msg <<- EOM