From 74ef6c0ed065fa5bf374f78f8bc3fcf08e0a8d2b Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 9 Sep 2022 15:30:28 -0400 Subject: [PATCH 1/2] Fix yaml for idh,es,kib,esalert --- salt/elastalert/defaults.yaml | 5 - salt/elastalert/elastalert_config.map.jinja | 8 +- salt/elasticsearch/config.map.jinja | 3 + salt/elasticsearch/defaults.yaml | 377 -------------------- salt/idh/defaults/defaults.yaml | 1 - salt/idh/opencanary_config.map.jinja | 4 +- salt/kibana/config.map.jinja | 3 + salt/kibana/defaults.yaml | 5 +- 8 files changed, 16 insertions(+), 390 deletions(-) diff --git a/salt/elastalert/defaults.yaml b/salt/elastalert/defaults.yaml index f21bab4c3..fe53b52c2 100644 --- a/salt/elastalert/defaults.yaml +++ b/salt/elastalert/defaults.yaml @@ -3,8 +3,6 @@ # https://securityonion.net/license; you may not use this file except in compliance with the # Elastic License 2.0. -{%- set ES_USER = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:user', '') %} -{%- set ES_PASS = salt['pillar.get']('elasticsearch:auth:users:so_elastic_user:pass', '') %} elastalert: config: rules_folder: /opt/elastalert/rules/ @@ -16,7 +14,6 @@ elastalert: minutes: 10 old_query_limit: minutes: 5 - es_host: {{salt['pillar.get']('global:managerip', '')}} es_port: 9200 es_conn_timeout: 55 max_query_size: 5000 @@ -26,8 +23,6 @@ elastalert: use_ssl: true verify_certs: false #es_send_get_body_as: GET - es_username: "{{ ES_USER }}" - es_password: "{{ ES_PASS }}" writeback_index: elastalert_status alert_time_limit: days: 2 diff --git a/salt/elastalert/elastalert_config.map.jinja b/salt/elastalert/elastalert_config.map.jinja index 270872fee..2b9895e1b 100644 --- a/salt/elastalert/elastalert_config.map.jinja +++ b/salt/elastalert/elastalert_config.map.jinja @@ -1,4 +1,8 @@ -{% import_yaml 'elastalert/defaults.yaml' as elastalert_defaults with context %} +{% import_yaml 'elastalert/defaults.yaml' as ELASTALERT with context %} {% set elastalert_pillar = salt['pillar.get']('elastalert:config', {}) %} -{% do salt['defaults.merge'](elastalert_defaults.elastalert.config, elastalert_pillar, in_place=True) %} \ No newline at end of file +{% do salt['defaults.merge'](ELASTALERT.elastalert.config, elastalert_pillar, in_place=True) %} + +{% do ELASTALERT.elastalert.config.update({'es_host': pillar.global.managerip}) %} +{% do ELASTALERT.elastalert.config.update({'es_username': pillar.elasticsearch.auth.users.so_elastic_user.user}) %} +{% do ELASTALERT.elastalert.config.update({'es_password': pillar.elasticsearch.auth.users.so_elastic_user.pass}) %} diff --git a/salt/elasticsearch/config.map.jinja b/salt/elasticsearch/config.map.jinja index 86b9c47ae..7cd79e7b9 100644 --- a/salt/elasticsearch/config.map.jinja +++ b/salt/elasticsearch/config.map.jinja @@ -31,6 +31,9 @@ {# merge with the elasticsearch pillar #} {% set ESCONFIG = salt['pillar.get']('elasticsearch:config', default=ESCONFIG.elasticsearch.config, merge=True) %} +{% do ESCONFIG.elasticsearch.config.node.update({'name': grains.host}) %} +{% do ESCONFIG.elasticsearch.config.cluster.update({'name': grains.host}) %} +{% do ESCONFIG.elasticsearch.config.transport.update({'publish_host': grains.host}) %} {% if salt['pillar.get']('elasticsearch:config:path:repo', False) %} {% for repo in pillar.elasticsearch.config.path.repo %} diff --git a/salt/elasticsearch/defaults.yaml b/salt/elasticsearch/defaults.yaml index 6fa356c61..96206fddd 100644 --- a/salt/elasticsearch/defaults.yaml +++ b/salt/elasticsearch/defaults.yaml @@ -1,11 +1,9 @@ elasticsearch: config: node: - name: {{ grains.host }} attr: box_type: hot cluster: - name: {{ grains.host }} routing: allocation: disk: @@ -22,7 +20,6 @@ elasticsearch: destructive_requires_name: true transport: bind_host: 0.0.0.0 - publish_host: {{ grains.host }} publish_port: 9300 xpack: ml: @@ -60,380 +57,6 @@ elasticsearch: elasticsearch: deprecation: ERROR index_settings: - so-logs-elastic_agent.apm_server: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.apm_server-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.apm_server@package" - - "so-logs-elastic_agent.apm_server@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.auditbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.auditbeat-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.auditbeat@package" - - "so-logs-elastic_agent.auditbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.cloudbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.cloudbeat-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.cloudbeat@package" - - "so-logs-elastic_agent.cloudbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.endpoint_security: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.endpoint_security-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.endpoint_security@package" - - "so-logs-elastic_agent.endpoint_security@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.filebeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.filebeat-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.filebeat@package" - - "so-logs-elastic_agent.filebeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.fleet_server: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.fleet_server-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.fleet_server@package" - - "so-logs-elastic_agent.fleet_server@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.heartbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.heartbeat-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.heartbeat@package" - - "so-logs-elastic_agent.heartbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent@package" - - "so-logs-elastic_agent@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.metricbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.metricbeat-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.metricbeat@package" - - "so-logs-elastic_agent.metricbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.osquerybeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.osquerybeat-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.osquerybeat@package" - - "so-logs-elastic_agent.osquerybeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false - so-logs-elastic_agent.packetbeat: - index_sorting: False - index_template: - index_patterns: - - "logs-elastic_agent.packetbeat-*" - template: - settings: - index: - mapping: - total_fields: - limit: 5000 - sort: - field: "@timestamp" - order: desc - mappings: - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - composed_of: - - "so-logs-elastic_agent.packetbeat@package" - - "so-logs-elastic_agent.packetbeat@custom" - - ".fleet_globals-1" - - ".fleet_agent_id_verification-1" - priority: 500 - _meta: - package: - name: elastic_agent - managed_by: fleet - managed: true - data_stream: - hidden: false - allow_custom_routing: false so-aws: warm: 7 close: 30 diff --git a/salt/idh/defaults/defaults.yaml b/salt/idh/defaults/defaults.yaml index 673b18c55..e5b966c10 100644 --- a/salt/idh/defaults/defaults.yaml +++ b/salt/idh/defaults/defaults.yaml @@ -1,7 +1,6 @@ idh: opencanary: config: - device.node_id: {{ grains.host }} logger: class: PyLogger kwargs: diff --git a/salt/idh/opencanary_config.map.jinja b/salt/idh/opencanary_config.map.jinja index dbd2fbad5..c4533682d 100644 --- a/salt/idh/opencanary_config.map.jinja +++ b/salt/idh/opencanary_config.map.jinja @@ -6,4 +6,6 @@ {% do salt['defaults.merge'](OPENCANARYCONFIG, SERVICECONFIG, in_place=True) %} {% endfor %} -{% set OPENCANARYCONFIG = salt['pillar.get']('idh:opencanary:config', default=OPENCANARYCONFIG.idh.opencanary.config, merge=True) %} \ No newline at end of file +{% set OPENCANARYCONFIG = salt['pillar.get']('idh:opencanary:config', default=OPENCANARYCONFIG.idh.opencanary.config, merge=True) %} +{% do OPENCANARYCONFIG.idh.opencanary.config.update({'device.node_id': grains.host}) %} + diff --git a/salt/kibana/config.map.jinja b/salt/kibana/config.map.jinja index 32768a5eb..eee52025c 100644 --- a/salt/kibana/config.map.jinja +++ b/salt/kibana/config.map.jinja @@ -1,6 +1,9 @@ {% import_yaml 'kibana/defaults.yaml' as KIBANACONFIG with context %} {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} +{% do KIBANACONFIG.kibana.config.server.update({'publicBaseUrl': 'https://' ~ pillar.global.url_base ~ '/kibana'}) %} +{% do KIBANACONFIG.kibana.config.elasticsearch.update({'hosts': 'https://' ~ pillar.global.managerip ~ ':9200'}) %} + {% do KIBANACONFIG.kibana.config.elasticsearch.update({'username': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user'), 'password': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass')}) %} {% if salt['pillar.get']('kibana:secrets') %} diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index 637e80cf7..c713f27e2 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -5,10 +5,7 @@ kibana: name: kibana host: "0.0.0.0" basePath: /kibana - publicBaseUrl: https://{{salt['pillar.get']('global:url_base')}}/kibana elasticsearch: - hosts: - - https://{{salt['pillar.get']('global:managerip')}}:9200 ssl: verificationMode: none requestTimeout: 90000 @@ -70,7 +67,7 @@ kibana: - type: system/metrics enabled: false - name: Endpoints-Initial - id: endpoints-default + id: endpoints description: "Initial Endpoint Policy" namespace: default monitoring_enabled: ['logs'] From 1f3b1702132a53d506d68755ca7a91f88884f0f6 Mon Sep 17 00:00:00 2001 From: Mike Reeves Date: Fri, 9 Sep 2022 15:36:57 -0400 Subject: [PATCH 2/2] Fix yaml for idh,es,kib,esalert --- salt/kibana/config.map.jinja | 2 +- salt/kibana/defaults.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/kibana/config.map.jinja b/salt/kibana/config.map.jinja index eee52025c..3d285d40d 100644 --- a/salt/kibana/config.map.jinja +++ b/salt/kibana/config.map.jinja @@ -2,7 +2,7 @@ {% set HIGHLANDER = salt['pillar.get']('global:highlander', False) %} {% do KIBANACONFIG.kibana.config.server.update({'publicBaseUrl': 'https://' ~ pillar.global.url_base ~ '/kibana'}) %} -{% do KIBANACONFIG.kibana.config.elasticsearch.update({'hosts': 'https://' ~ pillar.global.managerip ~ ':9200'}) %} +{% do KIBANACONFIG.kibana.config.elasticsearch.update({'hosts': ['https://' ~ pillar.global.managerip ~ ':9200']}) %} {% do KIBANACONFIG.kibana.config.elasticsearch.update({'username': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:user'), 'password': salt['pillar.get']('elasticsearch:auth:users:so_kibana_user:pass')}) %} diff --git a/salt/kibana/defaults.yaml b/salt/kibana/defaults.yaml index c713f27e2..317cb6730 100644 --- a/salt/kibana/defaults.yaml +++ b/salt/kibana/defaults.yaml @@ -67,7 +67,7 @@ kibana: - type: system/metrics enabled: false - name: Endpoints-Initial - id: endpoints + id: endpoints-default description: "Initial Endpoint Policy" namespace: default monitoring_enabled: ['logs']