Installing HDF Services on an Existing HDP Cluster
Also available as:
PDF

Configuring SAM log search and event sampling

You must perform several manual steps to enable log search and event sampling for SAM.

  1. From the Log Search UI, click the Configuration icon on the top right of your screen. Select Configuration Editor.
  2. From the left hand All Configuration pane, select Storm.
  3. In the Edit Configuration field, replace the existing json text with the following configuration:
    {
       "input":[
          {
             "type":"storm_drpc",
             "rowtype":"service",
             "path":"/var/log/storm/drpc.log"
          },
          {
             "type":"storm_logviewer",
             "rowtype":"service",
             "path":"/var/log/storm/logviewer.log"
          },
          {
             "type":"storm_nimbus",
             "rowtype":"service",
             "path":"/var/log/storm/nimbus.log"
          },
          {
             "type":"storm_supervisor",
             "rowtype":"service",
             "path":"/var/log/storm/supervisor.log"
          },
          {
             "type":"storm_ui",
             "rowtype":"service",
             "path":"/var/log/storm/ui.log"
          },
          {
             "type":"storm_worker",
             "rowtype":"service",
             "path":"/var/log/storm/workers-artifacts/*/*/worker.log",
             "cache_enabled":true,
             "cache_key_field":"log_message",
             "cache_last_dedup_enabled":true,
             "cache_size":100,
             "cache_dedup_interval":1000,
             "init_default_fields":true
          },
          {
             "type":"storm_worker_event",
             "rowtype":"service",
             "path":"/var/log/storm/workers-artifacts/*/*/events.log",
             "add_fields":{
                "level":"INFO"
             },
             "cache_enabled":true,
             "cache_key_field":"log_message",
             "cache_last_dedup_enabled":true,
             "cache_size":100,
             "cache_dedup_interval":1000,
             "init_default_fields":true
          }
       ],
       "filter":[
          {
             "filter":"grok",
             "conditions":{
                "fields":{
                   "type":[
                      "storm_supervisor",
                      "storm_logviewer",
                      "storm_drpc",
                      "storm_worker",
                      "storm_ui",
                      "storm_nimbus"
                   ]
                }
             },
             "skipOnError":false,
             "deepExtract":false,
             "sort_order":1,
             "post_map_values":{
                "logtime":[
                   {
                      "map_date":{
                         "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
                      }
                   }
                ]
             },
             "log4j_format":"",
             "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
             "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}\\s%{GREEDYDATA:thread_name}\\s\\[%{LOGLEVEL:level}\\]\\s%{GREEDYDATA:log_message}"
          },
          {
             "filter":"grok",
             "conditions":{
                "fields":{
                   "type":[
                      "storm_worker"
                   ]
                }
             },
             "skipOnError":false,
             "deepExtract":false,
             "sort_order":2,
             "source_field":"thread_name",
             "remove_source_field":false,
             "message_pattern":"(Thread\\-[0-9]+\\-[0-9]+\\-%{DATA:sdi_streamline_component_name}\\-executor%{DATA}|%{DATA:thread_name})"
          },
          {
             "filter":"grok",
             "conditions":{
                "fields":{
                   "type":[
                      "storm_worker"
                   ]
                }
             },
             "skipOnError":false,
             "deepExtract":false,
             "sort_order":3,
             "source_field":"path",
             "remove_source_field":false,
             "message_pattern":"/var/log/storm/workers-artifacts/%{DATA:sdi_storm_topology_id}/%{DATA:sdi_storm_worker_port}/worker\\.log"
          },
          {
             "filter":"grok",
             "conditions":{
                "fields":{
                   "type":[
                      "storm_worker"
                   ]
                }
             },
             "skipOnError":false,
             "deepExtract":false,
             "sort_order":4,
             "source_field":"sdi_storm_topology_id",
             "remove_source_field":false,
             "message_pattern":"(streamline\\-%{DATA:sdi_streamline_topology_id}\\-%{DATA:sdi_streamline_topology_name}\\-[0-9]+\\-[0-9]+)|(%{DATA:sdi_storm_topology_id})"
          },
          {
             "filter":"grok",
             "conditions":{
                "fields":{
                   "type":[
                      "storm_worker_event"
                   ]
                }
             },
             "skipOnError":false,
             "deepExtract":false,
             "sort_order":5,
             "post_map_values":{
                "logtime":[
                   {
                      "map_date":{
                         "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
                      }
                   }
                ]
             },
             "log4j_format":"",
             "message_pattern":"^%{TIMESTAMP_ISO8601:logtime}(!_DELIM_!<STREAMLINE_EVENT>!_DELIM_!%{DATA:sdi_streamline_component_name}!_DELIM_!%{DATA:sdi_streamline_event_id}!_DELIM_!%{DATA:sdi_streamline_root_ids}!_DELIM_!%{DATA:sdi_streamline_parent_ids}!_DELIM_!%{DATA:sdi_streamline_event_fields_and_values}!_DELIM_!%{DATA:sdi_streamline_event_headers}!_DELIM_!%{DATA:sdi_streamline_event_aux_fields_and_values})|(%{GREEDYDATA})"
          },
          {
             "filter":"grok",
             "conditions":{
                "fields":{
                   "type":[
                      "storm_worker_event"
                   ]
                }
             },
             "skipOnError":false,
             "deepExtract":false,
             "sort_order":6,
             "source_field":"path",
             "remove_source_field":false,
             "message_pattern":"/var/log/storm/workers-artifacts/%{DATA:sdi_storm_topology_id}/%{DATA:sdi_storm_worker_port}/events\\.log"
          },
          {
             "filter":"grok",
             "conditions":{
                "fields":{
                   "type":[
                      "storm_worker_event"
                   ]
                }
             },
             "skipOnError":false,
             "deepExtract":false,
             "sort_order":7,
             "source_field":"sdi_storm_topology_id",
             "remove_source_field":false,
             "message_pattern":"(streamline\\-%{DATA:sdi_streamline_topology_id}\\-%{DATA:sdi_streamline_topology_name}\\-[0-9]+\\-[0-9]+)|(%{DATA:sdi_storm_topology_id})"
          }
       ]
    }
    
  4. Verify that storm log directory is correct. The above json content is /var/log/storm/. You should replace it with the actual log directory path if your cluster uses different log directory.
  5. Click Save.