igozhang

——

    ELK_config,es,logstash,filebeat,kibana

    elasticsearch6.2.2
    
    # cat /opt/elasticsearch/config/elasticsearch.yml
    cluster.name: igo-log-es
    node.name: node4
    node.attr.rack: log-route-lb1
    path.data: /data/elasticsearch/data
    path.logs: /data/elasticsearch/logs
    network.host: 192.168.240.222
    http.port: 29200
    transport.tcp.port: 29300
    discovery.zen.ping.unicast.hosts: ["192.168.240.222:29300", "192.168.240.223:29300", "192.168.240.224:29300"]
    discovery.zen.minimum_master_nodes: 2
    
    tips
    推迟5分钟分片分配,支持运行时进行修改
    PUT /_all/_settings 
    {
      "settings": {
        "index.unassigned.node_left.delayed_timeout": "5m" 
      }
    }
    
    filebeat6.2
    
    # cat /opt/filebeat/filebeat.yml
    filebeat.inputs:
    - type: log
      enabled: true
      paths:
        - /data/volume/igozhang.com/game_flow_all.log
      fields:
        log_topics: area1_game_flow_all
      json.keys_under_root: true
      json.overwrite_keys: true
      json.message_key: message
    - type: log
      enabled: true
      paths:
        - /data/volume/igozhang.com/game_flow_all.log
      fields:
        log_topics: area2_game_flow_all
      json.keys_under_root: true
      json.overwrite_keys: true
      json.message_key: message
    - type: log
      enabled: true
      paths:
        - /data/volume/igozhang.com/game_flow_all.log
      fields:
        log_topics: area3_game_flow_all
      json.keys_under_root: true
      json.overwrite_keys: true
      json.message_key: message
    - type: log
      enabled: true
      paths:
        - /data/volume/igozhang.com/game_flow_all.log
      fields:
        log_topics: area4_game_flow_all
      json.keys_under_root: true
      json.overwrite_keys: true
      json.message_key: message
    processors:
    #  - add_host_metadata: ~
    #  - add_cloud_metadata: ~
      - drop_fields:
         fields: ["beat","input_type","source"]
    output.kafka:
      enabled: true
      max_message_bytes: 100000000
      hosts: ["192.168.240.171:29092","192.168.240.172:29092","192.168.240.173:29092"]
      topic: '%{[fields][log_topics]}'
    
    kibana6.2.2
    
    # cat /opt/kibana/config/kibana.yml
    i18n.defaultLocale: "en"
    server.host: "192.168.240.224"
    server.name: "log-kibana"
    elasticsearch.url: "http://192.168.240.224:29200"
    

    MP3