elk收集nginx日志

rpm -ivh logstash-6.0.0.rpm

input {
  file {
    #nginx日志存放的位置
    path => "/usr/local/nginx/logs/*.log"
    #表示从头开始读取
    start_position => "beginning"

    #记录读取文件内容和位置
    #sincedb_path => "/tmp/logstash_data/"
  }
}

filter {
    grok {
      match => {
        "message" => '%{IPORHOST:remote_ip} - %{DATA:user_name} \[%{HTTPDATE:time}\] "%{WORD:request_action} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:response} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:agent}"'
      }
    }
    #删除message字段
    mutate {
      remove_field => "message"
    }
    #常用插件，根据ip地址提供对应的地理信息
    geoip {
      source => ip
    }
    #日期格式插件
    date {
      match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
}

output {
  elasticsearch {
     hosts => ["192.168.1.8:9200"]
     index => "logstash-nginx-access-log"
  }
  #输出到控制台
  stdout { codec => rubydebug }
}

#解压
tar -zxvf elasticsearch-6.0.0.tar.gz
#移动到/usr/local目录下
mv elasticsearch-6.0.0 /usr/local

# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name: my-tao
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
node.name: node8
#
# Add custom attributes to the node:
#
#node.attr.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
path.data: /usr/local/elasticsearch-6.0.0/data
#
# Path to log files:
#
path.logs: /usr/local/elasticsearch-6.0.0/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
#bootstrap.memory_lock: true
#
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
network.host: 192.168.1.8
#
# Set a custom port for HTTP:
#
http.port: 9200
#
# For more information, consult the network module documentation.

#<domain>      <type>  <item>         <value>
*                soft    nofile          65536
*                hard    nofile          131072
tao              soft    nproc           4096
tao              hard    nproc           4096

#更改最大线程数
sudo sysctl -w vm.max_map_count=262144
#使修改生效
sudo sysctl -p

#注：在很多博客中我都看到以上操作配置会永久生效，但是实际我的操作中，重启机器后，配置会还原，所以我还是建议写入配置文件
# echo "vm.max_map_count=262144" >> /etc/sysctl.conf

# Kibana is served by a back end server. This setting specifies the port to use.
server.port: 5601

# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
# The default is 'localhost', which usually means remote machines will not be able to connect.
# To allow connections from remote users, set this parameter to a non-loopback address.
server.host: "192.168.1.8"

# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
# the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
# to Kibana. This setting cannot end in a slash.
#server.basePath: ""

# The maximum payload size in bytes for incoming server requests.
#server.maxPayloadBytes: 1048576

# The Kibana server's name.  This is used for display purposes.
#server.name: "your-hostname"

# The URL of the Elasticsearch instance to use for all your queries.
elasticsearch.url: "http://192.168.1.8:9200"

# When this setting's value is true Kibana uses the hostname specified in the server.host
# setting. When the value of this setting is false, Kibana uses the hostname of the host
# that connects to this Kibana instance.
#elasticsearch.preserveHost: true

# Kibana uses an index in Elasticsearch to store saved searches, visualizations and
# dashboards. Kibana creates a new index if the index doesn't already exist.
#kibana.index: ".kibana"