EFK收集messages线程

Server 192.168.132.7 :3G 2cpu(kibana filebeat logstash nginx)
Server 192.168.132.17 :1G (httpd-tools elasticsearch)
Server 192.168.132.24 :1G

全局环境jdk

rpm -ivh jdk-8u131-linux-x64_.rpm

三台虚拟机都做(zookeeper kafka)

1. systemctl stop firewalld
2. setenforce 0
3. vim /etc/hosts
   
4. hostnamectl set-hostname kafka01-03
5. ping kafka01-03

zookeeper

7. tar zxf zookeeper-3.4.14.tar.gz
8. mv zookeeper-3.4.14. /usr/local/zookeeper
9. cd /usr/local/zookeeper/conf
10. mv zoo_sample.cfg zoocfg
11. vim zoo.cfg
    
12. mkdir -p /tmp/zookeeper
13. echo ‘1’ >/tmp/zookeeper/myid (1.2.3)
14. ./bin/zkServer.sh start (三台都做完再开始)
15. 
16. ./bin/zkServer.sh status .
17. 
18. (follower leader follower)
19. #######################

kafka

20. tar zxf kafka_2.11-2.2.0.tgz
21. mv kafka_2.11-2.2.0 /usr/local/kafka
22. cd /usr/local/kafka/config/
23. vim server.properties
24. 
25. 启动 ./bin/kafka-server-start.sh -daemon config/server.properties
26. 验证
27. ./bin/kafka-topics.sh --create --zookeeper 192.168.132.7:2181 --replication-factor 2 --partitions 3 --topic msg
28. 
29. ./bin/kafka-topics.sh --list --zookeeper 192.168.132.7:2181
30. 
31. 生产 :./bin/kafka-console-producer.sh --broker-list 192.168.132.7:9092 --topic msg
32. 
33. 消费(192.168.132.17/192.168.132.24)
34. ./bin/kafka-console-consumer.sh --bootstrap-server 192.168.132.7:9092 --topic msg --from-beginning
35. 

FILEBEAT(192.168.132.7)

35. 配置yum源法:vim /etc/yum.repo.d/filebeat.repo
36. [filebeat-6.x]
    name=Elasticsearch repository for 6.x packages
    baseurl= https://artifacts.elastic.co/packages/6.x/yum
    gpgcheck=1
    gpgkey= https://artifacts.elastic.co/GPG-KEY-elasticsearch
    enabled=1
    autorefresh=1
    type=rpm-md
37. yum -y install filebeat
38. vim /etc/filebeat/filebeat.yml
39. 
40. systemctl enable filebeat
41. systemctl start filebeat
42. tailf /var/log/filebeat/filebeat
43. 上传安装包 tar zxf filebeat-6.8.8-linux-x86_64.tar.gz
44. https://www.elastic.co/cn/downloads/past-releases#filebeat
45. 
46. vim tar zxf filebeat-6.8.8-linux-x86_64/filebeat.yml
47. 
48. ./filebeat -c filebeat.yml -e

ELASTICSEARCH(192.168.132.17)

49. rpm -ivh elasticsearch-6.6.2.rpm
50. vim /etc/elasticsearch/elasticsearch.yml
    
51. systemctl start elasticseach
52. tailf /var/log/elasticsearch/ylm66.log
53. ss -ntlp | grep java(yum -y install net tools :netstat)

LOGSTASH(192.168.132.7)

55. rpm -ivh logstash-6.6.0.rpm
    vim /etc/logstash/confid/messages.conf
    
    input {
    kafka {
    bootstrap_servers => [“192.168.132.24:9092,192.168.132.7:9092,192.168.132.17:9092”]
    group_id => “logstash”
    topics => “msg”
    consumer_threads => 5
    }

}
output {
elasticsearch {
hosts => “192.168.132.24:9200”
index => “messages-%{+YYYY.MM.dd}”
}
}
56. chmod -R 777 /var/log
57. systemctl restart logstash
58. tailf /var/log/logstash(tab)

KIBANA(192.168.132.7)

1． rpm -ivh kibana-6.6.2-x86_64.rpm
2． vim /etc/kibana/kibana.yml

3． systemctl start kibana
4． ss -ntlp | grep 5601