今天我們來品一品這個ELK哈，首先要明確概念，ELK是什么呢，還是像往常一樣，百度一哈👾
關于ELK
Logstash
Logstash概述
LogStash的主要組件
LogStash主機分類
ElasticSearch
ElasticSearch概述
Elasticsearch的基礎核心概念
主要功能：
ELK的日志處理流程
部署ELK日志分析系統
搭建環境
node1 192.168.152.130 主要軟體：Elasticsearch kibana
node2 192.168.152.129 主要軟體： Elasticsearch
web 192.168.152.12 主要軟體： logstash apache搭建程序
配置elasticsearch環境
首先三臺都關閉防火墻，修改主機名字，這里以node1主機為例：
[root@server ~]# systemctl stop firewalld.service
[root@server ~]# systemctl disable firewalld.service
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@server ~]# setenforce 0
[root@server ~]# hostname node1
[root@server ~]# su
三臺主機都安裝Java環境：
[root@node1 ~]# java -version
openjdk version "1.8.0_181"
#從本機里匯入安裝包
[root@node1 ~]# ls
anaconda-ks.cfg initial-setup-ks.cfg jdk-8u91-linux-x64.tar.gz 公共 模板 視頻 圖片 檔案 下載 音樂 桌面
[root@node1 ~]# tar xf jdk-8u91-linux-x64.tar.gz -C /usr/local/ #解壓
[root@node1 ~]# cd /usr/local/
[root@node1 local]# ls
bin etc games include jdk1.8.0_91 lib lib64 libexec sbin share src
[root@node1 local]# mv jdk1.8.0_91/ jdk
[root@node1 local]# vim /etc/profile
#在里面添加進以下內容
export JAVA_HOME=/usr/local/jdk
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOME}/bin:$PATH
[root@node1 local]# source /etc/profile
[root@node1 local]# java -version
java version "1.8.0_91"
Java(TM) SE Runtime Environment (build 1.8.0_91-b14)
Java HotSpot(TM) 64-Bit Server VM (build 25.91-b14, mixed mode)
設定本地主機映射檔案，node1和node2節點操作：
[root@node1 ~]# vim /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.152.130 node1
192.168.152.129 node2配置elasticsearch軟體（node1.node2做相同操作）
[root@node1 local]# cd /opt
#上傳安裝包
[root@node1 opt]# ls
elasticsearch-5.5.0.rpm elasticsearch-head.tar.gz httpd-2.4.6-95.el7.centos.x86_64.rpm rh
[root@node1 opt]#
[root@node1 opt]# rpm -ivh elasticsearch-5.5.0.rpm
#加載系統服務
[root@node1 opt]# systemctl daemon-reload
[root@node1 opt]# systemctl enable elasticsearch.service
Created symlink from /etc/systemd/system/multi-user.target.wants/elasticsearch.service to /usr/lib/systemd/system/elasticsearch.service.
[root@node1 opt]#
#修改主組態檔
[root@node1 opt]# cd /etc/elasticsearch/
[root@node1 elasticsearch]# cp -a elasticsearch.yml elasticsearch.yml.bak
[root@node1 elasticsearch]# vim elasticsearch.yml
17/ cluster.name: my-elk-cluster
#集群名字
23/ node.name: node1
#節點名寧字
33/ path.data: /data/elk_data
#資料存放路徑
37/ path. logs: /var/log/elasticsearch/
#日志存放路徑
43/ bootstrap.memory_lock: false
#鎖定物理記憶體地址，防止es記憶體被交換出去，也就是避免es使用swap交換磁區，頻繁的交換，會導致Ios變高(性能測驗:每秒的讀寫次數)，
55/ network.host: 0.0.0.0
#提供服務系結的IP地址，0.0.0.0代表所有地址
59/ http.port: 9200
#偵聽埠為9200
68/ discoveryp zen.ping.unicast.hosts:["node1", "node2"]
#集群發現通過單播實作單播
[root@node1 elasticsearch]# grep -v "^#" /etc/elasticsearch/elasticsearch.yml
cluster.name: my-elk-cluster
node.name: node1
path.data: /data/elk_data
path.logs: /var/log/elasticsearch/
bootstrap.memory_lock: false
network.host: 0.0.0.0
http.port: 9200
discovery.zen.ping.unicast.hosts: ["node1", "node2"]
[root@node1 elasticsearch]#
#創建資料存放路徑并授權
[root@node1 elasticsearch]# mkdir -p /data/elk_data
[root@node1 elasticsearch]# chown elasticsearch:elasticsearch /data/elk_data/
#啟動elasticsearch
[root@node1 elasticsearch]# systemctl start elasticsearch.service
[root@node1 elasticsearch]# netstat -antp | grep 9200
tcp6 0 0 :::9200 :::* LISTEN 14330/java查看節點資訊
查看集群健康狀態資訊
安裝elasticsearch-head插件（node1.node2做相同操作）
#安裝node組件
[root@node1 ~]# cd /opt
[root@node1 opt]# yum install gcc gcc-c++ make -y
[root@node1 opt]# ls
elasticsearch-5.5.0.rpm elasticsearch-head.tar.gz httpd-2.4.6-95.el7.centos.x86_64.rpm node-v8.2.1.tar.gz rh
[root@node1 opt]# tar xzvf node-v8.2.1.tar.gz
[root@node1 opt]# cd node-v8.2.1/
[root@node1 node-v8.2.1]# ./configure
[root@node1 node-v8.2.1]# make -j3
[root@node1 node-v8.2.1]# make install
#安裝phantomjs前端框架
[root@node1 node-v8.2.1]# cd ..
[root@node1 opt]# ls
elasticsearch-5.5.0.rpm httpd-2.4.6-95.el7.centos.x86_64.rpm node-v8.2.1.tar.gz rh
elasticsearch-head.tar.gz node-v8.2.1 phantomjs-2.1.1-linux-x86_64.tar.bz2
[root@node1 opt]#
[root@node1 opt]# tar xjvf phantomjs-2.1.1-linux-x86_64.tar.bz2 -C /usr/local/src/
[root@node1 opt]# cd /usr/local/src/
[root@node1 src]# ls
phantomjs-2.1.1-linux-x86_64
[root@node1 phantomjs-2.1.1-linux-x86_64]# ls
bin ChangeLog examples LICENSE.BSD README.md third-party.txt
[root@node1 phantomjs-2.1.1-linux-x86_64]# cd bin
[root@node1 bin]# ls
phantomjs
[root@node1 bin]# cp phantomjs /usr/local/bin/
[root@node1 bin]#
#安裝elasticsearch-head資料可視化工具
[root@node1 bin]# cd /opt
[root@node1 opt]# tar xzvf elasticsearch-head.tar.gz -C /usr/local/src
[root@node1 opt]# cd /usr/local/src/
[root@node1 src]# ls
elasticsearch-head phantomjs-2.1.1-linux-x86_64
[root@node1 src]# cd elasticsearch-head/
[root@node1 elasticsearch-head]# npm install
#修改主組態檔
[root@node1 elasticsearch-head]# cd
[root@node1 ~]# vim /etc/elasticsearch/elasticsearch.yml
#在最后補充
http.cors.enabled: true #開啟跨域訪問支持，默認為false
http.cors.allow-origin: "*" #跨域訪問允許的域名地址
#啟動elasticsearch-head
[root@node1 ~]# cd /usr/local/src/elasticsearch-head/
[root@node1 elasticsearch-head]# npm run start &
[1] 60746
[root@node1 elasticsearch-head]#
> elasticsearch-head@0.0.0 start /usr/local/src/elasticsearch-head
> grunt server
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100 
[root@node1 elasticsearch-head]# npm run start &
[1] 64014
[root@node1 elasticsearch-head]#
> elasticsearch-head@0.0.0 start /usr/local/src/elasticsearch-head
> grunt server
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100
[root@node1 ~]# netstat -antp | grep 9100
[root@node2 elasticsearch-head]# npm run start &
[1] 119353
[root@node2 elasticsearch-head]#
> elasticsearch-head@0.0.0 start /usr/local/src/elasticsearch-head
> grunt server
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://localhost:9100
[root@node2 ~]# netstat -antp | grep 9100 
[root@node1 elasticsearch-head]# cd
[root@node1 ~]#
[root@node1 ~]#
[root@node1 ~]# curl -XPUT 'localhost:9200/index-demo/test/1?pretty&pretty' -H 'content-Type: application/json' -d '{"user":"aaa","mesg":"hello world"}'
{
"_index" : "index-demo",
"_type" : "test",
"_id" : "1",
"_version" : 1,
"result" : "created",
"_shards" : {
"total" : 2,
"successful" : 2,
"failed" : 0
},
"created" : true
} 
apache服務器部署logstash相關
安裝服務并開啟服務
[root@apache ~]# yum -y install httpd
[root@apache ~]# systemctl start httpd
[root@apache ~]# netstat -ntap |grep httpd
tcp6 0 0 :::80 :::* LISTEN 19669/httpd
[root@apache ~]#
#安裝logstash服務并啟動
[root@apache ~]# cd /opt
[root@apache opt]# ls
logstash-5.5.1.rpm rh
[root@apache opt]# rpm -ivh logstash-5.5.1.rpm
[root@apache opt]# ln -s /usr/share/logstash/bin/logstash /usr/local/bin/ #創建一個軟鏈接
[root@apache opt]# systemctl start logstash.service
[root@apache opt]# systemctl enable logstash.service
Created symlink from /etc/systemd/system/multi-user.target.wants/logstash.service to /etc/systemd/system/logstash.service.與elasticsearch（node）做對接測驗：
Logstash這個命令測驗，欄位描述解釋：
-f 通過這個選項可以指定logstash的組態檔，根據組態檔配置logstash
-e 后面跟著字串 該字串可以被當做logstash的配置（如果是” ”,則默認使用stdin做為輸入、stdout作為輸出）
-t 測驗組態檔是否正確，然后退出
#輸入采用標準輸入 輸出采用標準輸出，進行測驗
[root@apache opt]# logstash -e 'input { stdin{} } output { stdout{} }'
使用rubydebug顯示詳細輸出，codec為一種編解碼器
[root@apache opt]# logstash -e 'input { stdin{} } output { stdout{ codec=>rubydebug} }'
使用logstash將資訊寫入elasticsearch中，輸入輸出對接
[root@apache opt]# logstash -e 'input { stdin{} } output { elasticsearch { hosts=>["192.168.152.130:9200"]} }'
查看索引資訊
做對接配置
Logstash組態檔主要由三部分組成：input、output、filter（根據需要來處理）
[root@apache opt]# chmod o+r /var/log/messages #給其他用戶讀的權限
[root@apache opt]# ll /var/log/messages
-rw----r--. 1 root root 288219 8月 14 10:52 /var/log/messages
[root@apache opt]# vim /etc/logstash/conf.d/system.conf
input {
file{
path => "/var/log/messages" #收集資料的路徑
type => "system" #型別
start_position => "beginning" #從開頭收集資料
}
}
output {
elasticsearch {
hosts => ["192.168.152.130:9200"] #輸出到
index => "system-%{+YYYY.MM.dd}" #索引
}
}
[root@apache opt]# systemctl restart logstash.service 
node1主機安裝kibana
配置程序
#安裝kibana：
[root@node1 ~]# cd /usr/local/src/
[root@node1 src]# ls
elasticsearch-head kibana-5.5.1-x86_64.rpm phantomjs-2.1.1-linux-x86_64
[root@node1 src]# rpm -ivh kibana-5.5.1-x86_64.rpm
#修改組態檔：
[root@node1 src]# cd /etc/kibana/
[root@node1 kibana]# cp kibana.yml kibana.yml.bak
[root@node1 kibana]# vim kibana.yml
2# server.port: 5601 #kibana打開的埠
7# server.host: "0.0.0.0" #kibana偵聽的地址
21# elasticsearch.url: "http: //192.168.152.130:9200" #利和elasticsearch建立聯系
30# kibana .index : ".kibana" #在elasticsearch中添加.kibana索引
[root@node1 kibana]# grep -v "^#" kibana.yml
server.port: 5601
server.host: "0.0.0.0"
elasticsearch.url: "http://192.168.152.130:9200"
kibana.index: ".kibana"
#啟動服務
[root@node1 kibana]# systemctl start kibana.service
[root@node1 kibana]# systemctl enable kibana.service 宿主機登陸測驗，查看kibana
對接apache的日志
配置程序
[root@apache opt]# cd /etc/logstash/conf.d/
[root@apache conf.d]# ls
system.conf
[root@apache conf.d]# vim apache_log.conf
input {
file{
path => "/etc/httpd/logs/access_log"
type => "access"
start_position => "beginning"
}
file{
path => "/etc/httpd/logs/error_log"
type => "error"
start_position => "beginning"
}
}
output {
if [type] == "access" {
elasticsearch {
hosts => ["192.168.152.130:9200"]
index => "apache_access-%{+YYYY.MM.dd}"
}
}
if [type] == "error" {
elasticsearch {
hosts => ["192.168.152.130:9200"]
index => "apache_error-%{+YYYY.MM.dd}"
}
}
}
[root@apache conf.d]# /usr/share/logstash/bin/logstash -f apache_log.conf
# 指定組態檔做實驗查看索引
創建索引名稱 
轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/294151.html
標籤：其他
上一篇：Pandas必會的方法匯總，用Python做資料分析更加如魚得水！
下一篇：dubbo學習筆記