Linux-Manual

Logstash 知识

基础知识

logstash 5.5.0 安装

logstash 2.4.1 安装

Settings: Default pipeline workers: 1
Pipeline main started

{
       "message" => "hello world",
      "@version" => "1",
    "@timestamp" => "2017-03-14T06:56:44.690Z",
          "host" => "youmeeklocalhost"
}

Settings: Default pipeline workers: 1
Pipeline main started

配置文件中的 Filter 讲解

input {
	stdin {
	
	}
}

filter {
	grok {
		match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
	}
}

output {
	elasticsearch { 
		hosts => ["192.168.1.127:9200"]
		index => "filter-grok-test"
	}
}

配置文件中的 multiline 多行内容收集插件讲解

input {
	file {
		path => ["/usr/program/tomcat8/logs/logbackOutFile.log.*.log"]
		type => "tomcat-log"
		start_position => "beginning"
		codec => multiline {
		    pattern => "^\["
		    negate => true
		    what => "previous"
		}
	}
}

output {
	if [type] == "tomcat-log" {
		elasticsearch { 
			hosts => ["192.168.1.127:9200"]
			index => "tomcat-log-%{+YYYY.MM.dd}"
		}
	}
}

案例

测试模式

自己写正则表达式，匹配后输出到控制台先看下：

input {
	stdin {
		codec => multiline {
			pattern => "^\["
			negate => true
			what => "previous"
		}
	}
}

output {
	stdout { 
		codec => "rubydebug"
	}
}

读取文件，输出到控制台先看下：

input {
	file {
		path => ["/var/log/nginx/access.log"]
		type => "nginx-access-log"
		start_position => "beginning"
	}
}

output {
	stdout { 
		codec => "rubydebug"
	}
}

Nginx 日志收集

input {
	file {
		path => ["/var/log/nginx/access.log"]
		type => "nginx-access-log"
		start_position => "beginning"
	}
	
	file {
		path => ["/var/log/nginx/error.log"]
		type => "nginx-error-log"
		start_position => "beginning"
	}
}

output {
	if [type] == "nginx-access-log" {
		elasticsearch { 
			hosts => ["192.168.1.127:9200"]
			index => "nginx-access-log"
		}
	}
	
	if [type] == "nginx-error-log" {
		elasticsearch { 
			hosts => ["192.168.1.127:9200"]
			index => "nginx-error-log"
		}
	}
}

进一步优化：把 nginx 的日志输出格式改为 json

user root;
worker_processes  1;

events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    sendfile        on;
    keepalive_timeout  65;
    
    log_format json '{"@timestamp":"$time_iso8601",'
                     '"host":"$server_addr",'
                     '"clientip":"$remote_addr",'
                     '"size":$body_bytes_sent,'
                     '"responsetime":$request_time,'
                     '"upstreamtime":"$upstream_response_time",'
                     '"upstreamhost":"$upstream_addr",'
                     '"http_host":"$host",'
                     '"url":"$uri",'
                     '"xff":"$http_x_forwarded_for",'
                     '"referer":"$http_referer",'
                     '"agent":"$http_user_agent",'
                     '"status":"$status"}';
	#全局日志
    access_log /var/log/nginx/access.log;
    error_log /var/log/nginx/error.log;

    server {
        listen       80;
        server_name  localhost;
		
		# 针对服务的日志输出
		access_log /var/log/nginx/access-json.log json;

        location / {
            root   html;
            index  index.html index.htm;
        }

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

input {
	file {
		path => ["/var/log/nginx/access-json.log"]
		codec => json
		type => "nginx-access-json-log"
		start_position => "beginning"
	}

}

output {
	if [type] == "nginx-access-json-log" {
		elasticsearch { 
			hosts => ["192.168.1.127:9200"]
			index => "nginx-access-json-log"
		}
	}
}

Tomcat 日志收集

input {
	file {
		path => ["/usr/program/tomcat8/logs/logbackOutFile.log.*.log"]
		type => "tomcat-log"
		start_position => "beginning"
		codec => multiline {
		    pattern => "^\["
		    negate => true
		    what => "previous"
		}
	}
}

output {
	if [type] == "tomcat-log" {
		elasticsearch { 
			hosts => ["192.168.1.127:9200"]
			index => "tomcat-log-%{+YYYY.MM.dd}"
		}
	}
}

MySQL 慢 SQL 日志收集

"(?m)^#\s+User@Host:\s+%{USER:user}\[[^\]]+\]\s+@\s+%{USER:clienthost}\s+\[(?:%{IP:clientip})?\]\s+Id:\s+%{NUMBER:id:int}\n#\s+Schema:\s+%{USER:schema}\s+Last_errno:\s+%{NUMBER:lasterrorno:int}\s+Killed:\s+%{NUMBER:killedno:int}\n#\s+Query_time:\s+%{NUMBER:query_time:float}\s+Lock_time:\s+%{NUMBER:lock_time:float}\s+Rows_sent:\s+%{NUMBER:rows_sent:int}\s+Rows_examined:\s+%{NUMBER:rows_examined:int}\s+Rows_affected:\s+%{NUMBER:rows_affected:int}\n#\s+Bytes_sent:\s+%{NUMBER:bytes_sent:int}\n\s*(?:use\s+%{USER:usedatabase};\s*\n)?SET\s+timestamp=%{NUMBER:timestamp};\n\s*(?<query>(?<action>\w+)\b.*)\s*(?:\n#\s+Time)?.*$"

Logstash 不直接写到 ES 先写到 Redis 再写到 ES

一台 Logstash 把数据写到 Redis

input {
	stdin {
		
	}
}

output {
	redis {
		host => "192.168.1.125"
		port => "6379"
		db => "2"
		data_type => "list"
		key => "gitnavi-logstash-info"
	}
}

1) "{\"message\":\"hello\",\"@version\":\"1\",\"@timestamp\":\"2017-03-15T15:23:35.064Z\",\"host\":\"youmeekhost\"}"
2) "{\"message\":\"world\",\"@version\":\"1\",\"@timestamp\":\"2017-03-15T15:23:37.245Z\",\"host\":\"youmeekhost\"}"

一台 Logstash 把数据从 Redis 读取出来写到 ES

input {
	redis {
		type => "redis-log"
		host => "192.168.1.125"
		port => "6379"
		db => "2"
		data_type => "list"
		key => "gitnavi-logstash-info"
	}
}

output {
	if [type] == "redis-log" {
		elasticsearch {
	        hosts => ["192.168.1.127:9200"]
	        index => "redis-log"
	    }
    }
}

Logstash 不直接写到 ES 先写到 MQ 再写到 ES

一台 Logstash 把数据写到 rabbitMQ

input {
	file {
		path => "/usr/local/tomcat/logs/tomcat_json.log"
		codec => "json"
		type => "tomcat"
	}
}

output {
	rabbitmq { 
		host => "RabbitMQ_server"
		port => "5672"
		vhost => "elk"
		exchange => "elk_exchange"
		exchange_type => "direct"
		key => "elk_key"
		user => "liang"
		password => "liang123"
	}
	stdout { 
		codec => rubydebug 
	}
}

一台 Logstash 把数据从 rabbitMQ 读取出来写到 ES （还未测试）

input {
	rabbitmq {
		host => "127.0.0.1"
		subscription_retry_interval_seconds => "5"
		vhost => "elk"
		exchange => "elk_exchange"
		queue => "elk_queue"
		durable => "true"
		key => "elk_key"
		user => "liang"
		password => "liang123"
	}
}

output {

	if [type] == "nginx" {
		elasticsearch {
			hosts => "192.168.1.127:9200"
			user => "logstash"
			password => "123456"
			index => "nginx-%{+YYYY.MM.dd}"
		}
	}
	
	if [type] == "tomcat" {
		elasticsearch {
			hosts => "192.168.1.127:9200"
			user => "logstash"
			password => "123456"
			index => "tomcat-%{+YYYY.MM.dd}"
		}
	}

	stdout { 
		codec => rubydebug 
	}
}

资料