logback + ELK

logback + ELK

日志数据的流动

logback ==> logstash ==> elasticsearch ---可视化---> kibana

logback

spring boot 默认使用logback日志，logstash-logback-encoder这个包可以直接把日志发送到logstash。logback-spring.xml 配置如下

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <include resource="org/springframework/boot/logging/logback/base.xml" />
    <appender name="stash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
        <destination>192.168.77.205:9516</destination>
        <!-- encoder is required -->
        <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
    </appender>
    <springProfile name="dev,test">
        <logger name="com.example.demo" level="INFO" additivity="false">
            <appender-ref ref="CONSOLE" />
            <appender-ref ref="stash" />
        </logger>
        <logger name="org.springframework" level="DEBUG" additivity="false">
            <appender-ref ref="CONSOLE" />
        </logger>
    </springProfile>
    <springProfile name="prod">
        <logger name="com.example.demo" level="INFO" additivity="false">
            <appender-ref ref="CONSOLE" />
            <appender-ref ref="stash" />
        </logger>
    </springProfile>
</configuration>

maven pom.xml 如下:

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.example</groupId>
    <artifactId>logdemo</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>
    <name>logdemo</name>
    <description>demo log project</description>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>1.5.4.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <java.version>1.8</java.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-thymeleaf</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>net.logstash.logback</groupId>
            <artifactId>logstash-logback-encoder</artifactId>
            <version>4.11</version>
        </dependency>
        <dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.2.4</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

logger

在需要打印日志的地方，logger打印

import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Controller;
import org.springframework.ui.ModelMap;
import org.springframework.web.bind.annotation.RequestMapping;

@Controller
public class IndexController {

    Logger logger = LoggerFactory.getLogger(IndexController.class);

    @RequestMapping("/")
    public String index(final ModelMap model) {
        JsonObject object = new JsonObject();
        JsonArray languages = new JsonArray(); // 创建json数组
        JsonObject language = new JsonObject();
        language.addProperty("id", 1);
        language.addProperty("ide", "Eclipse");
        language.addProperty("name", "java");
        languages.add(language);               // 将json对象添加到数组

        language = new JsonObject();
        language.addProperty("id", 2);
        language.addProperty("ide", "XCode");
        language.addProperty("name", "Swift");
        languages.add(language);

        language = new JsonObject();
        language.addProperty("id", 3);
        language.addProperty("ide", "Visual Studio");
        language.addProperty("name", "C#");
        languages.add(language);

        object.add("languages", languages);   // 将数组添加到json对象
        object.addProperty("pop", true);

        String jsonStr = object.toString();   // 将json对象转化成json字符串
        model.put("log_message", jsonStr);
        logger.info(jsonStr);

        return "index";
    }
}

application.properties

logback-spring.xml 的配置启用了 spring profile，所以必须要在配置spring.profiles.active

spring.profiles.active=dev

ELK

安装

从官网下载 Elasticsearch，Kibana，Logstash 安装包，目前的测试环境是CentOS 7.3 x86_64

在所有安装工作之前，需要安装JDK，官网推荐使用Oracle JDK，而不是使用openjdk。

安装包

• elasticsearch-5.4.2.zip

• kibana-5.4.2-linux-x86_64.tar.gz

• logstash-5.4.2.tar.gz

配置

elasticsearch 单机模式，所以使用默认配置

kibana 配置，需要配置host和elasticsearch的地址

server.port: 5601
server.host: "192.168.77.205"
elasticsearch.url: "http://localhost:9200"

logstash example.conf 配置

input {
  tcp {
    port => 9516
    codec => json_lines
  }
}

output {
  stdout {
    codec => rubydebug
    }
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "logstash-test1"
  }
}

运行

#elasticsearch
cd elasticsearch-5.4.2
./bin/elasticsearch -d
#kibana
cd kibana-5.4.2-linux-x86_64
nohup ./bin/kibana &
#logstash
cd logstash-5.4.2
./bin/logstash -f example.conf