Logstash+Logback日志收集

日志客户端配置

添加Maven依赖

<dependency>
    <groupId>com.cwbase</groupId>
    <artifactId>logback-redis-appender</artifactId>
    <exclusions>
        <exclusion>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
        </exclusion>
    </exclusions>
</dependency>

配置logback.xml

<!--logback集成logstash-->
<appender name="Redis" class="com.cwbase.logback.RedisAppender">
    <!--应用名称，可自定义-->
    <source>woo-erp</source>
    <!--Redis服务器地址-->
    <host>119.119.23.35</host>
    <!--Redis服务器端口-->
    <port>6039</port>
    <!--Redis服务器密码(可选)-->
    <password>Ht123123123</password>
    <!-- 这个Key是Redis List的Key，需要和Logstash读取Redis的Key匹配 -->
    <key>logstash</key>
    <mdc>true</mdc>
    <location>true</location>
    <database>15</database>
    <!-- Use your own Custom Layout here -->
    <!--<layout class="com.cwbase.logback.JSONEventLayout">
        &lt;!&ndash; JSONEventLayout Attributes Here &ndash;&gt;
        <source>mySource</source>
        <sourcePath>mySourcePath</sourcePath>
        <type>myApplication</type>
        <tags>production</tags>
    </layout>-->
</appender>
<appender name="AsyncRedis" class="ch.qos.logback.classic.AsyncAppender">
    <appender-ref ref="Redis" />
</appender>


<root ....>
    <!-- 添加AsyncRedis -->
    <appender-ref ref="AsyncRedis" />
</root>

Logstash服务端配置

创建文件redis_input.conf放在conf.d目录下

input {
    # 从redis读取日志
    redis{
        data_type => "list"
        key => "logstash"
        host => "119.119.23.35"
        port => "6039"
        password => "Ht123123123"
        db => 15
        threads => 5
    }
}

输出到ES示例

创建文件es-output.conf放在conf.d目录下

output {
        elasticsearch {
                hosts => ["http://localhost:9200"]
                index => "%{source}_index_%{+YYYY.MM}"
                template => "/etc/logstash/conf.d/logstash_template.json"
                template_name => "logstash_template"
                template_overwrite => true
                user => "elastic"
                # password => "1qaz@WSX3edc"
                pool_max => 500
                pool_max_per_route => 2000
                retry_initial_interval => 4
                retry_max_interval => 16
        }
}

logstash_template.json示例

{
    "template": "default_template",
    "order": 1,
    "settings": {
        "index.number_of_shards": 5,
        "number_of_replicas": 0
    },
    "mappings": {
        "_default_": {
            "_all": {
                "enabled": true,
                "omit_norms": true
            },
            "dynamic_templates": [
                {
                    "message_field": {
                        "match": "message",
                        "match_mapping_type": "string",
                        "mapping": {
                            "type": "string",
                            "index": "analyzed",
                            "omit_norms": true,
                            "fielddata": {
                                "format": "disabled"
                            }
                        }
                    }
                },
                {
                    "number_fields": {
                        "match": "*",
                        "match_mapping_type": "integer",
                        "mapping": {
                            "type": "integer",
                            "index": "not_analyzed",
                            "doc_values": true
                        }
                    }
                },
                {
                    "string_fields": {
                        "match": "*",
                        "match_mapping_type": "string",
                        "mapping": {
                            "type": "string",
                            "index": "not_analyzed",
                            "doc_values": true
                        }
                    }
                }
            ]
        }
    }
}