Logstash/Logspout integration with Opendistro for Elasticsearch

#1

I am trying to push docker logs to Elasticsearch using logstash and logspout. But logs are not being pushed to Elasticsearch.

my docker compose file

version: '3'
services:
  odfe-node1:
    build:
      dockerfile: Dockerfile
      context: ./elasticsearch
    container_name: odfe-node1
    environment:
      - cluster.name=odfe-cluster
      - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
      - opendistro_security.ssl.http.enabled=false
      - discovery.zen.ping.unicast.hosts=odfe-node1

    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
       - odfe-data1:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      #- 9700:9600 # required for Performance Analyzer
    networks:
      - odfe-net
  odfe-node2:
    image: amazon/opendistro-for-elasticsearch:0.7.0
    container_name: odfe-node2
    environment:
     - cluster.name=odfe-cluster
     - bootstrap.memory_lock=true
     - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     - discovery.zen.ping.unicast.hosts=odfe-node1
     - opendistro_security.ssl.http.enabled=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
       - odfe-data2:/usr/share/elasticsearch/data
       - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
    networks:
      - odfe-net
  kibana:
    build:
      dockerfile: Dockerfile
      context: ./kibana
    container_name: odfe-kibana
    ports:
      - 5601:5601
    expose:
      - "5601"
    environment:
      ELASTICSEARCH_URL: http://odfe-node1:9200
      OPENDISTRO_SECURITY_MULTITENANCY_ENABLED: "false"
      XPACK_SECURITY_ENABLED: "false"
    networks:
      - odfe-net

  logstash:
    build:
      dockerfile: Dockerfile
      context: ./logstash
    volumes:
      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
      - ./logstash/pipeline:/usr/share/logstash/pipeline:ro
    ports:
      - "5000:5000"
      - "9600:9600"
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - odfe-net
    depends_on:
      - odfe-node1
  logspout:
    image: gliderlabs/logspout:v2
    restart: always
    command: 'udp://logstash:5044'
    #environment:
      #ROUTE_URIS: udp://logstash:5000
    links:
      - logstash
    volumes:
      - '/var/run/docker.sock:/tmp/docker.sock'
    depends_on:
      - odfe-node2
      - logstash
      - kibana
    networks:
      - odfe-net
volumes:
  odfe-data1:
  odfe-data2:

networks:
  odfe-net:
      driver: bridge

Logstash confiugration

input{
        udp{
            port => 5044


       }
}
filter {
  if [docker][image] =~ /^logstash/ {
    drop { }
  }
}

## Add your filters / logstash plugins configuration here

output {
        elasticsearch {
                hosts => "odfe-node2:9200"

Logstash and Logspout were starting successfully,please refer below logs,

[2019-03-26T08:10:29,250][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.queue", :path=>"/usr/share/logstash/data/queue"}
[2019-03-26T08:10:29,393][INFO ][logstash.setting.writabledirectory] Creating directory {:setting=>"path.dead_letter_queue", :path=>"/usr/share/logstash/data/dead_letter_queue"}
[2019-03-26T08:10:31,047][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2019-03-26T08:10:31,232][INFO ][logstash.runner          ] Starting Logstash {"logstash.version"=>"6.6.1"}
[2019-03-26T08:10:31,392][INFO ][logstash.agent           ] No persistent UUID file found. Generating new UUID {:uuid=>"dc72a279-239d-4edd-ad44-41b5da944490", :path=>"/usr/share/logstash/data/uuid"}
[2019-03-26T08:11:03,029][INFO ][logstash.pipeline        ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[2019-03-26T08:11:05,513][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://odfe-node2:9200/]}}
[2019-03-26T08:11:07,080][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://odfe-node2:9200/"}
[2019-03-26T08:11:07,681][INFO ][logstash.outputs.elasticsearch] ES Output version determined {:es_version=>6}
[2019-03-26T08:11:07,715][WARN ][logstash.outputs.elasticsearch] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>6}
[2019-03-26T08:11:07,934][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//odfe-node2:9200"]}
[2019-03-26T08:11:08,085][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[2019-03-26T08:11:08,490][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2019-03-26T08:11:08,650][INFO ][logstash.pipeline        ] Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x47ac2e3 run>"}
[2019-03-26T08:11:09,249][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2019-03-26T08:11:09,329][INFO ][logstash.inputs.udp      ] Starting UDP listener {:address=>"0.0.0.0:5044"}
[2019-03-26T08:11:10,395][INFO ][logstash.inputs.udp      ] UDP listener started {:address=>"0.0.0.0:5044", :receive_buffer_bytes=>"106496", :queue_size=>"2000"}
[2019-03-26T08:11:12,170][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}

Logspout logs

root@server1:~/opendistro-elk-logspout# docker logs 55ce7424f363
2019/03/26 08:08:16 routing all to udp://logstash:5044
2019/03/26 08:08:16 loading and persisting routes in /mnt/routes
2019/03/26 08:08:16 logspout v2 serving http on :8000