Logstash не читает Logfile

Question

Logstash не читает Logfile

Я пытаюсь отправить доступ к log-Access-Log от моего администратора в виде файла.log через Logstash на мои экземпляры Graylog2. Я делаю это с большим количеством файлов журнала nginx, поэтому мне интересно, где я застрял. Возможно, вы можете помочь мне немного дальше.

Я дважды проверил все с помощью отладчика Grok, и все идет хорошо. Я так застрял. в отладчике все работает, logstash ничего не находит в лог файлах. Никогда такого раньше не было.

Вот мой простой Конфиг для Logstash

input {
  file {
    path => ["/var/log/varnish/varnish.log"]
    type => "apachecombined"
  }
}
filter {
  grok { 
    type => "apachecombined"
    match => ["message", '%{IP:client} %{TIMESTAMP_ISO8601:timestamp} "%{WORD:verb} %{URIPROTO}://%{URIHOST:project}%{URIPATHPARAM} HTTP/%{NUMBER:httpversion}" %{NUMBER:status} %{NUMBER:responsesize} "%{DATA:referer}" "%{DATA:useragent}" %{NUMBER:time_till_first_byte} %{WORD:hit_miss}']
  }
}
output {
    gelf {
      host => "192.168.2.128"
      port => "12202"
      level => "info"
    }
}

Это извлечение из log-access-log:

85.182.210.74 2014-10-22T15:01:41+0200 "GET http://mydomain.de/favicon.ico HTTP/1.1" 200 1150 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36 OPR/25.0.1614.50" 0.000947 miss
85.182.210.74 2014-10-22T15:01:42+0200 "GET http://mydomain.de/api/doc/ HTTP/1.1" 401 596 "-" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.101 Safari/537.36 OPR/25.0.1614.50" 0.000836 miss
17.47.221.166 2014-10-22T15:02:46+0200 "GET http://mydomain.de/stream/contents/2/page? HTTP/1.1" 200 2559 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Mobile/12B411 (5475740496)" 0.125556 miss
17.47.221.166 2014-10-22T15:02:46+0200 "GET http://mydomain.de/stream/contents/1282/page? HTTP/1.1" 200 10312 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Mobile/12B411 (5475740496)" 0.130131 miss
17.47.221.166 2014-10-22T15:02:47+0200 "GET http://mydomain.de/stream/view/home?&page=0 HTTP/1.1" 200 53621 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Mobile/12B411 (5475740496)" 0.853203 miss
17.47.221.166 2014-10-22T15:02:59+0200 "GET http://mydomain.de/stream/view/home?&page=1 HTTP/1.1" 200 87788 "-" "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Mobile/12B411 (5475740496)" 0.857153 miss

Это отладочный вывод Logstash, который НЕ посылает и не распознает ничего:

Чтение конфигурационного файла {:file=>"logstash/agent.rb",:level=>:debug,:line=>"301"}

Compiled pipeline code:   

@inputs = []   

@filters = []   

@outputs = []   

@input_file_1 = plugin("input", "file", LogStash::Util.hash_merge_many({ "path" => [("/var/log/varnish/varnish.log".force_encoding("UTF-8"))] }, { "type" => ("apachecombined".force_encoding("UTF-8")) }))   



@inputs << @input_file_1   

@filter_grok_2 = plugin("filter", "grok", LogStash::Util.hash_merge_many({ "type" => ("apachecombined".force_encoding("UTF-8")) }, { "match" => [("message".force_encoding("UTF-8")), ("%{IP:client} %{TIMESTAMP_ISO8601:timestamp} \"%{WORD:verb} %{URIPROTO}://%{URIHOST:project}%{URIPATHPARAM} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status} %{NUMBER:responsesize} \"%{DATA:referer}\" \"%{DATA:useragent}\" %{NUMBER:time_till_first_byte} %{WORD:hit_miss}".force_encoding("UTF-8"))] }))   



@filters << @filter_grok_2   

@output_gelf_3 = plugin("output", "gelf", LogStash::Util.hash_merge_many({ "host" => ("192.168.2.128".force_encoding("UTF-8")) }, { "port" => ("12202".force_encoding("UTF-8")) }, { "level" => ("info".force_encoding("UTF-8")) }))   



@outputs << @output_gelf_3   

  @filter_func = lambda do |event, &block|   

    extra_events = []   

    @logger.debug? && @logger.debug("filter received", :event => event.to_hash)   

    newevents = []   

    extra_events.each do |event|   

      @filter_grok_2.filter(event) do |newevent|   

        newevents << newevent   

      end   

    end   

    extra_events += newevents   

    @filter_grok_2.filter(event) do |newevent|   

      extra_events << newevent   

    end   

    if event.cancelled?   

      extra_events.each(&block)   

      return   

    end   



    extra_events.each(&block)   

  end   

  @output_func = lambda do |event, &block|   

    @logger.debug? && @logger.debug("output received", :event => event.to_hash)   

    @output_gelf_3.handle(event)   



  end {:level=>:debug, :file=>"logstash/pipeline.rb", :line=>"26"}   

Using milestone 2 input plugin 'file'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones {:level=>:warn, :file=>"logstash/config/mixin.rb", :line=>"209"}   

config LogStash::Codecs::Plain/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@path = ["/var/log/varnish/varnish.log"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@type = "apachecombined" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@debug = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@codec = <LogStash::Codecs::Plain charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@stat_interval = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@discover_interval = 15 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@sincedb_write_interval = 15 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Inputs::File/@start_position = "end" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

You are using a deprecated config setting "type" set in grok. Deprecated settings will continue to work, but are scheduled for removal from logstash in the future. You can achieve this same behavior with the new conditionals, like: `if [type] == "sometype" { grok { ... } }`. If you have any questions about this, please visit the #logstash channel on freenode irc. {:name=>"type", :plugin=><LogStash::Filters::Grok --->, :level=>:warn, :file=>"logstash/config/mixin.rb", :line=>"69"}   

config LogStash::Filters::Grok/@type = "apachecombined" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@match = {"message"=>"%{IP:client} %{TIMESTAMP_ISO8601:timestamp} \"%{WORD:verb} %{URIPROTO}://%{URIHOST:project}%{URIPATHPARAM} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status} %{NUMBER:responsesize} \"%{DATA:referer}\" \"%{DATA:useragent}\" %{NUMBER:time_till_first_byte} %{WORD:hit_miss}"} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@add_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@remove_tag = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@add_field = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@remove_field = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@patterns_dir = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@drop_if_match = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@break_on_match = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@named_captures_only = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@keep_empty_captures = false {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@singles = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@tag_on_failure = ["_grokparsefailure"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Filters::Grok/@overwrite = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

Using milestone 2 output plugin 'gelf'. This plugin should be stable, but if you see strange behavior, please let us know! For more information on plugin milestones, see http://logstash.net/docs/1.4.2-modified/plugin-milestones {:level=>:warn, :file=>"logstash/config/mixin.rb", :line=>"209"}   

config LogStash::Codecs::Plain/@charset = "UTF-8" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@host = "192.168.2.128" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@port = 12202 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@level = ["info"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@type = "" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@exclude_tags = [] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@codec = <LogStash::Codecs::Plain charset=>"UTF-8"> {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@workers = 1 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@chunksize = 1420 {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@sender = "%{host}" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@ship_metadata = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@ship_tags = true {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@ignore_metadata = ["@timestamp", "@version", "severity", "host", "source_host", "source_path", "short_message"] {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@custom_fields = {} {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@full_message = "%{message}" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

config LogStash::Outputs::Gelf/@short_message = "short_message" {:level=>:debug, :file=>"logstash/config/mixin.rb", :line=>"105"}   

Registering file input {:path=>["/var/log/varnish/varnish.log"], :level=>:info, :file=>"logstash/inputs/file.rb", :line=>"74"}   

No sincedb_path set, generating one based on the file path {:sincedb_path=>"/root/.sincedb_a84185fecd92ea9dac476d36fa8cb7ff", :path=>["/var/log/varnish/varnish.log"], :level=>:info, :file=>"logstash/inputs/file.rb", :line=>"115"}   

_sincedb_open: reading from /root/.sincedb_a84185fecd92ea9dac476d36fa8cb7ff {:level=>:debug, :file=>"filewatch/tail.rb", :line=>"199"}   

_discover_file_glob: /var/log/varnish/varnish.log: glob is: [] {:level=>:debug, :file=>"filewatch/watch.rb", :line=>"117"}   

Grok patterns path {:patterns_dir=>["/opt/logstash/patterns/*"], :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"240"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/ruby", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/mongodb", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/redis", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/grok-patterns", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/linux-syslog", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/java", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/haproxy", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/mcollective", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/nagios", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/firewalls", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/mcollective-patterns", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/junos", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Grok loading patterns from file {:path=>"/opt/logstash/patterns/postgresql", :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"247"}   

Match data {:match=>{"message"=>"%{IP:client} %{TIMESTAMP_ISO8601:timestamp} \"%{WORD:verb} %{URIPROTO}://%{URIHOST:project}%{URIPATHPARAM} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status} %{NUMBER:responsesize} \"%{DATA:referer}\" \"%{DATA:useragent}\" %{NUMBER:time_till_first_byte} %{WORD:hit_miss}"}, :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"254"}   

Grok compile {:field=>"message", :patterns=>["%{IP:client} %{TIMESTAMP_ISO8601:timestamp} \"%{WORD:verb} %{URIPROTO}://%{URIHOST:project}%{URIPATHPARAM} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status} %{NUMBER:responsesize} \"%{DATA:referer}\" \"%{DATA:useragent}\" %{NUMBER:time_till_first_byte} %{WORD:hit_miss}"], :level=>:info, :file=>"logstash/filters/grok.rb", :line=>"265"}   

regexp: apachecombined/message {:pattern=>"%{IP:client} %{TIMESTAMP_ISO8601:timestamp} \"%{WORD:verb} %{URIPROTO}://%{URIHOST:project}%{URIPATHPARAM} HTTP/%{NUMBER:httpversion}\" %{NUMBER:status} %{NUMBER:responsesize} \"%{DATA:referer}\" \"%{DATA:useragent}\" %{NUMBER:time_till_first_byte} %{WORD:hit_miss}", :level=>:debug, :file=>"logstash/filters/grok.rb", :line=>"267"}   

Pipeline started {:level=>:info, :file=>"logstash/pipeline.rb", :line=>"78"}   

_discover_file_glob: /var/log/varnish/varnish.log: glob is: [] {:level=>:debug, :file=>"filewatch/watch.rb", :line=>"117"}   

_discover_file_glob: /var/log/varnish/varnish.log: glob is: [] {:level=>:debug, :file=>"filewatch/watch.rb", :line=>"117"}   

_discover_file_glob: /var/log/varnish/varnish.log: glob is: [] {:level=>:debug, :file=>"filewatch/watch.rb", :line=>"117"}   

_discover_file_glob: /var/log/varnish/varnish.log: glob is: [] {:level=>:debug, :file=>"filewatch/watch.rb", :line=>"117"}

1

logstash grok

Источник

user2776079 22 окт '14 в 13:37

0 ответов

Другие вопросы по тегам logstash grok