Fluentd + InfluxDB + Grafana 日志收集的实践

$ brew update
$ brew install influxdb

$ wget http://grafanarel.s3.amazonaws.com/grafana-1.9.1.tar.gz
$ tar zxf grafana-1.9.1.tar.gz
$ cd grafana-1.9.1
$ python -m SimpleHTTPServer
Serving HTTP on 0.0.0.0 port 8000 ...

$ gem install fluentd

equire 'logger'
require 'json'
require 'time'
module WhosvPerformance
  module Rack
    class WhosvRackDebugger
      def initialize(app)
        @app = app
      end

      def call(env)
        @env = env
        before_log
        status, headers, body = @app.call env
        log_runtime headers
        [status, headers, body]
      end
      private
      def before_log
        @whosv_performance_logger = Logger.new('rpm.log')
        original_formatter = Logger::Formatter.new
        @whosv_performance_logger.formatter = proc { |severity, datetime, progname, msg|
          {:severity => severity,:datetime => datetime.strftime("%Y-%m-%d %H:%M:%S")}.merge(msg).to_json << "\n"
        }
      end
      def log_runtime headers
        @whosv_performance_logger.info ({:method => @env['REQUEST_METHOD'],:path => @env['REQUEST_PATH'],:runtime => headers['X-Runtime'].to_f,:component=>"Rack"})
      end
    end
  end
end

config.middleware.insert_before Rack::Runtime,WhosvPerformance::Rack::WhosvRackDebugger

require 'logger'
require 'json'
require 'time'
module API
  module Middleware
    class WhosvPerformance < Grape::Middleware::Base
      def before
        @whosv_performance_logger = Logger.new('rpm.log')
        original_formatter = Logger::Formatter.new
        @whosv_performance_logger.formatter = proc { |severity, datetime, progname, msg|
          {:severity => severity,:datetime => datetime.strftime("%Y-%m-%d %H:%M:%S")}.merge(msg).to_json << "\n"
        }
        @whosv_start_time = Time.now
        puts @app
        nil
      end
      def after
        puts "End Grape"
        @whosv_end_time = Time.now
        whosv_api_runtime = @whosv_end_time - @whosv_start_time
        @whosv_performance_logger.info result_runtime(whosv_api_runtime)
        nil
      end

      private
      def result_runtime runtime
        {
            :method => @env['REQUEST_METHOD'],
            :path => @env['REQUEST_PATH'],
            :runtime => runtime,
            :component => 'Grape'
        }
      end
    end
  end
end

$ fluent-gem install fluent-plugin-influxdb

## File input
## read apache logs with tag=apache.access
<source>
  @type tail
  format json
  time_key datetime
  time_format %Y-%m-%d %H:%M:%S
  path /Users/megrez/Code/whosv/whosv-rails/rpm.log
  pos_file /Users/megrez/Code/whosv/whosv-rails/rpm.log.pos
  tag api.performance
</source>
<match api.performance>
  @type influxdb
  host localhost
  port 8086
  dbname whosv_rails_dev
  user root
  password root
  time_precision s
</match>