这个是我用的配置,如下:

  1. <match td.*.*>
  2. @type tdlog
  3. apikey YOUR_API_KEY
  4. auto_create_table
  5. buffer_type file
  6. buffer_path /var/log/td-agent/buffer/td
  7. <secondary>
  8. @type file
  9. path /var/log/td-agent/failed_records
  10. </secondary>
  11. </match>
  12. <match debug.**>
  13. @type stdout
  14. </match>
  15. <source>
  16. @type forward
  17. </source>
  18. <source>
  19. @type http
  20. port 8888
  21. </source>
  22. <source>
  23. @type debug_agent
  24. bind 127.0.0.1
  25. port 24230
  26. </source>
  27. <source>
  28. @type tail
  29. path /var/log/nginx/moat.access.log
  30. pos_file /var/log/td-agent/moat-nginx-access.log.pos
  31. tag moat.nginx.access
  32. format /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<forwarded>[^\"]*)") (?<request_id>[^ ]*) (?<request_time>[^ ]*)$/
  33. time_format %d/%b/%Y:%H:%M:%S %z
  34. </source>
  35. <source>
  36. @type tail
  37. path /var/log/nginx/moat.error.log
  38. pos_file /var/log/td-agent/moat-nginx-error.log.pos
  39. tag moat.nginx.error
  40. format /^(?<time>[^ ]+ [^ ]+) \[(?<log_level>.*)\] (?<pid>\d*).(?<tid>[^:]*): (?<message>.*)$/
  41. time_format %Y/%m/%d %H:%M:%S
  42. </source>
  43. <filter moat.nginx.*>
  44. @type record_transformer
  45. <record>
  46. hostname ${hostname}
  47. area usa
  48. </record>
  49. </filter>
  50. <match moat.nginx.*>
  51. @type elasticsearch
  52. logstash_format true
  53. host 172.31.4.58
  54. port 9200
  55. logstash_prefix shaohualee-nginx-qa
  56. type_name fluentd
  57. content_type application/json
  58. template_name shaohualee-nginx
  59. template_file /etc/td-agent/shaohualee-nginx.json
  60. template_overwrite true
  61. </match>
  62. <source>
  63. @type tail
  64. path /opt/app/security-moat-sqa/logs/moat.log
  65. pos_file /var/log/td-agent/moat-api-access.log.pos
  66. tag moat.api.access
  67. <parse>
  68. @type multi_format
  69. <pattern>
  70. format /^(?<level>[^ ]*) \[(?<time>[^\]]*)\] "(?<service>[\S\s]+)" -- "(?<path>[\S\s]+)" "(?<app_version>[\S\s]+)" "(?<phone_model>[\S\s]+)" (?<token>\S+) (?<open_udid>\S+) (?<country>\S+) (?<language>\S+) (?<request_id>\S+) (?<user_id>\S+) (?<station_sn>\S+) (?<device_sn>\S+) (?<latency>[^ ]*) (?<code>[^ ]*) "(?<message>[\S\s]+)" (?<params>[\s\S]*)$/
  71. time_key time
  72. </pattern>
  73. <pattern>
  74. format /^(?<level>[^ ]*) \[(?<time>[^\]]*)\] "(?<service>[\S\s]+)" ~~ "(?<tag>[\S\s]+)" (?<action_id>\S+) (?<action_sub_id>\S+) (?<latency>[^ ]*) "(?<message>[\S\s]+)" (?<params>[\s\S]*)$/
  75. time_key time
  76. </pattern>
  77. </parse>
  78. time_format %Y-%m-%dT%H:%M:%S%z
  79. </source>
  80. <source>
  81. @type tail
  82. path /opt/app/security-moat-dev/logs/moat.log
  83. pos_file /var/log/td-agent/boat-api-access.log.pos
  84. tag moat.api.access
  85. <parse>
  86. @type multi_format
  87. <pattern>
  88. format /^(?<level>[^ ]*) \[(?<time>[^\]]*)\] "(?<service>[\S\s]+)" -- "(?<path>[\S\s]+)" "(?<app_version>[\S\s]+)" "(?<phone_model>[\S\s]+)" (?<token>\S+) (?<open_udid>\S+) (?<country>\S+) (?<language>\S+) (?<request_id>\S+) (?<user_id>\S+) (?<station_sn>\S+) (?<device_sn>\S+) (?<latency>[^ ]*) (?<code>[^ ]*) "(?<message>[\S\s]+)" (?<params>[\s\S]*)$/
  89. time_key time
  90. </pattern>
  91. <pattern>
  92. format /^(?<level>[^ ]*) \[(?<time>[^\]]*)\] "(?<service>[\S\s]+)" ~~ "(?<tag>[\S\s]+)" (?<action_id>\S+) (?<action_sub_id>\S+) (?<latency>[^ ]*) "(?<message>[\S\s]+)" (?<params>[\s\S]*)$/
  93. time_key time
  94. </pattern>
  95. </parse>
  96. time_format %Y-%m-%dT%H:%M:%S%z
  97. </source>
  98. <filter moat.api.*>
  99. @type record_transformer
  100. <record>
  101. hostname ${hostname}
  102. area usa
  103. </record>
  104. </filter>
  105. <match moat.api.*>
  106. @type elasticsearch
  107. logstash_format true
  108. host 172.31.4.58
  109. port 9200
  110. logstash_prefix shaohualee-api-sqa
  111. type_name fluentd
  112. content_type application/json
  113. template_name shaohualee-api
  114. template_file /etc/td-agent/shaohualee-api.json
  115. template_overwrite true
  116. </match>
  117. <source>
  118. @type tail
  119. path /opt/app/security-moat-sqa/logs/logging.log
  120. pos_file /var/log/td-agent/moat-logging-report.pos
  121. tag logging.report.moat
  122. <parse>
  123. @type json
  124. </parse>
  125. </source>
  126. <source>
  127. @type tail
  128. path /opt/app/security-emqtt-sqa/logs/logging.log
  129. pos_file /var/log/td-agent/boat-logging-report.pos
  130. tag logging.report.boat
  131. <parse>
  132. @type json
  133. </parse>
  134. </source>
  135. <match logging.report.*>
  136. @type elasticsearch
  137. logstash_format true
  138. host 172.31.4.58
  139. port 9200
  140. logstash_prefix logging-short-qa
  141. type_name doc
  142. content_type application/json
  143. </match>
  144. <system>
  145. log_level error
  146. </system>
  147. #@include conf.d/*.conf

可以这样子在centos7上启动td-agent

  1. sudo systemctl start td-agent.service
  2. sudo systemctl status td-agent.service #这个很有用,可以查看日志在哪