Fix a few issues when using the logstash-docgen tools

Fixes the following issues:

- Make multiline plugin definition correctly take the description See
rabbit mq input or output
- Use a regexp to ignore encoding comment
- Use `TODO` The doc gen will now ignore some TODOs.
- rules are now executed in the correct order,
- make sure to ignore comment not associated with a descrition of an
attributes

Ref: #6692

Fixes #6714

tools/logstash-docgen/lib/logstash/docgen/static_parser.rb

@@ -4,7 +4,10 @@ module LogStash::Docgen
   # his ancestors, the result would be the description of the plugin and the
   # actual documentation for each of the option.
   class StaticParser
-    COMMENTS_IGNORE = ["encoding: utf-8"]
+    COMMENTS_IGNORE = Regexp.union(
+      Regexp.new(/encoding: utf-8/i),
+      Regexp.new(/TODO:/)
+    )
 
     VALID_CLASS_NAME = /^LogStash::(Codecs|Inputs|Filters|Outputs)::(\w+)/
     COMMENT_RE = /^ *#(?: (.*)| *$)/
@@ -12,20 +15,18 @@ module LogStash::Docgen
     ENDLINES_RE = /\r\n|\n/
     CLASS_DEFINITION_RE = /^ *class\s(.*) < *(::)?LogStash::(Outputs|Filters|Inputs|Codecs)::(\w)/ 
     NEW_CLASS_DEFINITION_RE = /^module (\w+) module (\w+) class\s(.*) < *(::)?LogStash::(Outputs|Filters|Inputs|Codecs)::(\w)/
-    NEW_CLASS_DEFINITION_RE_ML = /module LogStash\n\s+module (Inputs|Codec|Outputs|Filters)\n.+\s+class (\w+) < *(::)?LogStash::(Inputs|Outputs|Filters|Codec)::/m
+    NEW_CLASS_DEFINITION_RE_ML = /^\s*class\s(.*) < *(::)?LogStash::(Outputs|Filters|Inputs|Codecs)::(\w+)/
     CONFIG_OPTION_RE = /^\s*((mod|base).)?config +[^=].*/
     CONFIG_NAME_RE = /^ *config_name .*/
-    RESET_DOCUMENTATION_BUFFER_RE = /^ *(class|def|module) /
 
     def initialize(context)
-      @rules = {
+      @rules =  [
-        COMMENT_RE => :parse_comment,
+        [ COMMENT_RE, :parse_comment ],
-        CLASS_DEFINITION_RE => :parse_class_description,
+        [ CLASS_DEFINITION_RE, :parse_class_description ],
-        NEW_CLASS_DEFINITION_RE => :parse_new_class_description,
+        [ NEW_CLASS_DEFINITION_RE_ML, :parse_new_class_description ],
-        CONFIG_OPTION_RE => :parse_config,
+        [ CONFIG_OPTION_RE, :parse_config ],
-        CONFIG_NAME_RE => :parse_config_name,
+        [ CONFIG_NAME_RE, :parse_config_name ],
-        RESET_DOCUMENTATION_BUFFER_RE => :update_description
+      ]
-      }
 
       @context = context
 
@@ -45,18 +46,21 @@ module LogStash::Docgen
     end
 
     def parse_new_class_description(class_definition)
-      @context.section = class_definition[2].downcase.gsub(/s$/, '')
+      @context.section = class_definition[3].downcase.gsub(/s$/, '')
-      @context.name = "#{class_definition[1]}::#{class_definition[2]}::#{class_definition[3]}"
+      @context.name = "LogStash::#{class_definition[3]}::#{class_definition[2]}"
 
       update_description
     end
 
     # This is not obvious, but if the plugin define a class before the main class it can trip the buffer
     def update_description(match = nil)
+      return unless reading_header?
+
       description = flush_buffer
 
       # can only be change by the main file
       @context.description = description if !@context.has_description? && main?
+      transition_to_reading_attributes
     end
 
     def parse_config_name(match)
@@ -81,9 +85,25 @@ module LogStash::Docgen
 
     def parse(file, main = false)
       @main = main
+      main ? transition_to_reading_header() : transition_to_reading_attributes()
+
       reset_buffer
       string = read_file(file)
-      extract_lines(string).each { |line| parse_line(line) }
+      extract_lines(string).each do |line|
+        parse_line(line)
+      end
+    end
+
+    def transition_to_reading_attributes
+      @state = :reading_attributes
+    end
+
+    def transition_to_reading_header
+      @state = :reading_header
+    end
+
+    def reading_header?
+      @state == :reading_header
     end
 
     def main?
@@ -91,11 +111,13 @@ module LogStash::Docgen
     end
 
     def parse_line(line)
-      @rules.each do |re, action|
+      @rules.each do |rule|
+        re, action = rule
         if match = re.match(line)
           send(action, match)
+          break
         end
-      end 
+      end
     end
 
     def extract_lines(string)
@@ -115,7 +137,7 @@ module LogStash::Docgen
     end
 
     def ignore_comment?(comment)
-      COMMENTS_IGNORE.include?(comment)
+      COMMENTS_IGNORE.match(comment)
     end
 
     def comment?(line)
@@ -149,7 +171,7 @@ module LogStash::Docgen
     #    ....
     #    module Inputs
     #    ...
-    #    class File
+    #    class File # new kid on the block
     def extract_class_name(file)
       content = read_file(file)
       legacy_definition = content.match(CLASS_DEFINITION_RE)
@@ -162,7 +184,7 @@ module LogStash::Docgen
           legacy_definition[1]
         else
           m = content.match(NEW_CLASS_DEFINITION_RE_ML)
-          "LogStash::#{m[1]}::#{m[2]}"
+          "LogStash::#{m[3]}::#{m[1]}"
         end
       end
     end