# File lib/taskjuggler/TextParser/Scanner.rb, line 247
    def initialize(masterFile, log, tokenPatterns, defaultMode)
      @masterFile = masterFile
      @messageHandler = TaskJuggler::MessageHandlerInstance.instance
      @log = log
      # This table contains all macros that may be expanded when found in the
      # text.
      @macroTable = MacroTable.new
      # The currently processed IO object.
      @cf = nil
      # This Array stores the currently processed nested files. It's an Array
      # of Arrays. The nested Array consists of 2 elements, the IO object and
      # the @tokenBuffer.
      @fileStack = []
      # This flag is set if we have reached the end of a file. Since we will
      # only know when the next new token is requested that the file is really
      # done now, we have to use this flag.
      @finishLastFile = false
      # True if the scanner operates on a buffer.
      @fileNameIsBuffer = false
      # A SourceFileInfo of the start of the currently processed token.
      @startOfToken = nil
      # Line number correction for error messages.
      @lineDelta = 0
      # Lists of regexps that describe the detectable tokens. The Arrays are
      # grouped by mode.
      @patternsByMode = { }
      # The currently active scanner mode.
      @scannerMode = nil
      # The mode that the scanner is in at the start and end of file
      @defaultMode = defaultMode
      # Points to the currently active pattern set as defined by the mode.
      @activePatterns = nil

      @mrxs = MRXScanner.new

      tokenPatterns.each do |pat|
        type = pat[0]
        regExp = pat[1]
        mode = pat[3] || :tjp
        postProc = pat[4]
        addPattern(type, Regexp.new(regExp), mode, postProc)
        @mrxs.addRegExp(regExp, type, postProc, mode)
      end
      self.mode = defaultMode
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 480
    def addMacro(macro)
      @macroTable.add(macro)
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 301
    def addPattern(type, regExp, mode, postProc = nil)
      if mode.is_a?(Array)
        mode.each do |m|
          # The pattern is active in multiple modes
          @patternsByMode[m] = [] unless @patternsByMode.include?(m)
          @patternsByMode[m] << [ type, regExp, postProc ]
        end
      else
        # The pattern is only active in one specific mode.
        @patternsByMode[mode] = [] unless @patternsByMode.include?(mode)
        @patternsByMode[mode] << [ type, regExp, postProc ]
      end
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 345
    def close
      unless @fileNameIsBuffer
        @log.startProgressMeter("Reading file #{@masterFile}")
        @log.stopProgressMeter
      end
      @fileStack = []
      @cf = @tokenBuffer = nil
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 510
    def error(id, text, sfi = nil, data = nil)
      message(:error, id, text, sfi, data)
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 494
    def expandMacro(prefix, args, callLength)
      # Get the expanded macro from the @macroTable.
      macro, text = @macroTable.resolve(args, sourceFileInfo)
      unless macro && text
        error('undefined_macro', "Undefined macro '#{args[0]}' called")
      end

      # If the expanded macro is empty, we can ignore it.
      return if text == ''

      unless @cf.injectMacro(macro, args, prefix + text, callLength)
        error('macro_stack_overflow', "Too many nested macro calls.")
      end
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 402
    def fileName
      @cf ? @cf.fileName : @masterFile
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 358
    def include(includeFileName, sfi, &block)
      if includeFileName[0] != '/'
        pathOfCallingFile = @fileStack.last[0].dirname
        path = pathOfCallingFile.empty? ? '' : pathOfCallingFile + '/'
        # If the included file is not an absolute name, we interpret the file
        # name relative to the including file.
        includeFileName = path + includeFileName
      end

      # Try to dectect recursive inclusions. This will not work if files are
      # accessed via filesystem links.
      @fileStack.each do |entry|
        if includeFileName == entry[0].fileName
          error('include_recursion',
                "Recursive inclusion of #{includeFileName} detected", sfi)
        end
      end

      # Save @tokenBuffer in the record of the parent file.
      @fileStack.last[1] = @tokenBuffer unless @fileStack.empty?
      @tokenBuffer = nil
      @finishLastFile = false

      # Open the new file and push the handle on the @fileStack.
      begin
        @fileStack << [ (@cf = FileStreamHandle.new(includeFileName, @log,
                                                    self)), nil, block ]
        @log.msg { "Parsing file #{includeFileName}" }
      rescue StandardError
        error('bad_include', "Cannot open include file #{includeFileName}", sfi)
      end

      # Return the name of the included file.
      includeFileName
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 485
    def macroDefined?(name)
      @macroTable.include?(name)
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 317
    def mode=(newMode)
      #puts "**** New mode: #{newMode}"
      @activePatterns = @patternsByMode[newMode]
      raise "Undefined mode #{newMode}" unless @activePatterns
      @scannerMode = newMode
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 421
    def nextToken
      # If we have a pushed-back token, return that first.
      unless @tokenBuffer.nil?
        res = @tokenBuffer
        @tokenBuffer = nil
        return res
      end

      if @finishLastFile
        # The previously processed file has now really been processed to
        # completion. Close it and remove the corresponding entry from the
        # @fileStack.
        @finishLastFile = false
        #@log.msg { "Completed file #{@cf.fileName}" }

        # If we have a block to be executed on EOF, we call it now.
        onEof = @fileStack.last[2]
        onEof.call if onEof

        @cf.close if @cf
        @fileStack.pop

        if @fileStack.empty?
          # We are done with the top-level file now.
          @cf = @tokenBuffer = nil
          @finishLastFile = true
          return [ :endOfText, '<EOT>', @startOfToken ]
        else
          # Continue parsing the file that included the current file.
          @cf, tokenBuffer = @fileStack.last
          @log.msg { "Parsing file #{@cf.fileName} ..." }
          # If we have a left over token from previously processing this file,
          # return it now.
          if tokenBuffer
            @finishLastFile = true if tokenBuffer[0] == :eof
            return tokenBuffer
          end
        end
      end

      # Start processing characters from the input.
      # (Un)-comment to toggle between StringScanner (scanToken) and
      # MRXScanner (mrxScanToken).
      scanToken
      # mrxScanToken
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 327
    def open(fileNameIsBuffer = false)
      @fileNameIsBuffer = fileNameIsBuffer
      if fileNameIsBuffer
        @fileStack = [ [ @cf = BufferStreamHandle.new(@masterFile, @log, self),
                         nil, nil ] ]
      else
        begin
          @fileStack = [ [ @cf = FileStreamHandle.new(@masterFile, @log, self),
                           nil, nil ] ]
        rescue IOError, SystemCallError
          error('open_file', "Cannot open file #{@masterFile}: #{$!}")
        end
      end
      @masterPath = @cf.dirname + '/'
      @tokenBuffer = nil
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 470
    def returnToken(token)
      #@log.msg { "-> Returning Token: [#{token[0]}][#{token[1]}]" }
      unless @tokenBuffer.nil?
        $stderr.puts @tokenBuffer
        raise "Fatal Error: Cannot return more than 1 token in a row"
      end
      @tokenBuffer = token
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 395
    def sourceFileInfo
      @cf ? SourceFileInfo.new(fileName, @cf.lineNo - @lineDelta, 0) :
            SourceFileInfo.new(@masterFile, 0, 0)
    end

# File lib/taskjuggler/TextParser/Scanner.rb, line 514
    def warning(id, text, sfi = nil, data = nil)
      message(:warning, id, text, sfi, data)
    end