def new_token(type, value, opts = {})
if opts[:chunk]
line_no = opts[:chunk].count("\n") + 1
if line_no == 1
column = opts[:chunk].length
else
column = opts[:chunk].length - opts[:chunk].rindex("\n") - 1
end
column += 1 if column == 0
else
column = opts[:column]
line_no = opts[:line]
end
token = Token.new(type, value, line_no, column)
unless tokens.last.nil?
token.prev_token = tokens.last
tokens.last.next_token = token
unless FORMATTING_TOKENS.include?(token.type)
prev_nf_idx = tokens.rindex { |r| ! FORMATTING_TOKENS.include? r.type }
unless prev_nf_idx.nil?
prev_nf_token = tokens[prev_nf_idx]
prev_nf_token.next_code_token = token
token.prev_code_token = prev_nf_token
end
end
end
token
end