java JFlex.Main HTMLLexer.lex
*
* HTMLLexer shredder = new HTMLLexer(System.in);
* HTMLToken t;
* while ((t = shredder.getNextToken()) != null){
* System.out.println(t);
* }
*
*
*
*
* There are two HTML Lexers that come with this package. HTMLLexer is a basic HTML lexer
* that knows the difference between tags, text, and comments. HTMLLexer1 knows something
* about the structure of tags and can return names and values from name value pairs. It
* also knows about text elements such as words and character references. The two are
* similar but which you should use depends on your purpose. In my opinion the HTMLLexer1
* is much better for syntax highlighting.
*
* @see HTMLLexer1
* @see HTMLToken
*/
%%
%public
%class HTMLLexer
%implements Lexer
%function getNextToken
%type Token
%{
private int lastToken;
private int nextState=YYINITIAL;
/**
* next Token method that allows you to control if whitespace and comments are
* returned as tokens.
*/
public Token getNextToken(boolean returnComments, boolean returnWhiteSpace)throws IOException{
Token t = getNextToken();
while (t != null && ((!returnWhiteSpace && t.isWhiteSpace()) || (!returnComments && t.isComment()))){
t = getNextToken();
}
return (t);
}
/**
* Prints out tokens from a file or System.in.
* If no arguments are given, System.in will be used for input.
* If more arguments are given, the first argument will be used as
* the name of the file to use as input
*
* @param args program arguments, of which the first is a filename
*/
public static void main(String[] args) {
InputStream in;
try {
if (args.length > 0){
File f = new File(args[0]);
if (f.exists()){
if (f.canRead()){
in = new FileInputStream(f);
} else {
throw new IOException("Could not open " + args[0]);
}
} else {
throw new IOException("Could not find " + args[0]);
}
} else {
in = System.in;
}
HTMLLexer shredder = new HTMLLexer(in);
Token t;
while ((t = shredder.getNextToken()) != null) {
if (t.getID() != CToken.WHITE_SPACE){
System.out.println(t);
}
}
} catch (IOException e){
System.out.println(e.getMessage());
}
}
/**
* Closes the current input stream, and resets the scanner to read from a new input stream.
* All internal variables are reset, the old input stream cannot be reused
* (content of the internal buffer is discarded and lost).
* The lexical state is set to the initial state.
* Subsequent tokens read from the lexer will start with the line, char, and column
* values given here.
*
* @param reader The new input.
* @param yyline The line number of the first token.
* @param yychar The position (relative to the start of the stream) of the first token.
* @param yycolumn The position (relative to the line) of the first token.
* @throws IOException if an IOExecption occurs while switching readers.
*/
public void reset(java.io.Reader reader, int yyline, int yychar, int yycolumn) throws IOException{
yyreset(reader);
this.yyline = yyline;
this.yychar = yychar;
this.yycolumn = yycolumn;
}
%}
%line
%char
%full
%state SCRIPT
%state PRE
%state TEXTAREA
Digit=([0-9])
Letter=([a-zA-Z])
BLANK=([ ])
TAB=([\t])
FF=([\f])
CR=([\r])
LF=([\n])
EOL=({CR}|{LF}|{CR}{LF})
WhiteSpace=({BLANK}|{TAB}|{FF}|{EOL}|" ")
StringLiteral=(([\"][^\"]*[\"])|([\'][^\']*[\']))
NameToken=(({Letter}|{Digit}|[\.\-])+)
Value=({NameToken}|{StringLiteral})
Attribute=({NameToken}(({WhiteSpace}*)"="({WhiteSpace}*){Value})?)
TagStart=("<")
EndTagStart=("")
Doctype=([Dd][Oo][Cc][Tt][Yy][Pp][Ee])
DoctypeText=(([^\>\"\']|{StringLiteral})*)
NameScript=([Ss][Cc][Rr][Ii][Pp][Tt])
FalseEndScript=([\<]|[\<][\/]{WhiteSpace}*|[\<][\/]{WhiteSpace}*[Ss]|[\<][\/]{WhiteSpace}*[Ss][Cc]|[\<][\/]{WhiteSpace}*[Ss][Cc][Rr]|[\<][\/]{WhiteSpace}*[Ss][Cc][Rr][Ii]|[\<][\/]{WhiteSpace}*[Ss][Cc][Rr][Ii][Pp]|[\<][\/]{WhiteSpace}*[Ss][Cc][Rr][Ii][Pp][Tt]{WhiteSpace}*)
ScriptText=(([^\<]|{FalseEndScript}*[\<][^\/\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[^Ss\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Ss][^Cc\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Ss][Cc][^Rr\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Ss][Cc][Rr][^Ii\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Ss][Cc][Rr][Ii][^Pp\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Ss][Cc][Rr][Ii][Pp][^Tt\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Ss][Cc][Rr][Ii][Pp][Tt]{WhiteSpace}*[^\<\>])*)
NamePre=([Pp][Rr][Ee])
FalseEndPre=([\<]|[\<][\/]{WhiteSpace}*|[\<][\/]{WhiteSpace}*[Pp]|[\<][\/]{WhiteSpace}*[Pp][Rr]|[\<][\/]{WhiteSpace}*[Pp][Rr][Ee]{WhiteSpace}*)
PreText=(([^\<]|{FalseEndPre}*[\<][^\/\<]|{FalseEndPre}*[\<][\/]{WhiteSpace}*[^Pp\<]|{FalseEndPre}*[\<][\/]{WhiteSpace}*[Pp][^Rr\<]|{FalseEndPre}*[\<][\/]{WhiteSpace}*[Pp][Rr][^Ee\<]|{FalseEndPre}*[\<][\/]{WhiteSpace}*[Pp][Rr][Ee]{WhiteSpace}*[^\<\>])*)
NameTextArea=([Tt][Ee][Xx][Tt][Aa][Rr][Ee][Aa])
FalseEndTextArea=([\<]|[\<][\/]{WhiteSpace}*|[\<][\/]{WhiteSpace}*[Tt]|[\<][\/]{WhiteSpace}*[Tt][Ee]|[\<][\/]{WhiteSpace}*[Tt][Ee][Xx]|[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt]|[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa]|[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa][Rr]|[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa][Rr][Ee]|[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa][Rr][Ee][Aa]{WhiteSpace}*)
TextAreaText=(([^\<]|{FalseEndScript}*[\<][^\/\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[^Tt\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][^Ee\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][Ee][^Xx\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][^Tt\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][^Aa\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa][^Rr\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa][Rr][^Ee\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa][Rr][Ee][^Aa\<]|{FalseEndScript}*[\<][\/]{WhiteSpace}*[Tt][Ee][Xx][Tt][Aa][Rr][Ee][Aa]{WhiteSpace}*[^\<\>])*)
SGMLProcessingStart=("")
ASPProcessingStart=("<%")
ASPProcessingEnd=([\%]+">")
SGMLProcessingText=(([^\?]|[\?]+[^\>\?])*)
SGMLProcessing=({SGMLProcessingStart}{SGMLProcessingText}{SGMLProcessingEnd})
ASPProcessingText=(([^\%]|[\%]+[^\>\%])*)
ASPProcessing=({ASPProcessingStart}{ASPProcessingText}{ASPProcessingEnd})
Name=({Letter}({Letter}|{Digit}|[\.\-])*)
StartTag=({TagStart}{Name}(({WhiteSpace}*){Attribute})*({WhiteSpace}*){TagEnd})
ScriptStartTag=({TagStart}{NameScript}(({WhiteSpace}*){Attribute})*({WhiteSpace}*){TagEnd})
PreStartTag=({TagStart}{NamePre}(({WhiteSpace}*){Attribute})*({WhiteSpace}*){TagEnd})
TextAreaStartTag=({TagStart}{NameTextArea}(({WhiteSpace}*){Attribute})*({WhiteSpace}*){TagEnd})
EndTag=({EndTagStart}{WhiteSpace}*{Name}{WhiteSpace}*{TagEnd})
DoctypeTag=({DocTagStart}{Doctype}{DoctypeText}{TagEnd})
Comment=("--"([^\-]|([\-][^\-]))*"--")
CommentDeclaration=({DocTagStart}((({WhiteSpace}*){Comment}({WhiteSpace}*))*){TagEnd})
Word=(([^\ \r\n\f\t\<])*)
TagError=("<"[^\>]*">")
UnfinishedTag=("<"[^\>]*)
Text=(({Word}|{WhiteSpace})+)
%%