<DEFAULT> SKIP : {
" "
| "\t"
| "\f"
}
|
<DEFAULT> SKIP : {
<"#" (~["\n","\r"])*>
}
|
<DEFAULT> TOKEN : {
<EOL: "\r\n" | "\n" | "\r">
| <NUL: "\u0000">
}
|
<DEFAULT> TOKEN [IGNORE_CASE] : {
<USER_AGENT: "user-agent">
| <ALLOW: "allow">
| <DISALLOW: "disallow">
| <SITE_MAP: "sitemap">
| <OTHER_FIELD: (~[":","\n","\r","#","\u0000"])+>
}
|
<DEFAULT> TOKEN : {
<DELIMITER: [":"] (" " | "\t" | "\f")*> : IN_TEXT_VALUE
}
|
<IN_TEXT_VALUE> TOKEN : {
<TEXT_VALUE: (~["\n","\r","#","\u0000"])*> : DEFAULT
}
|
| parse | ::= | ( <EOL> )* ( nonGroupLine | otherLine )* ( entry )* <EOF> |
| entry | ::= | ( startGroupLine )+ ( groupMemberLine | nonGroupLine | otherLine )* |
| startGroupLine | ::= | <USER_AGENT> <DELIMITER> eol |
| groupMemberLine | ::= | ( <ALLOW> <DELIMITER> | <DISALLOW> <DELIMITER> ) eol |
| nonGroupLine | ::= | <SITE_MAP> <DELIMITER> eol |
| otherLine | ::= | <OTHER_FIELD> <DELIMITER> textValue eol |
| textValue | ::= | ( <TEXT_VALUE> | <EOF> ) |
| eol | ::= | ( ( <EOL> )+ | <EOF> ) |