// Notes on the grammar:
// The target of this grammar is to be able to make the pre-parsing step of f-strings.
// It tries to handle any error in the parsing to generate a valid AST (and still makes 
// those errors available for processing later on).
//
// Unlike the python grammar, it can't make strings a lexical entity because 
// that's dependent upon the context -- we just want to create strings inside 
// the expression part, out of it, we should just handle it as regular text.
//
// Also, if we have unbalanced chars for some expression, we generate the AST
// going to the end of the f-string (so, this would be an open partition) -- while 
// properly generating errors.
//
// This grammar is expected to be used along with the regular Python grammar:
// after finding an f-string in the regular python grammar, it should be verified
// with this grammar (and expressions should be extracted based on this AST) and later
// on, each expression can be fed to the python grammar again (to eval_input).
//
// it's even interesting to note that f-strings are actually a disguised eval -- so, things
// such as:
//
// a = 10
// print(f'''{f"{f'{a}'}"}''')
//
// are actually possible -- this means that to check for syntax errors we may need to 
// recursively check it (although that's a corner case, so, we may skip it initially).
//
// Known differences from the 'official version':
//
// f' <text> { <expression> <optional !s, !r, or !a> <optional : format specifier> } <text>
//
// the optional versions can't have any space between them and the ending '}', whereas
// this grammar lets that happen (spaces are always skipped).
//
// i.e.: f'''{text !r }''' -- this is not valid in the regular version but is valid in this one.

options
{
    // only non-defaults are given here.
    NODE_SCOPE_HOOK       = true;   // call jjtree*NodeScope()
    NODE_FACTORY          = true;
    NODE_USES_PARSER      = true;
    STATIC                = false;  // multiple parsers
    COMMON_TOKEN_ACTION   = true;   // CommonTokenAction(Token)
    DEBUG_PARSER          = false;  // must be used to enable tracing 
    DEBUG_LOOKAHEAD       = false;
    DEBUG_TOKEN_MANAGER   = false;  // used to debug the tokens we have generating 

    USER_CHAR_STREAM      = true;
    UNICODE_INPUT         = true;
    ERROR_REPORTING       = true;  // this is NOT used to shut down errors regarding javacc org.python.pydev.parser.jython.PythonGrammar$LookaheadSuccess
                                      // because the generated information is not complete enough if we do it.
}

PARSER_BEGIN(FStringsGrammar)
package org.python.pydev.parser.grammar_fstrings;

import org.python.pydev.parser.jython.FastCharStream;
import org.python.pydev.shared_core.string.FastStringBuffer;
import org.python.pydev.parser.jython.ParseException;
import org.python.pydev.parser.jython.Token;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.AbstractFStringsGrammar;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.AbstractFStringsTokenManager;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.JJTFStringsGrammarState;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.Node;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.SimpleNode;
import org.python.pydev.parser.fastparser.grammar_fstrings_common.FStringsAST;

public final class FStringsGrammar extends AbstractFStringsGrammar
{
}

PARSER_END(FStringsGrammar)


TOKEN_MGR_DECLS:
{
    /**
     * Called right after the creation of any token.
     */
    protected final void CommonTokenAction(final Token initial) {
    }
}

// ============= Lexical actions ============= 

SKIP: { " " | "\t" | "\n" | "\r" }
TOKEN : // What we care for.
{
    < LPAREN: "(" >
|   < RPAREN: ")" >
|   < LBRACE: "{" >
|   < RBRACE: "}" >
|   < LBRACKET: "[" >
|   < RBRACKET: "]" >
|   < EXCLAMATION: "!" >
|   < COLON: ":" >
|   < QUOTE: "'" > 
|   < QUOTE2: "\"" >
|   < BACKSLASH: "\\" >
}

TOKEN : // Anything which is not important but is a visible char should be marked as regular text.
{
<TEXT : ([
    //0 - 32: space-related chars (we already skip the important ones, such as space and tab, others are invalid).
    //33 = ! (exclamation)
    //34 = " (quote)
    "#"-"&", // 35 - 38
    //39 = ' (single quote)
    // 40, 41 = () (parenthesis)
    "*"-"/", // 42 - 47
    "0"-"9", // 48 - 57
    //: = 58
    ";"-"@", // 59 - 64
    "A"-"Z",  // 65 - 90
    // 91 == [
    // 92 == \
    // 93 = ]
    "^"-"`", // 94 - 96
    "a"-"z",  // 97 - 122
    // 123 = {
    "|", // 124
    // 125 = }
    "~", // 126
    // 127 = DEL (invisible)
    "\u0080"-"\uffff" //Anything more than 128 is considered valid (unicode range)
])+ >
}

// ============= Syntatic/Semantic actions ============= 

FStringsAST f_string(): {Token t;} 
{
    (
        f_string_expr()
        |<TEXT>
        |<QUOTE>
        |<QUOTE2>
        |<EXCLAMATION>
        |<COLON>
        |<BACKSLASH>
        |<LPAREN>
        |<RPAREN>
        |<LBRACKET>
        |<RBRACKET>
        |t=<RBRACE> {addParseError(t, "Single '}' not allowed.");}
    )* <EOF>
    
    // jjtree does most things automatically for us, so, just
    // popping its final node should do the trick for a reasonable AST.
    // Note that we may need to mark the start/end of the nodes we're 
    // interested in (as jjtree.markNodeStartEnd).
    {return new FStringsAST((SimpleNode)jjtree.popNode());} 
}

void f_string_expr(): {Token start, end, bStart, bEnd;boolean empty=true;SimpleNode bText;}
{
    start = <LBRACE>
    
        (
            {bStart=getToken(1);} 
            bText=balanced_expression_text()
            {
                bEnd=token;
                empty=false;
                jjtree.markNodeStartEnd(bText, bStart, bEnd);
            }
        )?
        (type_conversion())?
        (format_spec())?
        
    try{
        end = <RBRACE>
    }catch(ParseException e){
        addParseError(e, "Unbalanced '{'");
        end = token;
    }
    
    {
        jjtree.markNodeStartEnd(jjtThis, start, end);
        if(empty){
            errorPyExprEmpty(jjtThis);
        }
    }
    
}

void type_conversion(): {Token t;boolean foundText=false;}
{
    <EXCLAMATION>
    (
        t=<TEXT> {errorIfTextIsNotASR(t);}
        {foundText=true;}
        
        // It can't really be empty, but we make this node optional to generate a nice AST
        // and then report the error later on.
    )? 
    
    {if(!foundText){errorTypeConversionEmpty(jjtThis);}}
}

void format_spec(): {}
{
    <COLON>(<TEXT>)?
}

SimpleNode balanced_expression_text(): {Token t;}
{
    (
        initial_balanced_expression_text()
    )+
    {return (SimpleNode)jjtree.peekNode();}
}

SimpleNode initial_balanced_expression_text(): {Token t;}
{
    (
        (
            <LPAREN> 
            (inner_balanced_expression_text_with_exclamation_and_colon())* 
            try{
                <RPAREN>
            }catch(ParseException e){
                addParseError(e, "Unbalanced '('");
            }
        )
        |
        (
            <LBRACE> 
            (inner_balanced_expression_text_with_exclamation_and_colon())* 
            try{
                <RBRACE>
            }catch(ParseException e){
                addParseError(e, "Unbalanced '{'");
            }
        )
        |
        (
            <LBRACKET> 
            (inner_balanced_expression_text_with_exclamation_and_colon())* 
            try{
                <RBRACKET>
            }catch(ParseException e){
                addParseError(e, "Unbalanced '['");
            }
        )
        |
        string()
        |
        string2()
        |
        // The main difference from this one to inner_balanced_expression_text_with_exclamation_and_colon
        // is that at this level an exclamation or colon shoud not be matched, whereas in the next one it
        // can be matched (so, the user can do f"{{'c':20}}").
        (<TEXT>)+
        |
        t = <BACKSLASH> {errorBackSlashInvalidInFStrings(t);}
    )
    {return (SimpleNode)jjtree.peekNode();}
}

SimpleNode inner_balanced_expression_text_with_exclamation_and_colon(): {Token t;}
{
    (
        (
            <LPAREN> 
            (inner_balanced_expression_text_with_exclamation_and_colon())* 
            try{
                <RPAREN>
            }catch(ParseException e){
                addParseError(e, "Unbalanced '('");
            }
        )
        |
        (
            <LBRACE> 
            (inner_balanced_expression_text_with_exclamation_and_colon())* 
            try{
                <RBRACE>
            }catch(ParseException e){
                addParseError(e, "Unbalanced '{'");
            }
        )
        |
        (
            <LBRACKET> 
            (inner_balanced_expression_text_with_exclamation_and_colon())* 
            try{
                <RBRACKET>
            }catch(ParseException e){
                addParseError(e, "Unbalanced '['");
            }
        )
        |
        string()
        |
        string2()
        |
        (
        <TEXT>
        |<EXCLAMATION>
        |<COLON>
        )+
        |
        t = <BACKSLASH> {errorBackSlashInvalidInFStrings(t);}
    )
    {return (SimpleNode)jjtree.peekNode();}
}


void string(): {Token t;}
{
    <QUOTE> 
    (
        <TEXT>
        |<LPAREN>
        |<RPAREN>
        |<LBRACE>
        |<RBRACE>
        |<LBRACKET>
        |<RBRACKET>
        |<QUOTE2>
        |<EXCLAMATION>
        |<COLON>
        |t = <BACKSLASH> {errorBackSlashInvalidInFStrings(t);}
    )* 
    try{
        <QUOTE>
    }catch(ParseException e){
        addParseError(e, "Unbalanced \"'\"");
    }
}

void string2(): {Token t;}
{
    <QUOTE2> 
    (
        <TEXT>
        |<LPAREN>
        |<RPAREN>
        |<LBRACE>
        |<RBRACE>
        |<LBRACKET>
        |<RBRACKET>
        |<QUOTE>
        |<EXCLAMATION>
        |<COLON>
        |t = <BACKSLASH> {errorBackSlashInvalidInFStrings(t);}
    )* 
    try{
        <QUOTE2>
    }catch(ParseException e){
        addParseError(e, "Unbalanced '\"'");
    }
}

