From 9211fb6e2de425ff9b4669c6c6f6fabea7c3f639 Mon Sep 17 00:00:00 2001 From: John Naylor Date: Sun, 30 Sep 2018 13:08:25 +0700 Subject: [PATCH v1 3/4] Use start condition scopes in the core, psql, and ECPG scanners. This makes it easier to diff the core scanner with the ECPG scanner. --- src/backend/parser/scan.l | 12 ++- src/fe_utils/psqlscan.l | 8 +- src/interfaces/ecpg/preproc/pgc.l | 133 +++++++++++++++++++++--------- 3 files changed, 107 insertions(+), 46 deletions(-) diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l index a2454732a1..f20a6e58fe 100644 --- a/src/backend/parser/scan.l +++ b/src/backend/parser/scan.l @@ -430,19 +430,23 @@ other . (yyextra->xcdepth)--; } -{xcinside} { +{ +{xcinside} { /* ignore */ } -{op_chars} { +{op_chars} { /* ignore */ } -\*+ { +\*+ { /* ignore */ } -<> { yyerror("unterminated /* comment"); } +<> { + yyerror("unterminated /* comment"); + } +} /* */ {xbstart} { /* Binary bit type. diff --git a/src/fe_utils/psqlscan.l b/src/fe_utils/psqlscan.l index 25253b54ea..eb239685d5 100644 --- a/src/fe_utils/psqlscan.l +++ b/src/fe_utils/psqlscan.l @@ -417,17 +417,19 @@ other . ECHO; } -{xcinside} { +{ +{xcinside} { ECHO; } -{op_chars} { +{op_chars} { ECHO; } -\*+ { +\*+ { ECHO; } +} /* */ {xbstart} { BEGIN(xb); diff --git a/src/interfaces/ecpg/preproc/pgc.l b/src/interfaces/ecpg/preproc/pgc.l index 421c62339b..16b1c1bb08 100644 --- a/src/interfaces/ecpg/preproc/pgc.l +++ b/src/interfaces/ecpg/preproc/pgc.l @@ -395,11 +395,11 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ token_start = NULL; %} -{whitespace} { +{ +{whitespace} { /* ignore */ } - -{xcstart} { +{xcstart} { token_start = yytext; state_before = YYSTATE; xcdepth = 0; @@ -408,7 +408,9 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ yyless(2); fputs("/*", yyout); } -{xcstart} { +} /* */ + +{xcstart} { token_start = yytext; state_before = YYSTATE; xcdepth = 0; @@ -443,24 +445,30 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ token_start = NULL; } -{xcinside} { +{ +{xcinside} { ECHO; } -{op_chars} { +{op_chars} { ECHO; } -\*+ { +\*+ { ECHO; } +<> { + mmfatal(PARSE_ERROR, "unterminated /* comment"); + } +} /* */ -<> { mmfatal(PARSE_ERROR, "unterminated /* comment"); } - -{xbstart} { +{ +{xbstart} { token_start = yytext; BEGIN(xb); startlit(); addlitchar('b'); } +} /* */ + {quotestop} | {quotefail} { yyless(1); @@ -502,7 +510,9 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ BEGIN(xqc); startlit(); } -{xnstart} { + +{ +{xnstart} { /* National character. * Transfer it as-is to the backend. */ @@ -511,25 +521,27 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ BEGIN(xn); startlit(); } -{xqstart} { +{xqstart} { token_start = yytext; state_before = YYSTATE; BEGIN(xq); startlit(); } -{xestart} { +{xestart} { token_start = yytext; state_before = YYSTATE; BEGIN(xe); startlit(); } -{xusstart} { +{xusstart} { token_start = yytext; state_before = YYSTATE; BEGIN(xus); startlit(); addlit(yytext, yyleng); } +} /* */ + {quotestop} | {quotefail} { yyless(1); @@ -584,13 +596,15 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ addlitchar(yytext[0]); } <> { mmfatal(PARSE_ERROR, "unterminated quoted string"); } -{dolqfailed} { + +{ +{dolqfailed} { /* throw back all but the initial "$" */ yyless(1); /* and treat it as {other} */ return yytext[0]; } -{dolqdelim} { +{dolqdelim} { token_start = yytext; if (dolqstart) free(dolqstart); @@ -599,6 +613,8 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ startlit(); addlit(yytext, yyleng); } +} /* */ + {dolqdelim} { if (strcmp(yytext, dolqstart) == 0) { @@ -631,17 +647,21 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ addlitchar(yytext[0]); } <> { base_yyerror("unterminated dollar-quoted string"); } -{xdstart} { + +{ +{xdstart} { state_before = YYSTATE; BEGIN(xd); startlit(); } -{xuistart} { +{xuistart} { state_before = YYSTATE; BEGIN(xui); startlit(); addlit(yytext, yyleng); } +} /* */ + {xdstop} { BEGIN(state_before); if (literallen == 0) @@ -677,15 +697,43 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ startlit(); } {xdcinside} { addlit(yytext, yyleng); } -{typecast} { return TYPECAST; } -{dot_dot} { return DOT_DOT; } -{colon_equals} { return COLON_EQUALS; } -{equals_greater} { return EQUALS_GREATER; } -{less_equals} { return LESS_EQUALS; } -{greater_equals} { return GREATER_EQUALS; } -{less_greater} { return NOT_EQUALS; } -{not_equals} { return NOT_EQUALS; } -{informix_special} { + +{ +{typecast} { + return TYPECAST; + } + +{dot_dot} { + return DOT_DOT; + } + +{colon_equals} { + return COLON_EQUALS; + } + +{equals_greater} { + return EQUALS_GREATER; + } + +{less_equals} { + return LESS_EQUALS; + } + +{greater_equals} { + return GREATER_EQUALS; + } + +{less_greater} { + /* We accept both "<>" and "!=" as meaning NOT_EQUALS */ + return NOT_EQUALS; + } + +{not_equals} { + /* We accept both "<>" and "!=" as meaning NOT_EQUALS */ + return NOT_EQUALS; + } + +{informix_special} { /* are we simulating Informix? */ if (INFORMIX_MODE) { @@ -694,7 +742,7 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ else return yytext[0]; } -{self} { +{self} { /* * We may find a ';' inside a structure * definition in a TYPE or VAR statement. @@ -704,7 +752,7 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ BEGIN(C); return yytext[0]; } -{operator} { +{operator} { /* * Check for embedded slash-star or dash-dash; those * are comment starts, so operator must stop there. @@ -800,15 +848,18 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ base_yylval.str = mm_strdup(yytext); return Op; } -{param} { +{param} { base_yylval.ival = atol(yytext+1); return PARAM; } -{ip} { +{ip} { base_yylval.str = mm_strdup(yytext); return IP; } -{integer} { +} /* */ + +{ +{integer} { int val; char* endptr; @@ -823,29 +874,32 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ base_yylval.ival = val; return ICONST; } -{decimal} { +{decimal} { base_yylval.str = mm_strdup(yytext); return FCONST; } -{real} { +{real} { base_yylval.str = mm_strdup(yytext); return FCONST; } -{realfail1} { +} /* */ + +{ +{realfail1} { yyless(yyleng-1); base_yylval.str = mm_strdup(yytext); return FCONST; } -{realfail2} { +{realfail2} { yyless(yyleng-2); base_yylval.str = mm_strdup(yytext); return FCONST; } -:{identifier}((("->"|\.){identifier})|(\[{array}\]))* { +:{identifier}((("->"|\.){identifier})|(\[{array}\]))* { base_yylval.str = mm_strdup(yytext+1); return CVARIABLE; } -{identifier} { +{identifier} { const ScanKeyword *keyword; if (!isdefine()) @@ -873,9 +927,10 @@ cppline {space}*#([^i][A-Za-z]*|{if}|{ifdef}|{ifndef}|{import})((\/\*[^*/]*\*+ } } -{other} { +{other} { return yytext[0]; } +} /* */ {exec_sql} { BEGIN(SQL); return SQL_START; } {informix_special} { -- 2.17.1