From 9b67ffaf74069101fa9ca29b2023dbb8cd38cff5 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Fri, 10 Jan 2020 14:35:04 +0100 Subject: [PATCH] COMP: update lemon to 2019-12-19 version --- wmake/etc/lempar.c | 15 +++-- wmake/etc/lempar.c.patch | 20 +++--- wmake/src/lemon.c | 136 ++++++++++++++++++++++++++++++++------- wmake/src/lemon.c.patch | 38 +++++------ 4 files changed, 152 insertions(+), 57 deletions(-) diff --git a/wmake/etc/lempar.c b/wmake/etc/lempar.c index 49efd2f759..819e0f61fd 100644 --- a/wmake/etc/lempar.c +++ b/wmake/etc/lempar.c @@ -347,8 +347,8 @@ void ParseInit(void *yypRawParser ParseCTX_PDECL){ #if YYSTACKDEPTH>0 yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #endif -%namespace_end } +%namespace_end #ifndef Parse_ENGINEALWAYSONSTACK /* @@ -739,12 +739,15 @@ static YYACTIONTYPE yy_reduce( if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ yysize = yyRuleInfoNRhs[yyruleno]; if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + yyruleno, yyRuleName[yyruleno], + yyruleno0 +@@ -345,6 +348,7 @@ yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; #endif -+%namespace_end } ++%namespace_end #ifndef Parse_ENGINEALWAYSONSTACK + /* @@ -359,6 +363,7 @@ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. @@ -107,7 +107,7 @@ #endif /* -@@ -893,6 +907,7 @@ +@@ -891,6 +905,7 @@ ** Outputs: ** None. */ @@ -115,7 +115,7 @@ void Parse( void *yyp, /* The parser */ int yymajor, /* The major token code number */ -@@ -1062,11 +1077,13 @@ +@@ -1060,11 +1075,13 @@ #endif return; } @@ -128,9 +128,9 @@ +%namespace_begin int ParseFallback(int iToken){ #ifdef YYFALLBACK - if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ -@@ -1077,3 +1094,4 @@ - #endif + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); +@@ -1074,3 +1091,4 @@ return 0; + #endif } +%namespace_end diff --git a/wmake/src/lemon.c b/wmake/src/lemon.c index f6c9f051e6..b73dba1f70 100644 --- a/wmake/src/lemon.c +++ b/wmake/src/lemon.c @@ -1,7 +1,7 @@ /* - * Part of check-in [https://www.sqlite.org/src/info/980be1730dc1239c] + * Part of check-in [https://www.sqlite.org/src/info/fccfb8a9ed3c1df9] * artifact: - * https://www.sqlite.org/src/raw/tool/lemon.c?name=61d5f0af1eff8f754b75ddca668c9897fd30759e389bfffb42ce9e4d38fd4746 + * https://www.sqlite.org/src/raw/tool/lemon.c?name=a361b85fa230560b783006ac002a6a8bad214c3b9d7fa48980aecc2b691ddcad */ /* ** This file contains all sources (including headers) to the LEMON @@ -223,7 +223,7 @@ void Plink_delete(struct plink *); /********** From the file "report.h" *************************************/ void Reprint(struct lemon *); void ReportOutput(struct lemon *); -void ReportTable(struct lemon *, int); +void ReportTable(struct lemon *, int, int); void ReportHeader(struct lemon *); void CompressTables(struct lemon *); void ResortStates(struct lemon *); @@ -297,13 +297,15 @@ struct rule { const char *code; /* The code executed when this rule is reduced */ const char *codePrefix; /* Setup code before code[] above */ const char *codeSuffix; /* Breakdown code after code[] above */ - int noCode; /* True if this rule has no associated C code */ - int codeEmitted; /* True if the code has been emitted already */ struct symbol *precsym; /* Precedence symbol for this rule */ int index; /* An index number for this rule */ int iRule; /* Rule number as used in the generated tables */ + Boolean noCode; /* True if this rule has no associated C code */ + Boolean codeEmitted; /* True if the code has been emitted already */ Boolean canReduce; /* True if this rule is ever reduced */ Boolean doesReduce; /* Reduce actions occur after optimization */ + Boolean neverReduce; /* Reduce is theoretically possible, but prevented + ** by actions or other outside implementation */ struct rule *nextlhs; /* Next rule with the same LHS */ struct rule *next; /* Next rule in the global list */ }; @@ -390,6 +392,7 @@ struct lemon { int nstate; /* Number of states */ int nxstate; /* nstate with tail degenerate states removed */ int nrule; /* Number of rules */ + int nruleWithAction; /* Number of rules with actions */ int nsymbol; /* Number of terminal and nonterminal symbols */ int nterminal; /* Number of terminal symbols */ int minShiftReduce; /* Minimum shift-reduce action value */ @@ -913,9 +916,9 @@ void FindStates(struct lemon *lemp) sp = Symbol_find(lemp->start); if( sp==0 ){ ErrorMsg(lemp->filename,0, -"The specified start symbol \"%s\" is not \ -in a nonterminal of the grammar. \"%s\" will be used as the start \ -symbol instead.",lemp->start,lemp->startRule->lhs->name); + "The specified start symbol \"%s\" is not " + "in a nonterminal of the grammar. \"%s\" will be used as the start " + "symbol instead.",lemp->start,lemp->startRule->lhs->name); lemp->errorcnt++; sp = lemp->startRule->lhs; } @@ -931,9 +934,9 @@ symbol instead.",lemp->start,lemp->startRule->lhs->name); for(i=0; inrhs; i++){ if( rp->rhs[i]==sp ){ /* FIX ME: Deal with multiterminals */ ErrorMsg(lemp->filename,0, -"The start symbol \"%s\" occurs on the \ -right-hand side of a rule. This will result in a parser which \ -does not work properly.",sp->name); + "The start symbol \"%s\" occurs on the " + "right-hand side of a rule. This will result in a parser which " + "does not work properly.",sp->name); lemp->errorcnt++; } } @@ -1655,6 +1658,7 @@ int main(int argc, char **argv) static int mhflag = 0; static int nolinenosflag = 0; static int noResort = 0; + static int sqlFlag = 0; static struct s_options options[] = { {OPT_FLAG, "b", (char*)&basisflag, "Print only the basis in report."}, @@ -1674,6 +1678,8 @@ int main(int argc, char **argv) {OPT_FLAG, "r", (char*)&noResort, "Do not sort or renumber states"}, {OPT_FLAG, "s", (char*)&statistics, "Print parser stats to standard output."}, + {OPT_FLAG, "S", (char*)&sqlFlag, + "Generate the *.sql file describing the parser tables."}, {OPT_FLAG, "x", (char*)&version, "Print the version number."}, {OPT_FSTR, "T", (char*)handle_T_option, "Specify a template file."}, {OPT_FSTR, "W", 0, "Ignored. (Placeholder for '-W' compiler options.)"}, @@ -1735,6 +1741,7 @@ int main(int argc, char **argv) for(i=0, rp=lem.rule; rp; rp=rp->next){ rp->iRule = rp->code ? i++ : -1; } + lem.nruleWithAction = i; for(rp=lem.rule; rp; rp=rp->next){ if( rp->iRule<0 ) rp->iRule = i++; } @@ -1782,7 +1789,7 @@ int main(int argc, char **argv) if( !quiet ) ReportOutput(&lem); /* Generate the source code for the parser */ - ReportTable(&lem, mhflag); + ReportTable(&lem, mhflag, sqlFlag); /* Produce a header file for use by the scanner. (This step is ** omitted if the "-m" option is used because makeheaders will @@ -2291,14 +2298,16 @@ static void parseonetoken(struct pstate *psp) }else if( x[0]=='{' ){ if( psp->prevrule==0 ){ ErrorMsg(psp->filename,psp->tokenlineno, -"There is no prior rule upon which to attach the code \ -fragment which begins on this line."); + "There is no prior rule upon which to attach the code " + "fragment which begins on this line."); psp->errorcnt++; }else if( psp->prevrule->code!=0 ){ ErrorMsg(psp->filename,psp->tokenlineno, -"Code fragment beginning on this line is not the first \ -to follow the previous rule."); + "Code fragment beginning on this line is not the first " + "to follow the previous rule."); psp->errorcnt++; + }else if( strcmp(x, "{NEVER-REDUCE")==0 ){ + psp->prevrule->neverReduce = 1; }else{ psp->prevrule->line = psp->tokenlineno; psp->prevrule->code = &x[1]; @@ -2324,8 +2333,8 @@ to follow the previous rule."); psp->errorcnt++; }else if( psp->prevrule->precsym!=0 ){ ErrorMsg(psp->filename,psp->tokenlineno, -"Precedence mark on this line is not the first \ -to follow the previous rule."); + "Precedence mark on this line is not the first " + "to follow the previous rule."); psp->errorcnt++; }else{ psp->prevrule->precsym = Symbol_new(x); @@ -2931,7 +2940,8 @@ void Parse(struct lemon *gp) } if( c==0 ){ ErrorMsg(ps.filename,startline, -"String starting on this line is not terminated before the end of the file."); + "String starting on this line is not terminated before " + "the end of the file."); ps.errorcnt++; nextcp = cp; }else{ @@ -2970,7 +2980,8 @@ void Parse(struct lemon *gp) } if( c==0 ){ ErrorMsg(ps.filename,ps.tokenlineno, -"C code starting on this line is not terminated before the end of the file."); + "C code starting on this line is not terminated before " + "the end of the file."); ps.errorcnt++; nextcp = cp; }else{ @@ -4233,9 +4244,10 @@ static void writeRuleText(FILE *out, struct rule *rp){ /* Generate C source code for the parser */ void ReportTable( struct lemon *lemp, - int mhflag /* Output in makeheaders format if true */ + int mhflag, /* Output in makeheaders format if true */ + int sqlFlag /* Generate the *.sql file too */ ){ - FILE *out, *in; + FILE *out, *in, *sql; char line[LINESIZE]; int lineno; struct state *stp; @@ -4266,6 +4278,78 @@ void ReportTable( fclose(in); return; } + if( sqlFlag==0 ){ + sql = 0; + }else{ + sql = file_open(lemp, ".sql", "wb"); + if( sql==0 ){ + fclose(in); + fclose(out); + return; + } + fprintf(sql, + "BEGIN;\n" + "CREATE TABLE symbol(\n" + " id INTEGER PRIMARY KEY,\n" + " name TEXT NOT NULL,\n" + " isTerminal BOOLEAN NOT NULL,\n" + " fallback INTEGER REFERENCES symbol" + " DEFERRABLE INITIALLY DEFERRED\n" + ");\n" + ); + for(i=0; insymbol; i++){ + fprintf(sql, + "INSERT INTO symbol(id,name,isTerminal,fallback)" + "VALUES(%d,'%s',%s", + i, lemp->symbols[i]->name, + interminal ? "TRUE" : "FALSE" + ); + if( lemp->symbols[i]->fallback ){ + fprintf(sql, ",%d);\n", lemp->symbols[i]->fallback->index); + }else{ + fprintf(sql, ",NULL);\n"); + } + } + fprintf(sql, + "CREATE TABLE rule(\n" + " ruleid INTEGER PRIMARY KEY,\n" + " lhs INTEGER REFERENCES symbol(id),\n" + " txt TEXT\n" + ");\n" + "CREATE TABLE rulerhs(\n" + " ruleid INTEGER REFERENCES rule(ruleid),\n" + " pos INTEGER,\n" + " sym INTEGER REFERENCES symbol(id)\n" + ");\n" + ); + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + assert( i==rp->iRule ); + fprintf(sql, + "INSERT INTO rule(ruleid,lhs,txt)VALUES(%d,%d,'", + rp->iRule, rp->lhs->index + ); + writeRuleText(sql, rp); + fprintf(sql,"');\n"); + for(j=0; jnrhs; j++){ + struct symbol *sp = rp->rhs[j]; + if( sp->type!=MULTITERMINAL ){ + fprintf(sql, + "INSERT INTO rulerhs(ruleid,pos,sym)VALUES(%d,%d,%d);\n", + i,j,sp->index + ); + }else{ + int k; + for(k=0; knsubsym; k++){ + fprintf(sql, + "INSERT INTO rulerhs(ruleid,pos,sym)VALUES(%d,%d,%d);\n", + i,j,sp->subsym[k]->index + ); + } + } + } + } + fprintf(sql, "COMMIT;\n"); + } lineno = 1; tplt_xfer(lemp->name,lemp->namesp,in,out,&lineno); @@ -4441,6 +4525,8 @@ void ReportTable( ** been computed */ fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; + fprintf(out,"#define YYNRULE_WITH_ACTION %d\n",lemp->nruleWithAction); + lineno++; fprintf(out,"#define YYNTOKEN %d\n",lemp->nterminal); lineno++; fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; i = lemp->minShiftReduce; @@ -4760,7 +4846,10 @@ void ReportTable( assert( rp->noCode ); fprintf(out," /* (%d) ", rp->iRule); writeRuleText(out, rp); - if( rp->doesReduce ){ + if( rp->neverReduce ){ + fprintf(out, " (NEVER REDUCES) */ assert(yyruleno!=%d);\n", + rp->iRule); lineno++; + }else if( rp->doesReduce ){ fprintf(out, " */ yytestcase(yyruleno==%d);\n", rp->iRule); lineno++; }else{ fprintf(out, " (OPTIMIZED OUT) */ assert(yyruleno!=%d);\n", @@ -4788,6 +4877,7 @@ void ReportTable( acttab_free(pActtab); fclose(in); fclose(out); + if( sql ) fclose(sql); return; } diff --git a/wmake/src/lemon.c.patch b/wmake/src/lemon.c.patch index ce78936a8a..3a76c48d87 100644 --- a/wmake/src/lemon.c.patch +++ b/wmake/src/lemon.c.patch @@ -1,6 +1,6 @@ ---- lemon.c.orig 2019-09-26 14:00:01.572868618 +0200 -+++ lemon.c 2019-09-27 09:05:08.182250296 +0200 -@@ -404,6 +404,7 @@ +--- lemon.c.orig 2020-01-10 14:08:19.181711107 +0100 ++++ lemon.c 2020-01-10 14:12:00.621402133 +0100 +@@ -406,6 +406,7 @@ struct symbol *errsym; /* The error symbol */ struct symbol *wildcard; /* Token that matches anything */ char *name; /* Name of the generated parser */ @@ -8,7 +8,7 @@ char *arg; /* Declaration of the 3th argument to parser */ char *ctx; /* Declaration of 2nd argument to constructor */ char *tokentype; /* Type of terminal symbols in the parser stack */ -@@ -1557,6 +1558,23 @@ +@@ -1559,6 +1560,23 @@ lemon_strcpy(outputDir, z); } @@ -32,7 +32,7 @@ static char *user_templatename = NULL; static void handle_T_option(char *z){ user_templatename = (char *) malloc( lemonStrlen(z)+1 ); -@@ -1644,6 +1662,7 @@ +@@ -1647,6 +1665,7 @@ {OPT_FLAG, "c", (char*)&compress, "Don't compress the action table."}, {OPT_FSTR, "d", (char*)&handle_d_option, "Output directory. Default '.'"}, {OPT_FSTR, "D", (char*)handle_D_option, "Define an %ifdef macro."}, @@ -40,7 +40,7 @@ {OPT_FSTR, "f", 0, "Ignored. (Placeholder for -f compiler options.)"}, {OPT_FLAG, "g", (char*)&rpflag, "Print grammar without actions."}, {OPT_FSTR, "I", 0, "Ignored. (Placeholder for '-I' compiler options.)"}, -@@ -2482,6 +2501,9 @@ +@@ -2490,6 +2509,9 @@ if( strcmp(x,"name")==0 ){ psp->declargslot = &(psp->gp->name); psp->insertLineMacro = 0; @@ -50,7 +50,7 @@ }else if( strcmp(x,"include")==0 ){ psp->declargslot = &(psp->gp->include); }else if( strcmp(x,"code")==0 ){ -@@ -3467,20 +3489,83 @@ +@@ -3477,20 +3499,83 @@ #define LINESIZE 1000 /* The next cluster of routines are for reading the template file ** and writing the results to the generated parser */ @@ -135,7 +135,7 @@ if( name ){ for(i=0; line[i]; i++){ if( line[i]=='P' && strncmp(&line[i],"Parse",5)==0 -@@ -4175,13 +4260,14 @@ +@@ -4187,7 +4272,8 @@ in = tplt_open(lemp); if( in==0 ) return; @@ -145,6 +145,8 @@ if( out==0 ){ fclose(in); return; +@@ -4265,7 +4351,7 @@ + fprintf(sql, "COMMIT;\n"); } lineno = 1; - tplt_xfer(lemp->name,in,out,&lineno); @@ -152,7 +154,7 @@ /* Generate the include code, if any */ tplt_print(out,lemp,lemp->include,&lineno); -@@ -4190,7 +4276,7 @@ +@@ -4274,7 +4360,7 @@ fprintf(out,"#include \"%s\"\n", incName); lineno++; free(incName); } @@ -161,7 +163,7 @@ /* Generate #defines for all tokens */ if( mhflag ){ -@@ -4204,7 +4290,7 @@ +@@ -4288,7 +4374,7 @@ } fprintf(out,"#endif\n"); lineno++; } @@ -170,7 +172,7 @@ /* Generate the defines */ fprintf(out,"#define YYCODETYPE %s\n", -@@ -4367,7 +4453,7 @@ +@@ -4453,7 +4539,7 @@ fprintf(out,"#define YY_MIN_REDUCE %d\n", lemp->minReduce); lineno++; i = lemp->minReduce + lemp->nrule; fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; @@ -179,7 +181,7 @@ /* Now output the action table and its associates: ** -@@ -4489,7 +4575,7 @@ +@@ -4591,7 +4677,7 @@ } } fprintf(out, "};\n"); lineno++; @@ -188,7 +190,7 @@ /* Generate the table of fallback tokens. */ -@@ -4508,7 +4594,7 @@ +@@ -4612,7 +4698,7 @@ lineno++; } } @@ -197,7 +199,7 @@ /* Generate a table containing the symbolic name of every symbol */ -@@ -4516,7 +4602,7 @@ +@@ -4620,7 +4706,7 @@ lemon_sprintf(line,"\"%s\",",lemp->symbols[i]->name); fprintf(out," /* %4d */ \"%s\",\n",i, lemp->symbols[i]->name); lineno++; } @@ -206,7 +208,7 @@ /* Generate a table containing a text string that describes every ** rule in the rule set of the grammar. This information is used -@@ -4528,7 +4614,7 @@ +@@ -4632,7 +4718,7 @@ writeRuleText(out, rp); fprintf(out,"\",\n"); lineno++; } @@ -215,7 +217,7 @@ /* Generate code which executes every time a symbol is popped from ** the stack while processing errors or while destroying the parser. -@@ -4591,11 +4677,11 @@ +@@ -4695,11 +4781,11 @@ emit_destructor_code(out,lemp->symbols[i],lemp,&lineno); fprintf(out," break;\n"); lineno++; } @@ -229,7 +231,7 @@ /* Generate the tables of rule information. yyRuleInfoLhs[] and ** yyRuleInfoNRhs[]. -@@ -4608,13 +4694,13 @@ +@@ -4712,13 +4798,13 @@ rule_print(out, rp); fprintf(out," */\n"); lineno++; } @@ -245,7 +247,7 @@ /* Generate code which execution during each REDUCE action */ i = 0; -@@ -4664,19 +4750,19 @@ +@@ -4771,19 +4857,19 @@ } } fprintf(out," break;\n"); lineno++;