From 2f0413032f662b28e4db13bb779d919586125b18 Mon Sep 17 00:00:00 2001 From: selfmadekirich <tosha.kirichenko.2001@mail.ru> Date: Sun, 25 Sep 2022 19:09:33 +0300 Subject: [PATCH] lab3 done --- Module3/LexerAddon.cs | 38 ++- Module3/ScannerHelper.cs | 2 +- Module3/SimpleLex.cs | 492 ++++++++++++++++++++++-------------- Module3/SimpleLex.lex | 41 +++ Module3/a.txt | 2 +- Module3/mymain.cs | 19 ++ TestGeneratedLexer/Tests.cs | 23 +- 7 files changed, 417 insertions(+), 200 deletions(-) diff --git a/Module3/LexerAddon.cs b/Module3/LexerAddon.cs index c246779..7b8eaa5 100644 --- a/Module3/LexerAddon.cs +++ b/Module3/LexerAddon.cs @@ -19,7 +19,7 @@ namespace GeneratedLexer public int sumInt = 0; public double sumDouble = 0.0; public List<string> idsInComment = new List<string>(); - + public List<string> stringsInSingleApost = new List<string>(); public LexerAddon(string programText) { @@ -41,14 +41,50 @@ namespace GeneratedLexer System.Threading.Thread.CurrentThread.CurrentCulture = new System.Globalization.CultureInfo("en-US"); int tok = 0; + do { tok = myScanner.yylex(); + if(tok == (int)Tok.COMMENT || tok == (int)Tok.LONGCOMMENT) + { + continue; + } + + if(tok == (int)Tok.IDCOMMENT) + { + idsInComment.Add(myScanner.yytext); + } + + if (tok == (int)Tok.INUM) + { + sumInt += myScanner.LexValueInt; + } + + if (tok == (int)Tok.RNUM) + { + sumDouble += myScanner.LexValueDouble; + } + + if(tok == (int)Tok.STRINGAP) + { + stringsInSingleApost.Add(myScanner.yytext); + } + + if (tok == (int)Tok.ID) + { + idCount++; + minIdLength = Math.Min(minIdLength, myScanner.yyleng); + maxIdLength = Math.Max(maxIdLength, myScanner.yyleng); + avgIdLength += myScanner.yyleng; + } + if (tok == (int)Tok.EOF) { break; } } while (true); + + avgIdLength = idCount > 0 ? avgIdLength / idCount : 0; } } } diff --git a/Module3/ScannerHelper.cs b/Module3/ScannerHelper.cs index f56ce42..0324f37 100644 --- a/Module3/ScannerHelper.cs +++ b/Module3/ScannerHelper.cs @@ -1,4 +1,4 @@ п»їnamespace ScannerHelper { - public enum Tok { EOF = 0, ID, INUM, RNUM, COLON, SEMICOLON, ASSIGN, BEGIN, END, CYCLE, COMMENT, STRINGAP, LONGCOMMENT }; + public enum Tok { EOF = 0, ID, INUM, RNUM, COLON, SEMICOLON, ASSIGN, BEGIN, END, CYCLE, COMMENT, STRINGAP, LONGCOMMENT,IDCOMMENT,SPECCOMMENT }; } \ No newline at end of file diff --git a/Module3/SimpleLex.cs b/Module3/SimpleLex.cs index d8f9df8..e7aafd9 100644 --- a/Module3/SimpleLex.cs +++ b/Module3/SimpleLex.cs @@ -1,14 +1,10 @@ // // This CSharp output file generated by Gardens Point LEX -// Gardens Point LEX (GPLEX) is Copyright (c) John Gough, QUT 2006-2014. -// Output produced by GPLEX is the property of the user. -// See accompanying file GPLEXcopyright.rtf. -// -// GPLEX Version: 1.2.2 -// Machine: MAINHOMEPC2 -// DateTime: 30.09.2018 17:57:38 -// UserName: someone -// GPLEX input file <SimpleLex.lex - 30.09.2018 17:56:56> +// Version: 1.1.3.301 +// Machine: LAPTOP-DB9EH0OL +// DateTime: 25.09.2022 19:07:28 +// UserName: Anton +// GPLEX input file <SimpleLex.lex> // GPLEX frame file <embedded resource> // // Option settings: noParser, minimize @@ -16,8 +12,8 @@ // // -// Revised backup code -// Version 1.2.1 of 24-June-2013 +// Experimental embedded frame +// Version 1.1.3 of 18-April-2010 // // #define BACKUP @@ -126,11 +122,12 @@ namespace SimpleScanner enum Result {accept, noMatch, contextFound}; - const int maxAccept = 20; - const int initial = 21; + const int maxAccept = 35; + const int initial = 36; const int eofNum = 0; const int goStart = -1; const int INITIAL = 0; + const int COMMENT = 1; #region user code public int LexValueInt; @@ -166,112 +163,178 @@ public int LexValueInt; } }; - static int[] startState = new int[] {21, 0}; + static int[] startState = new int[] {36, 39, 0}; - static Table[] NxS = new Table[23] { + static Table[] NxS = new Table[40] { /* NxS[ 0] */ new Table(0, 0, 0, null), /* NxS[ 1] */ new Table(0, 0, -1, null), -/* NxS[ 2] */ new Table(46, 12, -1, new sbyte[] {22, -1, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2}), -/* NxS[ 3] */ new Table(61, 1, -1, new sbyte[] {19}), -/* NxS[ 4] */ new Table(0, 0, -1, null), -/* NxS[ 5] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 6] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 15, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 7] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 11, 5}), -/* NxS[ 8] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 9, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 9] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 10, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 10] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 11] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 12, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 12] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 13, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 13] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 14, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 14] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 15] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 16, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 16] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 17, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 17] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 18, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 18] */ new Table(48, 75, -1, new sbyte[] {5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, -1, -1, -1, -1, -1, -1, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, -1, -1, -1, -1, 5, -1, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5}), -/* NxS[ 19] */ new Table(0, 0, -1, null), -/* NxS[ 20] */ new Table(48, 10, -1, new sbyte[] {20, 20, 20, 20, 20, 20, - 20, 20, 20, 20}), -/* NxS[ 21] */ new Table(10, 113, 1, new sbyte[] {-1, 1, 1, -1, 1, 1, +/* NxS[ 2] */ new Table(39, 1, 38, new sbyte[] {25}), +/* NxS[ 3] */ new Table(47, 1, -1, new sbyte[] {24}), +/* NxS[ 4] */ new Table(46, 12, -1, new sbyte[] {37, -1, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4}), +/* NxS[ 5] */ new Table(61, 1, -1, new sbyte[] {22}), +/* NxS[ 6] */ new Table(0, 0, -1, null), +/* NxS[ 7] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 8] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 18, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 9] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 14, 7}), +/* NxS[ 10] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 12, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 11] */ new Table(0, 0, -1, null), +/* NxS[ 12] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 13, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 13] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 14] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 15, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 15] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 16, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 16] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 17, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 17] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 18] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 19, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 19] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 20, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 20] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 21, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 21] */ new Table(48, 75, -1, new sbyte[] {7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, -1, -1, -1, -1, -1, -1, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, -1, -1, -1, -1, 7, -1, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7}), +/* NxS[ 22] */ new Table(0, 0, -1, null), +/* NxS[ 23] */ new Table(48, 10, -1, new sbyte[] {23, 23, 23, 23, 23, 23, + 23, 23, 23, 23}), +/* NxS[ 24] */ new Table(10, 4, 24, new sbyte[] {-1, 24, 24, -1}), +/* NxS[ 25] */ new Table(0, 0, -1, null), +/* NxS[ 26] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 27] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 32, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 28] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 30, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 29] */ new Table(0, 0, -1, null), +/* NxS[ 30] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 31, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 31] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 32] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 26, + 26, 33, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 33] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 34, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 34] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 35, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 35] */ new Table(48, 75, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, -1, -1, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26}), +/* NxS[ 36] */ new Table(10, 114, 1, new sbyte[] {-1, 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 4, 1, 1, 1, 1, - 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 1, 1, 1, 1, 5, - 1, 5, 6, 7, 5, 8, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5}), -/* NxS[ 22] */ new Table(48, 10, -1, new sbyte[] {20, 20, 20, 20, 20, 20, - 20, 20, 20, 20}), + -1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 3, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 6, 1, 1, 1, 1, + 1, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 1, 1, 1, 1, 7, + 1, 7, 8, 9, 7, 10, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 11}), +/* NxS[ 37] */ new Table(48, 10, -1, new sbyte[] {23, 23, 23, 23, 23, 23, + 23, 23, 23, 23}), +/* NxS[ 38] */ new Table(39, 1, 38, new sbyte[] {25}), +/* NxS[ 39] */ new Table(65, 61, -1, new sbyte[] {26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, -1, -1, 26, -1, 26, 27, 26, 26, 28, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, -1, -1, 29}), }; int NextState() { @@ -356,7 +419,8 @@ int NextState() { #if !NOFILES public Scanner(Stream file) { SetSource(file); // no unicode option - } + } + #endif // !NOFILES public Scanner() { } @@ -384,7 +448,7 @@ int NextState() { if (next < 0xDC00 || next > 0xDFFF) code = ScanBuff.UnicodeReplacementChar; else - code = (0x10000 + ((code & 0x3FF) << 10) + (next & 0x3FF)); + code = (0x10000 + (code & 0x3FF << 10) + (next & 0x3FF)); } #endif cCol++; @@ -437,7 +501,9 @@ int NextState() { GetCode(); } +#if !NOFILES // ================ LineBuffer Initialization =================== + /// <summary> /// Create and initialize a LineBuff buffer object for this scanner /// </summary> @@ -451,7 +517,6 @@ int NextState() { GetCode(); } -#if !NOFILES // =============== StreamBuffer Initialization ================== /// <summary> @@ -553,12 +618,6 @@ int NextState() { } } - /// <summary> - /// Discards all but the first "n" codepoints in the recognized pattern. - /// Resets the buffer position so that only n codepoints have been consumed; - /// yytext is also re-evaluated. - /// </summary> - /// <param name="n">The number of codepoints to consume</param> [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void yyless(int n) { @@ -578,10 +637,6 @@ int NextState() { // but it does not seem possible to re-establish // the correct column counts except by going forward. // - /// <summary> - /// Removes the last "n" code points from the pattern. - /// </summary> - /// <param name="n">The number to remove</param> [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] void _yytrunc(int n) { yyless(yyleng - n); } @@ -594,23 +649,18 @@ int NextState() { // can't use (tokEPos - tokPos) because of the // possibility of surrogate pairs in the token. // - /// <summary> - /// The length of the pattern in codepoints (not the same as - /// string-length if the pattern contains any surrogate pairs). - /// </summary> [SuppressMessage("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] [SuppressMessage("Microsoft.Naming", "CA1709:IdentifiersShouldBeCasedCorrectly", MessageId = "yyleng")] [SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "yyleng")] public int yyleng { get { +#if BYTEMODE + return tokEPos - tokPos; +#else if (tokELin == tokLin) return tokECol - tokCol; - else -#if BYTEMODE - return tokEPos - tokPos; -#else - { + else { int ch; int count = 0; int save = buffer.Pos; @@ -619,7 +669,7 @@ int NextState() { ch = buffer.Read(); if (!char.IsHighSurrogate((char)ch)) count++; } while (buffer.Pos < tokEPos && ch != ScanBuff.EndOfFile); - buffer.Pos = save; + buffer.Pos = save; return count; } #endif // BYTEMODE @@ -644,55 +694,64 @@ int NextState() { // ============== The main tokenizer code ================= - int Scan() { - for (; ; ) { - int next; // next state to enter + int Scan() + { + for (; ; ) + { + int next; // next state to enter +#if BACKUP + Result rslt = Result.noMatch; +#endif // BACKUP #if LEFTANCHORS - for (;;) { + for (;;) + { // Discard characters that do not start any pattern. // Must check the left anchor condition after *every* GetCode! state = ((cCol == 0) ? anchorState[currentScOrd] : currentStart); - if ((next = NextState()) != goStart) break; // LOOP EXIT HERE... + if ((next = NextState()) != goStart) + break; // LOOP EXIT HERE... GetCode(); } #else // !LEFTANCHORS state = currentStart; - while ((next = NextState()) == goStart) { + while ((next = NextState()) == goStart) // At this point, the current character has no // transition from the current state. We discard // the "no-match" char. In traditional LEX such // characters are echoed to the console. GetCode(); - } #endif // LEFTANCHORS // At last, a valid transition ... MarkToken(); state = next; - GetCode(); + GetCode(); + + while ((next = NextState()) > eofNum) // Exit for goStart AND for eofNum #if BACKUP - bool contextSaved = false; - while ((next = NextState()) > eofNum) { // Exit for goStart AND for eofNum - if (state <= maxAccept && next > maxAccept) { // need to prepare backup data - // Store data for the *latest* accept state that was found. - SaveStateAndPos( ref ctx ); - contextSaved = true; + if (state <= maxAccept && next > maxAccept) // need to prepare backup data + { + // ctx is an object. The fields may be + // mutated by the call to Recurse2. + // On return the data in ctx is the + // *latest* accept state that was found. + + rslt = Recurse2(ref ctx, next); + if (rslt == Result.noMatch) + RestoreStateAndPos(ref ctx); + break; } - state = next; - GetCode(); - } - if (state > maxAccept && contextSaved) - RestoreStateAndPos( ref ctx ); -#else // BACKUP - while ((next = NextState()) > eofNum) { // Exit for goStart AND for eofNum - state = next; - GetCode(); - } + else #endif // BACKUP - if (state <= maxAccept) { + { + state = next; + GetCode(); + } + if (state <= maxAccept) + { MarkEnd(); #region ActionSwitch -#pragma warning disable 162, 1522 +#pragma warning disable 162 switch (state) { case eofNum: @@ -700,59 +759,111 @@ int NextState() { return (int)Tokens.EOF; break; case 1: + case 2: + case 3: LexError(); return 0; break; - case 2: + case 4: LexValueInt = int.Parse(yytext); return (int)Tok.INUM; break; - case 3: + case 5: return (int)Tok.COLON; break; - case 4: + case 6: return (int)Tok.SEMICOLON; break; - case 5: - case 6: case 7: case 8: case 9: - case 11: + case 10: case 12: - case 13: + case 14: case 15: case 16: - case 17: + case 18: + case 19: + case 20: return (int)Tok.ID; break; - case 10: + case 11: +BEGIN(COMMENT); + return (int)Tok.LONGCOMMENT; + break; + case 13: return (int)Tok.END; break; - case 14: + case 17: return (int)Tok.CYCLE; break; - case 18: + case 21: return (int)Tok.BEGIN; break; - case 19: + case 22: return (int)Tok.ASSIGN; break; - case 20: + case 23: LexValueDouble = double.Parse(yytext); return (int)Tok.RNUM; break; + case 24: +return (int)Tok.COMMENT; + break; + case 25: +return (int)Tok.STRINGAP; + break; + case 26: + case 27: + case 28: + case 30: + case 32: + case 33: + case 34: +return (int)Tok.IDCOMMENT; + break; + case 29: +BEGIN(INITIAL); + break; + case 31: +return (int)Tok.SPECCOMMENT; + break; + case 35: +return (int)Tok.SPECCOMMENT; + break; default: break; } -#pragma warning restore 162, 1522 +#pragma warning restore 162 #endregion } } } #if BACKUP - void SaveStateAndPos(ref Context ctx) { + Result Recurse2(ref Context ctx, int next) + { + // Assert: at entry "state" is an accept state AND + // NextState(state, code) != goStart AND + // NextState(state, code) is not an accept state. + // + SaveStateAndPos(ref ctx); + state = next; + GetCode(); + + while ((next = NextState()) > eofNum) + { + if (state <= maxAccept && next > maxAccept) // need to update backup data + SaveStateAndPos(ref ctx); + state = next; + if (state == eofNum) return Result.accept; + GetCode(); + } + return (state <= maxAccept ? Result.accept : Result.noMatch); + } + + void SaveStateAndPos(ref Context ctx) + { ctx.bPos = buffer.Pos; ctx.rPos = readPos; ctx.cCol = cCol; @@ -761,7 +872,8 @@ LexValueDouble = double.Parse(yytext); ctx.cChr = code; } - void RestoreStateAndPos(ref Context ctx) { + void RestoreStateAndPos(ref Context ctx) + { buffer.Pos = ctx.bPos; readPos = ctx.rPos; cCol = ctx.cCol; @@ -769,7 +881,8 @@ LexValueDouble = double.Parse(yytext); state = ctx.state; code = ctx.cChr; } -#endif // BACKUP + +#endif // BACKUP // ============= End of the tokenizer code ================ @@ -803,7 +916,7 @@ LexValueDouble = double.Parse(yytext); public void LexError() { - Console.WriteLine("({0},{1}): ГЌГҐГЁГ§ГўГҐГ±ГІГûé ñèìâîë {2}", yyline, yycol, yytext); + Console.WriteLine("({0},{1}): Неизвестный символ {2}", yyline, yycol, yytext); } public string TokToString(Tok tok) @@ -870,7 +983,6 @@ public string TokToString(Tok tok) return new LineBuffer(source); } -#if (!NOFILES) public static ScanBuff GetBuffer(Stream source) { return new BuildBuffer(source); @@ -881,8 +993,7 @@ public string TokToString(Tok tok) { return new BuildBuffer(source, fallbackCodePage); } -#endif // !BYTEMODE -#endif // !NOFILES +#endif } #region Buffer classes @@ -1005,7 +1116,7 @@ public string TokToString(Tok tok) { ix = lstart = 0; } - while (ix < numLines) + for (; ; ) { int len = line[ix].Length + 1; if (pos < lstart + len) break; @@ -1063,8 +1174,7 @@ public string TokToString(Tok tok) { cPos = value; findIndex(cPos, out cLine, out curLineStart); - // cLine should be the *next* line after curLine. - curLine = (cLine < numLines ? line[cLine++] : ""); + curLine = line[cLine]; curLineEnd = curLineStart + curLine.Length; } } @@ -1072,7 +1182,7 @@ public string TokToString(Tok tok) public override string ToString() { return "LineBuffer"; } } -#if (!NOFILES) + // ============================================================== // ===== class BuildBuff : for unicode text files ======== // ============================================================== @@ -1313,13 +1423,12 @@ public string TokToString(Tok tok) } #endif // !BYTEMODE } -#endif // !NOFILES #endregion Buffer classes // ============================================================== // ============ class CodePageHandling ============= // ============================================================== -#if (!NOFILES) + public static class CodePageHandling { public static int GetCodePage(string option) @@ -1530,7 +1639,6 @@ public string TokToString(Tok tok) #endif // !BYTEMODE #endregion -#endif // !NOFILES // End of code copied from embedded resource diff --git a/Module3/SimpleLex.lex b/Module3/SimpleLex.lex index c0434f0..9edd5dc 100644 --- a/Module3/SimpleLex.lex +++ b/Module3/SimpleLex.lex @@ -7,6 +7,8 @@ AlphaDigit {Alpha}|{Digit} INTNUM {Digit}+ REALNUM {INTNUM}\.{INTNUM} ID {Alpha}{AlphaDigit}* +DotChr [^\r\n] +OneLineCmnt \/\/{DotChr}* // Здесь можно делать описания типов, переменных и методов - они попадают в класс Scanner %{ @@ -14,6 +16,8 @@ ID {Alpha}{AlphaDigit}* public double LexValueDouble; %} +%x COMMENT + %% {INTNUM} { LexValueInt = int.Parse(yytext); @@ -53,11 +57,48 @@ cycle { return (int)Tok.SEMICOLON; } +"{" { + BEGIN(COMMENT); + return (int)Tok.LONGCOMMENT; +} + + [^ \r\n] { LexError(); return 0; // конец разбора } +{OneLineCmnt} { + return (int)Tok.COMMENT; +} + +\'[^']*\' { + return (int)Tok.STRINGAP; +} + + +<COMMENT> "}" { + // переход в состояние INITIAL + BEGIN(INITIAL); +} + +<COMMENT> begin { + // обрабатывается ID внутри комментария + return (int)Tok.SPECCOMMENT; +} + +<COMMENT> end { + // обрабатывается ID внутри комментария + return (int)Tok.SPECCOMMENT; +} + + +<COMMENT>{ID} { + // обрабатывается ID внутри комментария + return (int)Tok.IDCOMMENT; +} + + %% // Здесь можно делать описания переменных и методов - они тоже попадают в класс Scanner diff --git a/Module3/a.txt b/Module3/a.txt index cda6431..944acd5 100644 --- a/Module3/a.txt +++ b/Module3/a.txt @@ -1,3 +1,3 @@ -begin ggg : ; :+= 7 99.9 5 +begin ggg 7 99.9 5 1 ppp end diff --git a/Module3/mymain.cs b/Module3/mymain.cs index 9a33c59..64848e5 100644 --- a/Module3/mymain.cs +++ b/Module3/mymain.cs @@ -30,6 +30,25 @@ namespace GeneratedLexer do { tok = scanner.yylex(); + if(tok == (int)Tok.INUM) + { + sum_int += scanner.LexValueInt; + } + + if (tok == (int)Tok.RNUM) + { + sum_d += scanner.LexValueDouble; + } + + if (tok == (int)Tok.ID) + { + cnt_id++; + min_id_len = Math.Min(min_id_len, scanner.yyleng); + max_id_len = Math.Max(max_id_len, scanner.yyleng); + avg_id_len += scanner.yyleng; + } + + if (tok == (int)Tok.EOF) { Console.WriteLine(); diff --git a/TestGeneratedLexer/Tests.cs b/TestGeneratedLexer/Tests.cs index 57d7d4c..4e530af 100644 --- a/TestGeneratedLexer/Tests.cs +++ b/TestGeneratedLexer/Tests.cs @@ -1,6 +1,7 @@ п»їusing System; using NUnit.Framework; using GeneratedLexer; +using System.Collections.Generic; namespace TestGeneratedLexer { @@ -40,17 +41,22 @@ namespace TestGeneratedLexer } [Test] - [Ignore("This test is disabled")] public void TestString() { LexerAddon lexer = new LexerAddon(@"3 389 3 'ssfsf ' "); lexer.Lex(); - + + Assert.AreEqual(new List<string>() { "'ssfsf '" }, lexer.stringsInSingleApost); // TODO: checks in this test + + LexerAddon lexerNums = new LexerAddon(@"3 389 3 '12 r' '1234 5' "); + lexerNums.Lex(); + + Assert.AreEqual(new List<string>() { "'12 r'", "'1234 5'" }, lexerNums.stringsInSingleApost); + } [Test] - [Ignore("This test is disabled")] public void TestSingleLineCmt() { LexerAddon lexer = new LexerAddon(@"i22d1 5.6 // i 32 id3 @@ -63,7 +69,6 @@ namespace TestGeneratedLexer } [Test] - [Ignore("This test is disabled")] public void TestMultiLineCmt() { LexerAddon lexer = new LexerAddon(@"i22d1 5.6 { i 32 id3 @@ -73,10 +78,18 @@ namespace TestGeneratedLexer Assert.AreEqual(2, lexer.idCount); Assert.AreEqual(43, lexer.sumInt); Assert.AreEqual(14.5, lexer.sumDouble, 0.001); + + LexerAddon lexer1 = new LexerAddon(@"i22d1 5.6 { i 32 id3 + Md4 2.3 2 33} 8.9 {inNd5 1 42} "); + lexer1.Lex(); + + Assert.AreEqual(1, lexer1.idCount); + Assert.AreEqual(0, lexer1.sumInt); + Assert.AreEqual(14.5, lexer1.sumDouble, 0.001); + } [Test] - [Ignore("This test is disabled")] public void TestMultiLineCmtIds() { LexerAddon lexer = new LexerAddon(@"i22d1 5.6 { i 32 id3 -- GitLab