The lexer, more compact, and some test code.
Well, as I mentioned in a note, I realized there was some legacy code left in the Lexer that I previously posted. I am now posting a much more compact lexer for you guys to check out. I've also included some test code, so that you can load up any file (I recommend loading up and seeing the lexing of C# source). Enjoy! PS > That parser is still on the way. I don't know what to call the language since I haven't found any specific applications that use the type of the configuration file I'm loading. Maybe I'll get some response on the newsgroups and we'll come up with a name for it.
using System;
using System.Collections;
using System.IO;
public class Token {
public string TokenData;
public Token(string tokenData) { TokenData = tokenData; }
}
public class BasicLex {
public static Token[] StringToTokens(string tokenString) {
return StringToTokens(tokenString, " \n\r\t{}\"=;.()[],", " \t\r\n");
}
public static Token[] StringToTokens(string tokenString, string breakers, string toss) {
ArrayList tokens = new ArrayList();
int tokenStart = 0, tokenPointer = 0;
while(tokenPointer <= tokenString.Length) {
if ( tokenPointer == tokenString.Length || breakers.IndexOf(tokenString[tokenPointer]) > -1 ) {
if ( tokenStart != tokenPointer ) {
tokens.Add(new Token(tokenString.Substring(tokenStart, tokenPointer - tokenStart)));
}
if ( tokenPointer < tokenString.Length && toss.IndexOf(tokenString[tokenPointer]) == -1 ) {
tokens.Add(new Token(tokenString.Substring(tokenPointer, 1)));
}
tokenStart = tokenPointer + 1;
}
tokenPointer++;
}
return (Token[]) tokens.ToArray(typeof(Token));
}
}
public class LexRunner {
private static void Main(string[] args) {
string source = null;
using(StreamReader sr = new StreamReader(args[0])) {
source = sr.ReadToEnd();
sr.Close();
}
if ( source != null ) {
Token[] tokens = BasicLex.StringToTokens(source);
for(int i = 0; i < tokens.Length; i++) { Console.WriteLine(tokens[i].TokenData); }
}
}
}