modified: app/Lexer.hs

modified:   app/Main.hs

app/Lexer.hs

@@ -1,13 +1,15 @@
-module Lexer (tokenize, Token (..), TokenType (..) ) where
+module Lexer (tokenize, Token (..), TokenType (..), TrailType (..), TrailRule (..) ) where
 
 import Data.List
 import Data.Char (ord)
 
-data TokenType = Quotes | Dot | Comma | Colon | EndStatement | EOF | QString | Literal | Number | Letter | Digit | Assignment | OpenParen | CloseParen | OpenCurved | CloseCurved | OpenSquared | CloseSquared | Arithmetic | Comparison | Bitwise | Empty deriving (Show, Eq)
+data TrailType = OnlyDigits | LettersOrDigits | OnlySpecial deriving (Show, Eq)
+data TokenType = Quotes | Dot | Comma | Colon | EndStatement | EOF | QString | Numeric | Literal | Digit | Assignment | OpenParen | CloseParen | OpenCurved | CloseCurved | OpenSquared | CloseSquared | Arithmetic | Comparison | Bitwise | Empty deriving (Show, Eq)
 data Token = Token {value :: [Char], token_type :: TokenType} deriving (Show)
+data TrailRule = TrailRule {rule_val :: [Char], trail_type :: TrailType} deriving (Show)
 
-parseSingleChar :: Char -> Token
-parseSingleChar c
+parseSingleToken :: [Char] -> Token
+parseSingleToken code
     | c == '(' = Token [c] OpenParen
     | c == ')' = Token [c] CloseParen
     | c == '{' = Token [c] OpenCurved
@@ -23,55 +25,33 @@ parseSingleChar c
     | c == '.' = Token [c] Dot
     | c == '\'' = Token [c] Quotes
     | c == ',' = Token [c] Comma
-    | elem c ['0'..'9'] = Token [c] Digit
-    | elem c $ ['a'..'z'] ++ ['A'..'Z'] = Token [c] Letter
+    | elem c ['0'..'9'] = Token code Numeric
+    | elem c $ ['a'..'z'] ++ ['A'..'Z'] = Token code Literal
     | otherwise = Token " " Empty
+    where c = head code
 
+-- shall return [Token]
 tokenize :: [Char] -> [Token]
-tokenize sourceCode = filter (\t -> token_type t /= Empty && token_type t /= EOF) (reduceTokens ((map parseSingleChar sourceCode) ++ [Token " " EOF]) 0)
+tokenize sourceCode
+    | elem f ['0'..'9'] = parseRule t $ TrailRule [f] OnlyDigits
+    | elem f $ ['a'..'z'] ++ ['A'..'Z'] = parseRule t $ TrailRule [f] LettersOrDigits
+    | otherwise = parseRule t $ TrailRule [f] OnlySpecial
+    where f = head sourceCode
+          t = tail sourceCode
 
-conTokType :: TokenType -> TokenType
-conTokType tt
-    | tt == Digit = Number
-    | otherwise = Literal
+joinTokens :: (Token, [Char]) -> [Token]
+joinTokens t = (fst t):(tokenize $ snd t)
 
-reduceTokens :: [Token] -> Int -> [Token]
-reduceTokens all_tokens i
-    | token_type ct == Digit || token_type ct == Letter = (Token (makeIdentifier (drop i all_tokens) 0) (conTokType $ token_type ct)):(_reduceTokens all_tokens (findFirstEmpty all_tokens i))
-    | token_type ct == Quotes = (Token (_makeQuoteString (drop (succ i) all_tokens) 0) QString):(_reduceTokens all_tokens (findFirstQuotes all_tokens (succ i)))
-    | otherwise = ct:(_reduceTokens all_tokens (succ i))
-    where ct = all_tokens !! i
+parseRule :: [Char] -> TrailRule -> [Token]
+parseRule code rule
+    | tt == OnlyDigits = joinTokens $ parseDigitsRule code rule
+    | otherwise = []
+    where tt = trail_type rule
 
-_reduceTokens :: [Token] -> Int -> [Token]
-_reduceTokens all_tokens i
-    | i >= length all_tokens = [Token " " EOF]
-    | otherwise = reduceTokens all_tokens i
-
-_makeQuoteString :: [Token] -> Int -> [Char]
-_makeQuoteString t i
-    | i >= length t = ""
-    | otherwise = makeQuoteString t i
-
-makeQuoteString :: [Token] -> Int -> [Char]
-makeQuoteString t i
-    | tt == Quotes = ""
-    | otherwise = ((value $ t !! i) !! 0):(_makeQuoteString t (succ i))
-    where tt = token_type $ t !! i
-
-makeIdentifier :: [Token] -> Int -> [Char]
-makeIdentifier t i
-    | i >= length t || (tt /= Letter && tt /= Digit) = ""
-    | otherwise = ((value $ t !! i) !! 0):(makeIdentifier t (succ i))
-    where tt = token_type $ t !! i
-
-findFirstEmpty :: [Token] -> Int -> Int
-findFirstEmpty t i
-    | i >= length t || (tt /= Letter && tt /= Digit) = i
-    | otherwise = findFirstEmpty t $ succ i
-    where tt = token_type $ t !! i
-
-findFirstQuotes :: [Token] -> Int -> Int
-findFirstQuotes t i
-    | i >= length t || tt == Quotes = succ i
-    | otherwise = findFirstQuotes t $ succ i
-    where tt = token_type $ t !! i
+parseDigitsRule :: [Char] -> TrailRule -> (Token, [Char])
+parseDigitsRule code rule
+    | elem c ['0'..'9'] = parseDigitsRule t $ TrailRule (c:rv) OnlyDigits
+    | otherwise = parseSingleToken rv code
+    where rv = rule_val rule
+          c  = head code
+          t  = tail code

app/Main.hs

@@ -10,12 +10,11 @@ main = do
     print (_extractExpression x) -}
 
 main = do
-    handle <- openFile "../../../../../../../../../as/test.as" ReadMode
+    handle <- openFile "as/test.as" ReadMode
     contents <- hGetContents handle
     let x = tokenize contents
     print x
     --let y = parseIntoTree x
-    --print y
     --print $ findFirstEmpty x 0
     --print $ take (findFirstEmpty x 0) x
     --print $ makeInt x