(*** Lexical Analysis -- Scanning.  From Chapter 9.  ***)

infix mem;

(*All characters are covered except octal 0-41 (nul-space) and 177 (del),
  which are ignored. *)
functor LexicalFUN (structure Basic: BASIC 
		    and       Keyword: KEYWORD) : LEXICAL =
  struct
  local open Basic in
  datatype token = Key of string  |  Id of string  | Num of int;

  fun is_letter c =
      "A"<=c andalso c<="Z" orelse
      "a"<=c andalso c<="z" 
      
  and is_digit c =
      "0"<=c andalso c<="9"

  and is_underscore c = c = "_";

  val specials = explode"!@#$%^&*()+-={}[]:\"|;'\\,./?`~<>";

  (*scanning of an alpha identifier or keyword*)
  fun alpha (id, c::cs) =
	if is_letter c orelse is_digit c
                       then  alpha (id^c, cs)
		       else  (id, c::cs)
    | alpha (id, []) = (id, []);

  fun digit c = ord c - ord "0" ;
  
  (* scanning a numeral *)    
  fun number (n, c::cs) =
        if is_digit c 
        then number (10*n + digit c, cs)
        else (n, c::cs)
    | number (id, []) = (id, [])

  fun tokenof a = if a mem Keyword.alphas  then  Key(a)  else  Id(a);

  (*scanning of a symbolic keyword*)
  fun symbolic (sy, c::cs) =
	if not (c mem specials)
        then  (sy, c::cs)
	else  symbolic (sy^c, cs)
    | symbolic (sy, []) = (sy, []);

  fun scanning (toks, []) = rev toks    (*end of chars*)
    | scanning (toks, c::cs) =
	if is_letter c orelse is_underscore c
	then (*identifier or keyword*)
	     let val (id, cs') = alpha(c, cs)
	     in  scanning (tokenof id :: toks, cs')
	     end
        else if is_digit c
        then (* decimal numeral *)
             let val (nr, cs') = number(digit c, cs)
             in  scanning (Num nr :: toks, cs')
             end
	else if c mem specials
	then (*special symbol*)
	     let val (sy, cs') = symbolic(c, cs)
	     in  scanning (Key sy :: toks, cs')
	     end
	else (*spaces, line breaks, strange characters are ignored*)
	     scanning (toks, cs);

  (*Scanning a list of characters into a list of tokens*)
  fun scan a = scanning([], explode a);
  end
end;