## nada-token-table-g.pkg
# Compiled by:
# src/lib/compiler/front/parser/parser.sublib# Lexer support: Reserved-word and reserved-operator hashtables for Nada syntax.
### "High thoughts must have a high language."
###
### -- Aristophanes
# Used in ROOT/src/lib/compiler/front/parser/lex/nada.lex
stipulate
package fs = fast_symbol; # fast_symbol is from src/lib/compiler/front/basics/map/fast-symbol.pkg package hs = hash_string; # hash_string is from src/lib/src/hash-string.pkg package wht = word_string_hashtable; # word_string_hashtable is from src/lib/compiler/front/basics/hash/wordstr-hashtable.pkgherein
generic package mada_token_table_g (tokens: Nada_Tokens) # Nada_Tokens is from src/lib/compiler/front/parser/yacc/nada.grammar.api : (weak)
api {
/* Construct a lexer token for alpha input token 'foo'.
* If 'foo' is a reserved word in Nada syntax, the
* token will reflect that, otherwise it will be a
* generic VALUE_ID token:
*/
check_value_id: ((String, Int)) -> tokens::Token( tokens::Semantic_Value, Int );
/* Same as above but for operators '**'
* and such rather than identifiers:
*/
check_operator_id: ((String, Int)) -> tokens::Token( tokens::Semantic_Value, Int );
/* We have no reserved words in typevariable space,
* so this just constructs a TYPEVAR_ID token:
*/
check_typevar_id: ((String, Int)) -> tokens::Token( tokens::Semantic_Value, Int );
/* We have no reserved words in constructor space,
* so this just constructs a CONSTRUCTOR_ID token:
*/
check_constructor_id: ((String, Int)) -> tokens::Token( tokens::Semantic_Value, Int );
/* We have no reserved words in typename space,
* so this just constructs a typename token:
*/
check_type_id: ((String, Int)) -> tokens::Token( tokens::Semantic_Value, Int );
}
{
exception NOT_TOKEN;
hash_string = hs::hash_string;
/* Create a hashtable.
*
* 'size' requests a size for the hashtable vector.
* It should typically be about twice the
* length of keyvalpairList, for a 50%
* hashtable load factor.
*
* 'keyvalpairList'
* holds the contents for the hashtable as a
* list of key-my pairs where the keys are
* strings naming reserved words etc, and the
* values are functions which construct
* corresponding lexer tokens.
*
*/
fun make_table (size_hint, keyvalpair_list)
=
{ hashtable = wht::make_hashtable { size_hint, not_found_exception => NOT_TOKEN };
fun insert (token_name, token_construction_function)
=
wht::set hashtable ((hash_string token_name, token_name), token_construction_function);
list::apply insert keyvalpair_list;
hashtable;
};
reserved_words
=
make_table (64, [
("also", \\ yypos => tokens::also_t (yypos, yypos+4); end ),
("api", \\ yypos => tokens::api_t (yypos, yypos+3); end ),
("as", \\ yypos => tokens::as_t (yypos, yypos+2); end ),
("begin", \\ yypos => tokens::case_t (yypos, yypos+5); end ),
("case", \\ yypos => tokens::case_t (yypos, yypos+4); end ),
("else", \\ yypos => tokens::else_t (yypos, yypos+4); end ),
("end", \\ yypos => tokens::end_t (yypos, yypos+3); end ),
("enum", \\ yypos => tokens::enum_t (yypos, yypos+4); end ),
("eqtype", \\ yypos => tokens::eqtype_t (yypos, yypos+6); end ),
("exception", \\ yypos => tokens::exception_t (yypos, yypos+9); end ),
("do", \\ yypos => tokens::do_t (yypos, yypos+2); end ),
("except", \\ yypos => tokens::except_t (yypos, yypos+6); end ),
("fi", \\ yypos => tokens::fi_t (yypos, yypos+2); end ),
("\\", \\ yypos => tokens::fn_t (yypos, yypos+2); end ),
("fun", \\ yypos => tokens::fun_t (yypos, yypos+3); end ),
("if", \\ yypos => tokens::if_t (yypos, yypos+2); end ),
("in", \\ yypos => tokens::in_t (yypos, yypos+2); end ),
("include", \\ yypos => tokens::include_t (yypos, yypos+7); end ),
("lazy", \\ yypos =>
if *nada_parser::lazy_is_a_keyword
tokens::lazy_t (yypos, yypos+4);
else raise exception NOT_TOKEN;fi; end ),
("let", \\ yypos => tokens::let_t (yypos, yypos+3); end ),
("local", \\ yypos => tokens::local_t (yypos, yypos+5); end ),
("macro", \\ yypos => tokens::macro (yypos, yypos+5); end ),
("my", \\ yypos => tokens::my_t (yypos, yypos+2); end ),
("of", \\ yypos => tokens::of_t (yypos, yypos+2); end ),
("opaque", \\ yypos => tokens::opaque (yypos, yypos+6); end ),
("raise", \\ yypos => tokens::raise_t (yypos, yypos+5); end ),
("rec", \\ yypos => tokens::rec_t (yypos, yypos+3); end ),
("sharing", \\ yypos => tokens::sharing_t (yypos, yypos+7); end ),
("package", \\ yypos => tokens::package_t (yypos, yypos+7); end ),
("then", \\ yypos => tokens::then_t (yypos, yypos+4); end ),
("transparent",\\ yypos => tokens::transparent (yypos, yypos+11); end ),
("type", \\ yypos => tokens::type_t (yypos, yypos+4); end ),
("use", \\ yypos => tokens::use (yypos, yypos+3); end ),
("where", \\ yypos => tokens::where_t (yypos, yypos+5); end ),
("while", \\ yypos => tokens::while_t (yypos, yypos+5); end ),
("with", \\ yypos => tokens::with_t (yypos, yypos+4); end ),
("or", \\ yypos => tokens::or_t (yypos, yypos+2); end ),
("and", \\ yypos => tokens::and_t (yypos, yypos+3); end )
]);
# overload_hash = hash_string "overload";
# lazy_hash = hash_string "lazy";
# See if an identifier is in the reserved-word hashtable.
# If it is, construct a matching token
# recording its position in the input.
# Otherwise it is a regular identifier:
fun check_value_id (string, yypos)
=
{ hash = hash_string string;
fun make_id ()
=
tokens::value_id (fs::raw_symbol (hash, string), yypos, yypos+size (string));
wht::get reserved_words (hash, string) yypos
except
NOT_TOKEN
=>
make_id (); end ;
};
fun check_operator_id (string, yypos)
=
{ hash = hash_string string;
tokens::loose_infix_op (fs::raw_symbol (hash, string), yypos, yypos+size (string));
};
# XXX BUGGO FIXME We should actually generate a
# LOOSE or TIGHT infix op per surrouding whitespace,
# and probably allow prefix and suffix too.
#
# But we're actually only handling backquoted
# valueIds here, which is obscure enough that
# I'm willing to let it slide for now in favor
# of working on more critical stuff. CrT 2006-11-04
fun check_type_id (string, yypos)
=
{ hash = hash_string string;
tokens::type_id (fs::raw_symbol (hash, string), yypos, yypos + size (string));
};
fun check_constructor_id (string, yypos)
=
{ hash = hash_string string;
tokens::constructor_id (fs::raw_symbol (hash, string), yypos, yypos + size (string));
};
fun check_typevar_id (string, yypos)
=
{ hash = hash_string string;
tokens::typevar_id (fs::raw_symbol (hash, string), yypos, yypos + size (string));
};
};
end;
## COPYRIGHT (c) 1996 Bell Laboratories.
## Subsequent changes by Jeff Prothero Copyright (c) 2010-2015,
## released per terms of SMLNJ-COPYRIGHT.