## mythryl-token-table-g.pkg
# Compiled by:
# src/lib/compiler/front/parser/parser.sublib### "The range of our projectiles---even ... the artillery -- however great,
### will never exceed four of those miles of which as many thousand separate
### us from the center of the earth."
###
### --- Galileo,
### Dialogues Concerning Two New Sciences
/***************************************************************************
hashtable for token recognition
***************************************************************************/
# Used in ROOT/src/lib/compiler/front/parser/lex/mythryl.lex
stipulate
package fs = fast_symbol; # fast_symbol is from src/lib/compiler/front/basics/map/fast-symbol.pkg package hs = hash_string; # hash_string is from src/lib/src/hash-string.pkg package wht = word_string_hashtable; # word_string_hashtable is from src/lib/compiler/front/basics/hash/wordstr-hashtable.pkgherein
generic package mythryl_token_table_g (tokens: Mythryl_Tokens) # Mythryl_Tokens is from src/lib/compiler/front/parser/yacc/mythryl.grammar.api : (weak)
api {
check_id: (String, Int) -> tokens::Token( tokens::Semantic_Value, Int );
check_passive_id: (String, Int) -> tokens::Token( tokens::Semantic_Value, Int );
check_symbol_id: (String, Int) -> tokens::Token( tokens::Semantic_Value, Int );
check_passive_symbol_id: (String, Int) -> tokens::Token( tokens::Semantic_Value, Int );
check_type_var: (String, Int) -> tokens::Token( tokens::Semantic_Value, Int );
new_check_type_var: (String, Int) -> tokens::Token( tokens::Semantic_Value, Int );
check_implicit_thunk_parameter: (String, Int) -> tokens::Token( tokens::Semantic_Value, Int );
}
{ exception NO_TOKEN;
hash_string = hs::hash_string;
fun make_table (size_hint, l)
=
{ t = wht::make_hashtable { size_hint, not_found_exception => NO_TOKEN };
fun ins (str, tokfn)
=
wht::set t ((hash_string str, str), tokfn);
list::apply ins l;
t;
};
symbol_id_table
=
make_table (16, [
# ("-", fn yypos = tokens::dash (yypos, yypos+1)),
# ("/", fn yypos = tokens::slash (yypos, yypos+1)),
# ("*", fn yypos = tokens::star (yypos, yypos+1)),
# ("~", fn yypos = tokens::tilda (yypos, yypos+1)),
# ("|", fn yypos = tokens::bar (yypos, yypos+1)),
(":", fn yypos = tokens::colon (yypos, yypos+1)),
("=", fn yypos = tokens::equal_op (yypos, yypos+1)),
("#", fn yypos = tokens::hash (yypos, yypos+1)),
("==", fn yypos = tokens::eqeq_op (yypos, yypos+2)),
("&=", fn yypos = tokens::amper_eq (yypos, yypos+2)),
("@=", fn yypos = tokens::atsign_eq (yypos, yypos+2)),
("\\=", fn yypos = tokens::back_eq (yypos, yypos+2)),
# ("!=", fn yypos = tokens::bang_eq (yypos, yypos+2)), # Don't do this!
("$=", fn yypos = tokens::buck_eq (yypos, yypos+2)),
("^=", fn yypos = tokens::caret_eq (yypos, yypos+2)),
("-=", fn yypos = tokens::dash_eq (yypos, yypos+2)),
(".=", fn yypos = tokens::dot_eq (yypos, yypos+2)),
("%=", fn yypos = tokens::percnt_eq (yypos, yypos+2)),
("+=", fn yypos = tokens::plus_eq (yypos, yypos+2)),
("?=", fn yypos = tokens::qmark_eq (yypos, yypos+2)),
("/=", fn yypos = tokens::slash_eq (yypos, yypos+2)),
("*=", fn yypos = tokens::star_eq (yypos, yypos+2)),
("~=", fn yypos = tokens::tilda_eq (yypos, yypos+2)),
("++=", fn yypos = tokens::plusplus_eq (yypos, yypos+3)),
("--=", fn yypos = tokens::dashdash_eq (yypos, yypos+3)),
("->", fn yypos = tokens::arrow (yypos, yypos+2)),
("=>", fn yypos = tokens::darrow (yypos, yypos+2)),
("~~", fn yypos = tokens::tilda_tilda (yypos, yypos+2)),
("::", fn yypos = tokens::colon_colon (yypos, yypos+2)),
("??", fn yypos = tokens::what_what (yypos, yypos+2)),
("?:", fn yypos = tokens::what_colon (yypos, yypos+2)), # XXX BUGGO FIXME should kill this
(":?", fn yypos = tokens::colon_what (yypos, yypos+2)), # XXX BUGGO FIXME should kill this
("++", fn yypos = tokens::plus_plus (yypos, yypos+2)),
("--", fn yypos = tokens::dash_dash (yypos, yypos+2))
]);
id_table
=
make_table (64, [
("abstype", fn yypos = tokens::abstype_t (yypos, yypos+7)),
("also", fn yypos = tokens::also_t (yypos, yypos+4)),
("and", fn yypos = tokens::and_t (yypos, yypos+3)),
("api", fn yypos = tokens::api_t (yypos, yypos+3)),
("as", fn yypos = tokens::as_t (yypos, yypos+2)),
("case", fn yypos = tokens::case_t (yypos, yypos+4)),
("class", fn yypos = tokens::class_t (yypos, yypos+5)), # This should be moved to the non-reserved section by switching from "class foo { ... }" to "class package foo { ... }".
("class2", fn yypos = tokens::class2_t (yypos, yypos+6)), # This should die in due course.
("elif", fn yypos = tokens::elif_t (yypos, yypos+4)),
("else", fn yypos = tokens::else_t (yypos, yypos+4)),
("end", fn yypos = tokens::end_t (yypos, yypos+3)),
("eqtype", fn yypos = tokens::eqtype_t (yypos, yypos+6)),
("esac", fn yypos = tokens::esac_t (yypos, yypos+4)),
("except", fn yypos = tokens::except_t (yypos, yypos+6)),
("exception", fn yypos = tokens::exception_t (yypos, yypos+9)),
("fi", fn yypos = tokens::fi_t (yypos, yypos+2)),
("fn", fn yypos = tokens::fn_t (yypos, yypos+2)),
("for", fn yypos = tokens::for_t (yypos, yypos+3)),
("fprintf", fn yypos = tokens::fprintf_t (yypos, yypos+7)),
("fun", fn yypos = tokens::fun_t (yypos, yypos+3)),
("herein", fn yypos = tokens::herein_t (yypos, yypos+6)),
("if", fn yypos = tokens::if_t (yypos, yypos+2)),
("include", fn yypos = tokens::include_t (yypos, yypos+7)),
("lazy", fn yypos = *mythryl_parser::lazy_is_a_keyword ?? tokens::lazy_t (yypos, yypos+4) :: (raise exception NO_TOKEN)), # We should find a way to make this non-reserved even when in use, assuming we decide to support it at all.
("my", fn yypos = tokens::my_t (yypos, yypos+2)),
("or", fn yypos = tokens::or_t (yypos, yypos+2)),
("package", fn yypos = tokens::package_t (yypos, yypos+7)),
("printf", fn yypos = tokens::printf_t (yypos, yypos+6)),
("sharing", fn yypos = tokens::sharing_t (yypos, yypos+7)),
("sprintf", fn yypos = tokens::sprintf_t (yypos, yypos+7)),
("stipulate", fn yypos = tokens::stipulate_t (yypos, yypos+9)),
# ("val", fn yypos = tokens::my_t (yypos, yypos+3)), # WE CAN DUMP RESERVED WORD "val" by changing "field val foo ..." to "my <field> foo ..." etc, where the angle brackets are NOT metasyntax. XXX BUGGO FIXME.
("where", fn yypos = tokens::where_t (yypos, yypos+5)),
("with", fn yypos = tokens::with_t (yypos, yypos+4)),
("withtype", fn yypos = tokens::withtype_t (yypos, yypos+8)),
#
# The above totals 35 reserved words. Of those
#
# "lazy" is not currently supported and probably unneeded anyhow;
# "class" can be phased out.
# "class2" can be phased out.
# "fprintf" can be phased out if we introduce a way to allow user-defined compiletime functions.
# "printf" can be phased out if we introduce a way to allow user-defined compiletime functions.
# "sprintf" can be phased out if we introduce a way to allow user-defined compiletime functions.
# "val" can be phased out.
#
# That leaves 28 reserved words. Of those
# "eqstype" could be phased out if we rewrote to use Harper-Stone semantics/typechecking (instead of the Definition).
# "abstype" is generally considered obsolete.
# Possibly we could do with just one of the "with"/"withtype" duo?
# "include" could be phased out by using syntax ''include my foo;'' Since use of "include" should imho be discouraged anyhow, the extra verbosity is more feature than bug.
#
# Obviously, we could trivially replace "fi" and "esac" with "end"
# if we wanted to, but I don't -- I think they pull their weight
# by improving readability, easily.
#
# Bottom line: We have 25-26 hard-core alphabetic reserved words which
# I expect us to support basically forever -- not bad!
# The following are NOT reserved words, just identifiers which have
# special meaning to the compiler in certain contexts. They can all
# still be used as regular identifiers by the user, without conflict:
#
("field", fn yypos = tokens::field_t (yypos, yypos+5)), # Used in: field my ...
("generic", fn yypos = tokens::generic_t (yypos, yypos+7)), # Used in: generic package ...
("in", fn yypos = tokens::in_t (yypos, yypos+2)), #
("infix", fn yypos = tokens::infix_t (yypos, yypos+5)), # Used in: infix my ...
("infixr", fn yypos = tokens::infixr_t (yypos, yypos+6)), # Used in: infixr my ...
("message", fn yypos = tokens::message_t (yypos, yypos+6)), # Used in: message fun ...
("method", fn yypos = tokens::method_t (yypos, yypos+6)), # Used in: method fun ...
("nonfix", fn yypos = tokens::nonfix_t (yypos, yypos+6)), # Used in: nonfix my ...
("overloaded", fn yypos = tokens::overloaded_t(yypos, yypos+10)), # Used in: overloaded my ...
("raise", fn yypos = tokens::raise_t (yypos, yypos+5)), # Used in: raise exception ...
("recursive", fn yypos = tokens::recursive_t (yypos, yypos+9)) # Used in: recursive my ...
]);
# overload_hash = hash_string "overload";
# lazy_hash = hash_string "lazy";
# Look up an identifier.
# If the symbol is found
# the corresponding token is
# generated, otherwise it is
# a regular lowercase id:
#
fun check_id (str, yypos)
=
{ hash = hash_string str;
fun make_id ()
=
tokens::lowercase_id (fs::raw_symbol (hash, str), yypos, yypos+(size str));
wht::get id_table (hash, str) yypos
except
NO_TOKEN
=
make_id ();
};
fun check_passive_id (string, yypos)
=
{
len = string::length string;
string = string::extract( string, 1, THE (len - 2) ); # Drop leading and trailing parentheses from string.
hash = hash_string string;
fun make_id ()
=
tokens::passiveop_id (fs::raw_symbol (hash, string), yypos, yypos+(size string));
wht::get id_table (hash, string) yypos
except
NO_TOKEN
=
make_id ();
};
# Look up an identifier with a leading '#' on its name.
# We use these for implicit thunk parameters, so that we
# can write
#
# .{ #a < #b; }
#
# as an abbreviation for
#
# fn (a, b) = a < b;
#
fun check_implicit_thunk_parameter (str, yypos)
=
{ hash = hash_string str;
fun make_id ()
=
tokens::implicit_thunk_parameter (fs::raw_symbol (hash, str), yypos, yypos+(size str));
# wht::lookup_implicit_thunk_parameter_table (hash, str) yypos # Eventually we'll want this for #if #else #elif #end
# except
# NO_TOKEN
# =
make_id ();
};
fun check_symbol_id (str, yypos)
=
{ hash = hash_string str;
wht::get symbol_id_table (hash, str) yypos
except
NO_TOKEN
=
tokens::operators_id (fs::raw_symbol (hash, str), yypos, yypos+(size str));
};
fun check_passive_symbol_id (string, yypos)
=
{ len = string::length string;
string = string::extract( string, 1, THE (len - 2) ); # Drop leading and trailing parentheses from string.
hash = hash_string string;
tokens::passiveop_id (fs::raw_symbol (hash, string), yypos+1, yypos+1+(size string));
};
fun check_type_var (str, yypos)
=
{ hash = hash_string str;
tokens::tyvar (fs::raw_symbol (hash, str), yypos, yypos+(size str));
};
fun new_check_type_var (str, yypos)
=
{ string = /* "$" + */ str;
hash = hash_string string;
tokens::tyvar (fs::raw_symbol (hash, string), yypos, yypos+(size str));
};
};
end;
## COPYRIGHT (c) 1996 Bell Laboratories.
## Subsequent changes by Jeff Prothero Copyright (c) 2010-2013,
## released per terms of SMLNJ-COPYRIGHT.