honk

1 files changed, 1856 insertions, 85 deletions

diff --git a/compiler_and_linker/FrontEnd/C/CPrepTokenizer.c b/compiler_and_linker/FrontEnd/C/CPrepTokenizer.c
index 5b77268..62753f8 100644
--- a/compiler_and_linker/FrontEnd/C/CPrepTokenizer.c
+++ b/compiler_and_linker/FrontEnd/C/CPrepTokenizer.c
@@ -1,156 +1,1889 @@
 #include "compiler.h"
 #include "compiler/tokens.h"
+#include "compiler/CError.h"
+#include "compiler/CInt64.h"
+#include "cos.h"
 
 static Boolean prepnextstringchar_foundnl;
 
+#define UCD (unsigned char *)
+#define CD (char *)
+
 short prepskipnextchar() {
+    short c;
+    unsigned char *start;
+    unsigned char *p;
+
+    start = UCD pos;
+    p = UCD pos;
 
+loop:
+    switch ((c = *p)) {
+        case 0:
+            if (tokenstacklevel > 0 || CD p >= prep_file_end) {
+                pos = CD p;
+                nextcharpos = CD p;
+                return c;
+            } else {
+                pos = CD p++;
+                nextcharpos = CD p;
+                return c;
+            }
+        case 4:
+        case 5:
+            p++;
+            goto loop;
+        case ' ':
+        case '\t':
+        case '\n':
+        case '\v':
+        case '\f':
+            spaceskip = 1;
+            p++;
+            goto loop;
+        case ';':
+            pos = CD p++;
+            if (tokenstacklevel <= 0 && in_assembler && copts.asmsemicolcomment) {
+                spaceskip = 1;
+                nextcharpos = CPrep_SkipNewComment(CD p);
+                return CPrep_SkipNewCommentChar;
+            }
+            nextcharpos = CD p;
+            return c;
+        case '#':
+            pos = CD p++;
+            if (tokenstacklevel <= 0 && in_assembler && copts.asmpoundcomment) {
+                spaceskip = 1;
+                nextcharpos = CPrep_SkipNewComment(CD p);
+                return CPrep_SkipNewCommentChar;
+            }
+            nextcharpos = CD p;
+            return c;
+        case '/':
+            pos = CD p++;
+            if (copts.multibyteaware) {
+                while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
+                    foundnl();
+                    if (p[2] == '\n')
+                        p += 3;
+                    else
+                        p += 2;
+                }
+            } else {
+                while (p[0] == '\\' && p[1] == '\r') {
+                    foundnl();
+                    if (p[2] == '\n')
+                        p += 3;
+                    else
+                        p += 2;
+                }
+            }
+            if (tokenstacklevel <= 0) {
+                if (p[0] == '/' && (!copts.ANSI_strict || copts.cplusplus || copts.c9x)) {
+                    spaceskip = 1;
+                    nextcharpos = CPrep_SkipNewComment(CD p + 1);
+                    return CPrep_SkipNewCommentChar;
+                } else if (p[0] == '*') {
+                    spaceskip = 1;
+                    p++;
+                    for (;;) {
+                        if (!(c = *(p++))) {
+                            if (tokenstacklevel > 0 || CD p >= prep_file_end) {
+                                CPrep_Error(103);
+                                nextcharpos = CD p - 1;
+                                return c;
+                            }
+                            p[-1] = ' ';
+                        }
+                        if (c == '*') {
+                            if (copts.multibyteaware) {
+                                while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
+                                    foundnl();
+                                    if (p[2] == '\n')
+                                        p += 3;
+                                    else
+                                        p += 2;
+                                }
+                            } else {
+                                while (p[0] == '\\' && p[1] == '\r') {
+                                    foundnl();
+                                    if (p[2] == '\n')
+                                        p += 3;
+                                    else
+                                        p += 2;
+                                }
+                            }
+                            if (*p == '/') {
+                                p++;
+                                goto loop;
+                            }
+                        }
+                        if (c == '\r')
+                            foundnl();
+                    }
+                }
+            }
+            nextcharpos = CD p;
+            return c;
+        case '%':
+            pos = CD p++;
+            if (*p == ':' && copts.cplusplus) {
+                nextcharpos = CD p + 1;
+                return '#';
+            }
+            nextcharpos = CD p;
+            return c;
+        case '\\':
+            pos = CD p++;
+        mid_backslash:
+            if (*p == '\r') {
+                if (!copts.multibyteaware || !COS_IsMultiByte(CD start, CD p - 1)) {
+                    foundnl();
+                    if (p[1] == '\n')
+                        p += 2;
+                    else
+                        p += 1;
+                    goto loop;
+                }
+            }
+            nextcharpos = CD p;
+            return c;
+        case '?':
+            pos = CD p++;
+            if (copts.trigraphs && p[0] == '?') {
+                switch (p[1]) {
+                    case '=':
+                        nextcharpos = CD p + 2;
+                        return '#';
+                    case '/':
+                        c = '\\';
+                        p += 2;
+                        goto mid_backslash;
+                    case '\'':
+                        nextcharpos = CD p + 2;
+                        return '^';
+                    case '(':
+                        nextcharpos = CD p + 2;
+                        return '[';
+                    case ')':
+                        nextcharpos = CD p + 2;
+                        return ']';
+                    case '!':
+                        nextcharpos = CD p + 2;
+                        return '|';
+                    case '<':
+                        nextcharpos = CD p + 2;
+                        return '{';
+                    case '>':
+                        nextcharpos = CD p + 2;
+                        return '}';
+                    case '-':
+                        nextcharpos = CD p + 2;
+                        return '~';
+                }
+            }
+            nextcharpos = CD p;
+            return c;
+        default:
+            pos = CD p++;
+            nextcharpos = CD p;
+            return c;
+    }
 }
 
 short prepnextchar() {
+    short c;
+    unsigned char *start;
+    unsigned char *p;
+
+    start = UCD pos;
+    p = UCD pos;
 
+loop:
+    switch ((c = *(p++))) {
+        case 0:
+            pos = CD p - 1;
+            return c;
+        case '/':
+            if (copts.multibyteaware) {
+                while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
+                    foundnl();
+                    if (p[2] == '\n')
+                        p += 3;
+                    else
+                        p += 2;
+                }
+            } else {
+                while (p[0] == '\\' && p[1] == '\r') {
+                    foundnl();
+                    if (p[2] == '\n')
+                        p += 3;
+                    else
+                        p += 2;
+                }
+            }
+            if (p[0] == '/' && (!copts.ANSI_strict || copts.cplusplus)) {
+                spaceskip = 1;
+                pos = CPrep_SkipNewComment(CD p + 1);
+                return CPrep_SkipNewCommentChar;
+            } else if (p[0] == '*') {
+                spaceskip = 1;
+                p++;
+                for (;;) {
+                    if (!(c = *(p++))) {
+                        if (tokenstacklevel > 0 || CD p >= prep_file_end) {
+                            pos = CD p - 1;
+                            CPrep_Error(103);
+                            return c;
+                        }
+                        p[-1] = ' ';
+                    }
+                    if (c == '*') {
+                        if (copts.multibyteaware) {
+                            while (p[0] == '\\' && !COS_IsMultiByte(CD start, CD p) && p[1] == '\r') {
+                                foundnl();
+                                if (p[2] == '\n')
+                                    p += 3;
+                                else
+                                    p += 2;
+                            }
+                        } else {
+                            while (p[0] == '\\' && p[1] == '\r') {
+                                foundnl();
+                                if (p[2] == '\n')
+                                    p += 3;
+                                else
+                                    p += 2;
+                            }
+                        }
+                        if (*p == '/')
+                            break;
+                    }
+                    if (c == '\r')
+                        foundnl();
+                }
+                pos = CD p + 1;
+                return ' ';
+            } else {
+                pos = CD p;
+                return c;
+            }
+        case '\\':
+        backslash:
+            if (*p == '\r') {
+                if (!copts.multibyteaware || !COS_IsMultiByte(CD start, CD p - 1)) {
+                    foundnl();
+                    if (p[1] == '\n')
+                        p += 2;
+                    else
+                        p += 1;
+                    goto loop;
+                }
+            }
+            pos = CD p;
+            return c;
+        case '%':
+            if (*p == ':' && copts.cplusplus) {
+                pos = CD p + 1;
+                return '#';
+            }
+            pos = CD p;
+            return c;
+        case '?':
+            if (copts.trigraphs && p[0] == '?') {
+                switch (p[1]) {
+                    case '=':
+                        pos = CD p + 2;
+                        return '#';
+                    case '/':
+                        c = '\\';
+                        p += 2;
+                        goto backslash;
+                    case '\'':
+                        pos = CD p + 2;
+                        return '^';
+                    case '(':
+                        pos = CD p + 2;
+                        return '[';
+                    case ')':
+                        pos = CD p + 2;
+                        return ']';
+                    case '!':
+                        pos = CD p + 2;
+                        return '|';
+                    case '<':
+                        pos = CD p + 2;
+                        return '{';
+                    case '>':
+                        pos = CD p + 2;
+                        return '}';
+                    case '-':
+                        pos = CD p + 2;
+                        return '~';
+                }
+            }
+            pos = CD p;
+            return c;
+        default:
+            pos = CD p;
+            return c;
+    }
 }
 
 short prepnextstringchar(char *str, Boolean flag) {
+    unsigned char *p;
+    short c;
 
+    p = UCD pos;
+
+loop:
+    switch ((c = *(p++))) {
+        case 0:
+            pos = CD p - 1;
+            return c;
+        case '\\':
+        backslash:
+            if (p[0] == '\r' && (!copts.multibyteaware || !COS_IsMultiByte(str, CD p - 1))) {
+                if (flag)
+                    foundnl();
+                else
+                    prepnextstringchar_foundnl = 1;
+                if (p[1] == '\n')
+                    p += 2;
+                else
+                    p += 1;
+                goto loop;
+            }
+            pos = CD p;
+            return c;
+        case '?':
+            if (copts.trigraphs && p[0] == '?') {
+                switch (p[1]) {
+                    case '=':
+                        pos = CD p + 2;
+                        return '#';
+                    case '/':
+                        c = '\\';
+                        p += 2;
+                        goto backslash;
+                    case '\'':
+                        pos = CD p + 2;
+                        return '^';
+                    case '(':
+                        pos = CD p + 2;
+                        return '[';
+                    case ')':
+                        pos = CD p + 2;
+                        return ']';
+                    case '!':
+                        pos = CD p + 2;
+                        return '|';
+                    case '<':
+                        pos = CD p + 2;
+                        return '{';
+                    case '>':
+                        pos = CD p + 2;
+                        return '}';
+                    case '-':
+                        pos = CD p + 2;
+                        return '~';
+                }
+            }
+            pos = CD p;
+            return c;
+        default:
+            pos = CD p;
+            return c;
+    }
 }
 
 void CPrep_MatchChar(char ch, Boolean flag) {
+    char *p;
+    char *start;
+    char c;
+    Boolean r26;
+
+    p = start = pos;
+    r26 = 0;
 
+loop:
+    if (!(c = *(p++))) {
+        if (tokenstacklevel > 0 || p >= prep_file_end)
+            CPrep_Error(102);
+        else if (ch == '"')
+            CPrep_Error(101);
+        else if (ch == '\'')
+            CPrep_Error(100);
+        CPrep_Error(117);
+        pos = p - 1;
+        return;
+    }
+
+    if (c == '\r') {
+        if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
+            if (flag)
+                foundnl();
+            if (*p == '\n')
+                p++;
+            CPrep_Error(112);
+            pos = p;
+            return;
+        }
+        r26 = 0;
+        goto loop;
+    }
+
+    if (c == '?' && p[0] == '?' && copts.trigraphs) {
+        switch (p[1]) {
+            case '!':
+            case '\'':
+            case '(':
+            case ')':
+            case '-':
+            case '<':
+            case '=':
+            case '>':
+                r26 = 0;
+                p += 2;
+                goto loop;
+            case '/':
+                p += 2;
+                goto backslash;
+        }
+    }
+
+    if (c == '\\' && !r26 && (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1))) {
+    backslash:
+        if (*p == '\r') {
+            if (flag)
+                foundnl();
+            if (*(++p) == '\n')
+                p++;
+            r26 = 0;
+        } else {
+            r26 = 1;
+        }
+        goto loop;
+    }
+
+    if (c == ch && !r26) {
+        pos = p;
+    } else {
+        r26 = 0;
+        goto loop;
+    }
 }
 
-char *CPrep_MatchChar2(char *str, char ch, Boolean flag) {
+char *CPrep_MatchChar2(char *start, char ch, Boolean flag) {
+    char *p;
+    char c;
+    Boolean r28;
+
+    p = start;
+    r28 = 0;
 
+loop:
+    if (!(c = *(p++))) {
+        if (tokenstacklevel > 0 || p >= prep_file_end)
+            CPrep_Error(102);
+        else if (ch == '"')
+            CPrep_Error(101);
+        else if (ch == '\'')
+            CPrep_Error(100);
+        CPrep_Error(117);
+        return p - 1;
+    }
+
+    if (c == '\r') {
+        if (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1)) {
+            if (flag)
+                foundnl();
+            if (*p == '\n')
+                p++;
+            CPrep_Error(112);
+            return p;
+        }
+        r28 = 0;
+        goto loop;
+    }
+
+    if (c == '?' && p[0] == '?' && copts.trigraphs) {
+        switch (p[1]) {
+            case '!':
+            case '\'':
+            case '(':
+            case ')':
+            case '-':
+            case '<':
+            case '=':
+            case '>':
+                r28 = 0;
+                p += 2;
+                goto loop;
+            case '/':
+                p += 2;
+                goto backslash;
+        }
+    }
+
+    if (c == '\\' && !r28 && (!copts.multibyteaware || !COS_IsMultiByte(start, p - 1))) {
+        backslash:
+        if (*p == '\r') {
+            if (flag)
+                foundnl();
+            if (*(++p) == '\n')
+                p++;
+            r28 = 0;
+        } else {
+            r28 = 1;
+        }
+        goto loop;
+    }
+
+    if (c == ch && !r28) {
+        return p;
+    } else {
+        r28 = 0;
+        goto loop;
+    }
 }
 
 short prepcurchar() {
+    char *save_pos;
+    short t;
+
+    save_pos = pos;
+    t = prepnextchar();
+    nextcharpos = pos;
+    pos = save_pos;
 
+    return t;
 }
 
 static short prepcurstringchar(char *str) {
+    char *save_pos;
+    short t;
+
+    save_pos = pos;
+    prepnextstringchar_foundnl = 0;
+    t = prepnextstringchar(str, 0);
+    nextcharpos = pos;
+    pos = save_pos;
+
+    return t;
 }
 
 static void prepcurstringchar_skip() {
+    pos = nextcharpos;
+    if (prepnextstringchar_foundnl)
+        foundnl();
 }
 
-char *ReadIdentifier(char *str) {
+char *ReadIdentifier(char *p) {
+    char buf[256];
+    char *start;
+    int len;
+
+    start = p;
+    len = 0;
+loop:
+    for (;;) {
+        if (!cprep_idarray[(unsigned char) *p])
+            break;
+        if (len < 255)
+            buf[len++] = *p;
+        p++;
+    }
+
+    if (copts.multibyteaware) {
+        if ((unsigned char) *p == '\\' && !COS_IsMultiByte(start, p) && p[1] == '\r') {
+            if (p[2] == '\n')
+                p += 3;
+            else
+                p += 2;
+            foundnl();
+            goto loop;
+        }
+    } else {
+        if ((unsigned char) *p == '\\' && p[1] == '\r') {
+            if (p[2] == '\n')
+                p += 3;
+            else
+                p += 2;
+            foundnl();
+            goto loop;
+        }
+    }
+
+    buf[len] = 0;
+    tkidentifier = GetHashNameNodeExport(buf);
+    return p;
 }
 
 static short intsuffix(short token, Boolean flag) {
+    Boolean is_unsigned;
+    Boolean is_long;
+    Boolean is_longlong;
+    short t;
+
+    is_unsigned = is_long = is_longlong = 0;
+    t = token;
+    switch (t) {
+        case 'u':
+        case 'U':
+            is_unsigned = 1;
+            pos = nextcharpos;
+            t = prepcurchar();
+            if (t == 'l' || t == 'L') {
+                is_long = 1;
+                pos = nextcharpos;
+                t = *pos;
+                if (copts.longlong) {
+                    pos = nextcharpos;
+                    t = prepcurchar();
+                    if (t == 'l' || t == 'L') {
+                        is_longlong = 1;
+                        pos = nextcharpos;
+                        t = *pos;
+                    }
+                }
+            }
+            break;
+        case 'l':
+        case 'L':
+            is_long = 1;
+            pos = nextcharpos;
+            t = prepcurchar();
+            if (t == 'u' || t == 'U') {
+                is_unsigned = 1;
+                pos = nextcharpos;
+                t = *pos;
+            } else if (copts.longlong && (t == 'l' || t == 'L')) {
+                is_longlong = 1;
+                pos = nextcharpos;
+                t = prepcurchar();
+                if (t == 'u' || t == 'U') {
+                    is_unsigned = 1;
+                    pos = nextcharpos;
+                    t = *pos;
+                }
+            }
+            break;
+        case 'i':
+        case 'I':
+            if (copts.cpp_extensions && !copts.ANSI_strict && copts.longlong) {
+                pos = nextcharpos;
+                t = prepcurchar();
+                if (t == '6') {
+                    pos = nextcharpos;
+                    t = prepcurchar();
+                    if (t == '4') {
+                        is_long = is_longlong = 1;
+                        pos = nextcharpos;
+                        t = *pos;
+                    }
+                }
+            }
+            break;
+        case 'b':
+        case 'B':
+        case 'f':
+        case 'F':
+            if (InlineAsm_gccmode)
+                return IT_INT;
+            break;
+    }
+
+    if ((t >= 'a' && t <= 'z') || (t >= 'A' && t <= 'Z') || (t == '_') || (t >= '0' && t <= '9'))
+        CPrep_Error(105);
+
+    if (!is_longlong && !is_long) {
+        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedint.size))
+            return IT_INT;
+        if ((flag || is_unsigned) && CInt64_IsInURange(tkintconst, stunsignedint.size))
+            return IT_UINT;
+    }
+    if (!is_longlong) {
+        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedlong.size))
+            return IT_LONG;
+        if (CInt64_IsInURange(tkintconst, stunsignedlong.size))
+            return IT_ULONG;
+    }
+    if (copts.longlong) {
+        if (!is_unsigned && CInt64_IsInRange(tkintconst, stsignedlonglong.size))
+            return IT_LONGLONG;
+        if (CInt64_IsInURange(tkintconst, stsignedlonglong.size))
+            return IT_ULONGLONG;
+    }
+    CError_Error(154);
+    tkintconst = cint64_zero;
+    return IT_INT;
 }
 
 static short floatsuffix(short token) {
+    short whichtype;
+
+    switch (token) {
+        case 'f':
+        case 'F':
+            pos = nextcharpos;
+            token = *pos;
+            whichtype = IT_FLOAT;
+            if (stfloat.size != stlongdouble.size)
+                tkfloatconst = CMach_CalcFloatConvert((Type *) &stfloat, tkfloatconst);
+            break;
+        case 'l':
+        case 'L':
+            pos = nextcharpos;
+            token = *pos;
+            whichtype = IT_LONGDOUBLE;
+            break;
+        case 'd':
+        case 'D':
+            if (!copts.ANSI_strict) {
+                pos = nextcharpos;
+                token = *pos;
+                whichtype = IT_DOUBLE;
+                if (stdouble.size != stlongdouble.size)
+                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stdouble, tkfloatconst);
+            }
+            break;
+        default:
+            if (copts.float_constants) {
+                whichtype = IT_FLOAT;
+                if (stfloat.size != stlongdouble.size)
+                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stfloat, tkfloatconst);
+            } else {
+                whichtype = IT_DOUBLE;
+                if (stdouble.size != stlongdouble.size)
+                    tkfloatconst = CMach_CalcFloatConvert((Type *) &stdouble, tkfloatconst);
+            }
+    }
+
+    if ((token >= 'a' && token <= 'z') || (token >= 'A' && token <= 'Z') || (token == '_') || (token >= '0' && token <= '9')) {
+        CPrep_Error(105);
+        whichtype = IT_DOUBLE;
+    }
+
+    return whichtype;
 }
 
 static short tohex(short token) {
+    if (token >= '0' && token <= '9')
+        return token - '0';
+    if (token >= 'a' && token <= 'f')
+        return token - 'a' + 10;
+    if (token >= 'A' && token <= 'F')
+        return token - 'A' + 10;
+    return -1;
 }
 
-static short nextchar(char *str) {
+static SInt32 nextchar(char *str) {
+    // minor register allocation issues
+    SInt32 chr_max;
+    short tmpchr;
+    SInt32 chr;
+    Boolean out_of_bounds;
+    short i;
+    short t;
+
+    chr_max = 0xFF;
+    if (widestring) chr_max = 0xFFFF;
+
+    was_escchar = 0;
+    t = prepnextstringchar(str, 1);
+
+    if (t == '\\' && (!copts.multibyteaware || !COS_IsMultiByte(str, pos - 1))) {
+        was_escchar = 1;
+        t = prepnextstringchar(str, 1);
+        switch (t) {
+            case 'a': return 7;
+            case 'b': return 8;
+            case 't': return 9;
+            case 'v': return 11;
+            case 'f': return 12;
+            case 'n':
+                if (copts.mpwc_newline)
+                    return 13;
+                else
+                    return 10;
+            case 'r':
+                if (copts.mpwc_newline)
+                    return 10;
+                else
+                    return 13;
+            case 'e': return 27;
+            case 'x':
+                chr = tohex(prepnextstringchar(str, 1));
+                if (chr == -1) {
+                    CPrep_Error(100);
+                    return ' ';
+                }
+                out_of_bounds = 0;
+                while (tohex(tmpchr = prepcurstringchar(str)) != -1) {
+                    chr = (chr << 4) + tohex(tmpchr);
+                    prepcurstringchar_skip();
+                    if (chr & ~chr_max)
+                        out_of_bounds = 1;
+                }
+                if (out_of_bounds)
+                    CError_Error(100);
+                break;
+            default:
+                if (t >= '0' && t <= '7') {
+                    out_of_bounds = 0;
+                    i = 1;
+                    chr = t - '0';
+                    while (i < 3 && (tmpchr = prepcurstringchar(str)) >= '0' && tmpchr <= '9') {
+                        chr = (chr << 3) + (tmpchr - '0');
+                        prepcurstringchar_skip();
+                        if (chr & ~chr_max)
+                            out_of_bounds = 1;
+                        i++;
+                    }
+                    if (out_of_bounds)
+                        CError_Error(100);
+                } else {
+                    chr = t;
+                }
+        }
+    } else {
+        chr = t;
+    }
+
+    if (chr_max == 0xFF)
+        return (char) (chr & chr_max);
+    else
+        return (chr & chr_max);
 }
 
 char *CPrep_SkipNewComment(char *str) {
-
 }
 
-Boolean skipendoflinematch(char *str, Boolean flag) {
+Boolean skipendoflinematch(char *str, short token) {
 }
 
 void skipendofline() {
 }
 
 void CPrep_SkipAsmComment() {
+    Boolean save_macrocheck = macrocheck;
+    skipendofline();
+    macrocheck = save_macrocheck;
+}
+
+static short tille() { return TK_NEG6; }
+
+static short tcret() {
+    newline();
+    if (cprep_eoltokens)
+        return TK_NEG7;
+    else
+        return 0;
+}
+
+static short tapos() {
+    // has register allocation issues
+    UInt32 maxvalue;
+    int bytecount;
+    CInt64 ci58;
+    CInt64 ci50;
+    CInt64 ci48;
+    SInt32 t;
+    SInt32 t2;
+    int i;
+    char *start;
+
+    start = pos;
+    t = nextchar(pos);
+    if ((t == '\'' || t == 0 || t == '\r') && !was_escchar) {
+        CPrep_Error(100);
+        tkintconst = cint64_zero;
+        tksize = IT_INT;
+        return TK_INTCONST;
+    }
+
+    t2 = nextchar(start);
+    if (t2 == '\'' && !was_escchar) {
+        if (widestring) {
+            switch (stwchar.size) {
+                case 1:
+                    CInt64_SetULong(&tkintconst, (UInt8) t);
+                    tksize = copts.cplusplus ? IT_SCHAR : IT_INT;
+                    break;
+                case 2:
+                    CInt64_SetULong(&tkintconst, (UInt16) t);
+                    tksize = IT_UINT;
+                    break;
+                case 4:
+                    CInt64_SetULong(&tkintconst, (UInt32) t);
+                    tksize = IT_UINT;
+                    break;
+                default:
+#line 1386
+                    CError_FATAL();
+            }
+        } else {
+            if (copts.unsignedchars)
+                t &= 0xFF;
+            CInt64_SetLong(&tkintconst, t);
+            tksize = copts.cplusplus ? IT_SCHAR : IT_INT;
+        }
+    } else {
+        if (widestring) {
+            switch (stwchar.size) {
+                case 1:
+                    CInt64_SetULong(&ci50, 8);
+                    maxvalue = 0xFF;
+                    bytecount = 1;
+                    break;
+                case 2:
+                    CInt64_SetULong(&ci50, 16);
+                    maxvalue = 0xFFFF;
+                    bytecount = 2;
+                    break;
+                case 4:
+                    CInt64_SetULong(&ci50, 32);
+                    maxvalue = 0xFFFFFFFF;
+                    bytecount = 4;
+                    break;
+                default:
+#line 1424
+                    CError_FATAL();
+            }
+        } else {
+            CInt64_SetULong(&ci50, 8);
+            maxvalue = 0xFF;
+            bytecount = 1;
+        }
+
+        CInt64_SetULong(&tkintconst, t & maxvalue);
+        i = bytecount;
+        do {
+            if ((t2 == 0 || t2 == '\r') && !was_escchar) {
+                CPrep_Error(100);
+                goto after_char_const;
+            }
+            CInt64_SetULong(&ci48, t2 & maxvalue);
+            ci58 = CInt64_Shl(tkintconst, ci50);
+            tkintconst = CInt64_Or(ci58, ci48);
+            i += bytecount;
+            t2 = nextchar(start);
+            if (t2 == '\'' && !was_escchar)
+                goto after_char_const;
+        } while (i < stunsignedlonglong.size);
+        CPrep_Error(100);
+    after_char_const:
+        if (i <= stunsignedint.size && i < stunsignedlong.size)
+            tksize = IT_UINT;
+        else if (i <= stunsignedlong.size)
+            tksize = IT_ULONG;
+        else
+            tksize = IT_ULONGLONG;
+    }
+
+    return TK_INTCONST;
+}
+
+static short tquot() {
+    // has register allocation issues
+    TypeIntegral *type; // r22
+    char *string_ptr; // r21
+    SInt32 r20; // r20
+    char *save_pos; // r19
+    short c; // r19
+    char *start_pos; // r18
+    CInt64 ch64;
+
+    start_pos = pos;
+    at_linestart = 0;
+    if (cprep_nostring)
+        return '"';
+
+    string_ptr = *stringmem;
+    ispascalstring = 0;
+    r20 = 0;
+    if (widestring)
+        type = &stwchar;
+    else
+        type = &stchar;
+
+    if (prepcurstringchar(start_pos) == '\\') {
+        save_pos = pos;
+        prepcurstringchar_skip();
+        if (prepcurstringchar(start_pos) == 'p') {
+            prepcurstringchar_skip();
+            ispascalstring = 1;
+            string_ptr += type->size;
+        } else {
+            pos = save_pos;
+        }
+    }
+
+    do {
+        c = nextchar(start_pos);
+        if ((c == '"' || c == '\r' || c == 0) && !was_escchar) {
+            if (c == 0 && (tokenstacklevel > 0 || pos >= prep_file_end))
+                CPrep_Error(102);
+            else if (c != '"')
+                CPrep_Error(101);
+            break;
+        }
+        if ((r20 + type->size) >= maxstringsize) {
+            string_ptr = (char *) (string_ptr - *stringmem);
+            if (!COS_ResizeHandle(stringmem, maxstringsize += 256))
+                CError_NoMem();
+            string_ptr = *stringmem + (SInt32) string_ptr;
+        }
+        if (type->size != 1) {
+            CInt64_SetLong(&ch64, c);
+            CMach_InitIntMem((Type *) type, ch64, string_ptr);
+            string_ptr += type->size;
+            r20 += type->size;
+        } else {
+            *string_ptr = c;
+            string_ptr++;
+            r20++;
+        }
+    } while (1);
+
+    if (ispascalstring) {
+        if (r20 > 255 && type->size == 1) {
+            CPrep_Error(106);
+            r20 = 255;
+        }
+        CInt64_SetLong(&ch64, r20 / type->size);
+        CMach_InitIntMem((Type *) type, ch64, *stringmem);
+        r20 += type->size;
+    } else {
+        if ((r20 + 1) >= maxstringsize) {
+            if (!COS_ResizeHandle(stringmem, maxstringsize += 256))
+                CError_NoMem();
+            CMach_InitIntMem((Type *) type, cint64_zero, *stringmem + r20);
+            r20 += type->size;
+        }
+    }
+
+    tksize = r20;
+    tkstring = galloc(r20);
+    memcpy(tkstring, *stringmem, tksize);
+    if (widestring)
+        return TK_STRING_WIDE;
+    else
+        return TK_STRING;
+}
+
+static short thash() {
+    if (at_linestart) {
+        preprocessor();
+        return 0;
+    }
+    return '#';
+}
+
+static short tmult() {
+    at_linestart = 0;
+    if (prepcurchar() == '=') {
+        pos = nextcharpos;
+        return TK_MULT_ASSIGN;
+    }
+    return '*';
+}
+
+static short tcolo() {
+    char t2;
+    at_linestart = 0;
+    if (copts.cplusplus) {
+        t2 = prepcurchar();
+        if (t2 == ':') {
+            pos = nextcharpos;
+            return TK_COLON_COLON;
+        }
+        if (t2 == '>' && !preprocessing_only) {
+            pos = nextcharpos;
+            return ']';
+        }
+    }
+    return ':';
+}
+
+static short tless() {
+    short t2;
+    at_linestart = 0;
+    t2 = prepcurchar();
+    if (t2 == '=') {
+        pos = nextcharpos;
+        return TK_LESS_EQUAL;
+    }
+    if (t2 == '<') {
+        pos = nextcharpos;
+        if (prepcurchar() == '=') {
+            pos = nextcharpos;
+            return TK_SHL_ASSIGN;
+        } else {
+            return TK_SHL;
+        }
+    }
+    if (t2 == '%' && copts.cplusplus && !preprocessing_only) {
+        pos = nextcharpos;
+        return '{';
+    }
+    if (t2 == ':' && copts.cplusplus && !preprocessing_only) {
+        pos = nextcharpos;
+        return '[';
+    }
+    return '<';
+}
+
+static short tequa(short) {
+    at_linestart = 0;
+    if (prepcurchar() == '=') {
+        pos = nextcharpos;
+        return TK_LOGICAL_EQ;
+    }
+    return '=';
+}
+
+static short tgrea(short) {
+    short t2;
+    at_linestart = 0;
+    t2 = prepcurchar();
+    if (t2 == '=') {
+        pos = nextcharpos;
+        return TK_GREATER_EQUAL;
+    }
+    if (t2 == '>') {
+        pos = nextcharpos;
+        if (prepcurchar() == '=') {
+            pos = nextcharpos;
+            return TK_SHR_ASSIGN;
+        } else {
+            return TK_SHR;
+        }
+    }
+    return '>';
+}
+
+static short tatsg(short) {
+    char *start_pos;
+    Boolean save_idarray;
+
+    start_pos = pos;
+    at_linestart = 0;
+
+    if (copts.objective_c || in_assembler) {
+        save_idarray = cprep_idarray['@'];
+        cprep_idarray['@'] = 1;
+        pos = ReadIdentifier(start_pos - 1);
+        cprep_idarray['@'] = save_idarray;
+
+        if (copts.objective_c) {
+            if (!strcmp("@interface", tkidentifier->name))
+                return TK_AT_INTERFACE;
+            if (!strcmp("@implementation", tkidentifier->name))
+                return TK_AT_IMPLEMENTATION;
+            if (!strcmp("@protocol", tkidentifier->name))
+                return TK_AT_PROTOCOL;
+            if (!strcmp("@end", tkidentifier->name))
+                return TK_AT_END;
+            if (!strcmp("@private", tkidentifier->name))
+                return TK_AT_PRIVATE;
+            if (!strcmp("@protected", tkidentifier->name))
+                return TK_AT_PROTECTED;
+            if (!strcmp("@public", tkidentifier->name))
+                return TK_AT_PUBLIC;
+            if (!strcmp("@class", tkidentifier->name))
+                return TK_AT_CLASS;
+            if (!strcmp("@selector", tkidentifier->name))
+                return TK_AT_SELECTOR;
+            if (!strcmp("@encode", tkidentifier->name))
+                return TK_AT_ENCODE;
+            if (!strcmp("@defs", tkidentifier->name))
+                return TK_AT_DEFS;
+        }
+
+        if (in_assembler)
+            return TK_IDENTIFIER;
+
+        pos = start_pos;
+    }
+
+    return '@';
+}
+
+static short tperc(short) {
+    char t2;
+    at_linestart = 0;
+    t2 = prepcurchar();
+    if (t2 == '=') {
+        pos = nextcharpos;
+        return TK_MOD_ASSIGN;
+    }
+    if (t2 == '>' && copts.cplusplus && !preprocessing_only) {
+        pos = nextcharpos;
+        return '}';
+    }
+    return '%';
+}
+
+static short texcl() {
+    at_linestart = 0;
+    if (prepcurchar() == '=') {
+        pos = nextcharpos;
+        return TK_LOGICAL_NE;
+    }
+    return '!';
+}
+
+static short tplus() {
+    short t2;
+    at_linestart = 0;
+    t2 = prepcurchar();
+    if (t2 == '=') {
+        pos = nextcharpos;
+        return TK_ADD_ASSIGN;
+    }
+    if (t2 == '+') {
+        pos = nextcharpos;
+        return TK_INCREMENT;
+    }
+    return '+';
+}
+
+static short tminu() {
+    short t2;
+    at_linestart = 0;
+    t2 = prepcurchar();
+    if (t2 == '=') {
+        pos = nextcharpos;
+        return TK_SUB_ASSIGN;
+    }
+    if (t2 == '-') {
+        pos = nextcharpos;
+        return TK_DECREMENT;
+    }
+    if (t2 == '>') {
+        pos = nextcharpos;
+        if (copts.cplusplus && prepcurchar() == '*') {
+            pos = nextcharpos;
+            return TK_ARROW_STAR;
+        }
+        return TK_ARROW;
+    }
+    return '-';
+}
+
+static short torrr() {
+    short t2;
+    at_linestart = 0;
+    t2 = prepcurchar();
+    if (t2 == '=') {
+        pos = nextcharpos;
+        return TK_OR_ASSIGN;
+    }
+    if (t2 == '|') {
+        pos = nextcharpos;
+        return TK_LOGICAL_OR;
+    }
+    return '|';
+}
+
+static short tampe() {
+    short t2;
+    at_linestart = 0;
+    t2 = prepcurchar();
+    if (t2 == '=') {
+        pos = nextcharpos;
+        return TK_AND_ASSIGN;
+    }
+    if (t2 == '&') {
+        pos = nextcharpos;
+        return TK_LOGICAL_AND;
+    }
+    return '&';
+}
+
+static short tpowe() {
+    at_linestart = 0;
+    if (prepcurchar() == '=') {
+        pos = nextcharpos;
+        return TK_XOR_ASSIGN;
+    }
+    return '^';
+}
+
+static short tdivi() {
+    at_linestart = 0;
+    if (prepcurchar() == '=') {
+        pos = nextcharpos;
+        return TK_DIV_ASSIGN;
+    }
+    return '/';
 }
 
-static short tille() {}
-static short tcret() {}
-static short tapos() {}
-static short tquot() {}
-static short thash() {}
-static short tmult() {}
-static short tcolo() {}
-static short tless() {}
-static short tequa() {}
-static short tgrea() {}
-static short tatsg() {}
-static short tperc() {}
-static short texcl() {}
-static short tplus() {}
-static short tminu() {}
-static short torrr() {}
-static short tampe() {}
-static short tpowe() {}
-static short tdivi() {}
 static short tzero() {}
 static short tpoin() {}
 static short tnumb() {}
-static short tiden() {}
+
+static short tiden() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    else
+        return TK_IDENTIFIER;
+}
+
 static short tchrL() {}
-static short tchra() {}
-static short tchrb() {}
-static short tchrc() {}
-static short tchrd() {}
-static short tchre() {}
-static short tchrf() {}
-static short tchrg() {}
-static short tchri() {}
-static short tchrl() {}
-static short tchrm() {}
-static short tchrn() {}
-static short tchro() {}
-static short tchrp() {}
-static short tchrr() {}
-static short tchrs() {}
-static short tchrt() {}
-static short tchru() {}
-static short tchrv() {}
-static short tchrw() {}
-static short tchrx() {}
-static short tchr_() {}
-static short token() {}
-static short tdoll() {}
-static short tisid() {}
-static short tnull() {}
-static short t0x1a() {}
-
-static short (*cprep_tokenize[256])() = {
-        &tnull, &tille, &tille, &tille, &tisid, &tisid, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tcret, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &t0x1a, &tille, &tille, &tille, &tille, &tille,
-        &tille, &texcl, &tquot, &thash, &tdoll, &tperc, &tampe, &tapos,
-        &token, &token, &tmult, &tplus, &token, &tminu, &tpoin, &tdivi,
-        &tzero, &tnumb, &tnumb, &tnumb, &tnumb, &tnumb, &tnumb, &tnumb,
-        &tnumb, &tnumb, &tcolo, &token, &tless, &tequa, &tgrea, &token,
-        &tatsg, &tiden, &tiden, &tiden, &tiden, &tiden, &tiden, &tiden,
-        &tiden, &tiden, &tiden, &tiden, &tchrL, &tiden, &tchrn, &tiden,
-        &tchrp, &tiden, &tiden, &tiden, &tiden, &tiden, &tiden, &tiden,
-        &tchrx, &tiden, &tiden, &token, &token, &token, &tpowe, &tchr_,
-        &token, &tchra, &tchrb, &tchrc, &tchrd, &tchre, &tchrf, &tchrg,
-        &tiden, &tchri, &tiden, &tiden, &tchrl, &tchrm, &tchrn, &tchro,
-        &tchrp, &tiden, &tchrr, &tchrs, &tchrt, &tchru, &tchrv, &tchrw,
-        &tchrx, &tiden, &tiden, &token, &torrr, &token, &token, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille,
-        &tille, &tille, &tille, &tille, &tille, &tille, &tille, &tille
+
+static short tchra() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("auto", tkidentifier->name))
+        return TK_AUTO;
+    if ((copts.cplusplus || !copts.only_std_keywords) && !strcmp("asm", tkidentifier->name))
+        return TK_ASM;
+    if (copts.cplusplus && !in_assembler && !preprocessing_only) {
+        if (!strcmp("and", tkidentifier->name))
+            return TK_LOGICAL_AND;
+        if (!strcmp("and_eq", tkidentifier->name))
+            return TK_AND_ASSIGN;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrb() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("break", tkidentifier->name))
+        return TK_BREAK;
+    if (copts.cplusplus) {
+        if (copts.booltruefalse && !strcmp("bool", tkidentifier->name))
+            return TK_BOOL;
+        if (!in_assembler && !preprocessing_only) {
+            if (!strcmp("bitand", tkidentifier->name))
+                return '&';
+            if (!strcmp("bitor", tkidentifier->name))
+                return '|';
+        }
+    }
+    if (copts.objective_c) {
+        if (!strcmp("bycopy", tkidentifier->name))
+            return TK_BYCOPY;
+        if (!strcmp("byref", tkidentifier->name))
+            return TK_BYREF;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrc() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("case", tkidentifier->name))
+        return TK_CASE;
+    if (!strcmp("char", tkidentifier->name))
+        return TK_CHAR;
+    if (!strcmp("const", tkidentifier->name))
+        return TK_CONST;
+    if (!strcmp("continue", tkidentifier->name))
+        return TK_CONTINUE;
+    if (copts.cplusplus) {
+        if (!strcmp("const_cast", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_CONST_CAST;
+        }
+        if (!strcmp("catch", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_CATCH;
+        }
+        if (!strcmp("class", tkidentifier->name))
+            return TK_CLASS;
+        if (!in_assembler && !preprocessing_only) {
+            if (!strcmp("compl", tkidentifier->name))
+                return '~';
+        }
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrd() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("default", tkidentifier->name))
+        return TK_DEFAULT;
+    if (!strcmp("do", tkidentifier->name))
+        return TK_DO;
+    if (!strcmp("double", tkidentifier->name))
+        return TK_DOUBLE;
+    if (copts.cplusplus) {
+        if (!strcmp("delete", tkidentifier->name))
+            return TK_DELETE;
+        if (!strcmp("dynamic_cast", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_DYNAMIC_CAST;
+        }
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchre() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("else", tkidentifier->name))
+        return TK_ELSE;
+    if (!strcmp("enum", tkidentifier->name))
+        return TK_ENUM;
+    if (!strcmp("extern", tkidentifier->name))
+        return TK_EXTERN;
+    if (copts.cplusplus) {
+        if (!strcmp("explicit", tkidentifier->name))
+            return TK_EXPLICIT;
+        if (!strcmp("export", tkidentifier->name) && !copts.ecplusplus)
+            return TK_EXPORT;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrf() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("float", tkidentifier->name))
+        return TK_FLOAT;
+    if (!strcmp("for", tkidentifier->name))
+        return TK_FOR;
+    if (copts.cplusplus) {
+        if (!strcmp("friend", tkidentifier->name))
+            return TK_FRIEND;
+        if (copts.booltruefalse && !strcmp("false", tkidentifier->name))
+            return TK_FALSE;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrg() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("goto", tkidentifier->name))
+        return TK_GOTO;
+    return TK_IDENTIFIER;
+}
+
+static short tchri() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("if", tkidentifier->name))
+        return TK_IF;
+    if (!strcmp("int", tkidentifier->name))
+        return TK_INT;
+    if (copts.cplusplus || copts.c9x) {
+        if (!strcmp("inline", tkidentifier->name))
+            return TK_INLINE;
+    } else if (!copts.only_std_keywords) {
+        if (!strcmp("inline", tkidentifier->name))
+            return TK_INLINE;
+    }
+    if (copts.objective_c) {
+        if (!strcmp("in", tkidentifier->name))
+            return TK_IN;
+        if (!strcmp("inout", tkidentifier->name))
+            return TK_INOUT;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrl() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("long", tkidentifier->name))
+        return TK_LONG;
+    return TK_IDENTIFIER;
+}
+
+static short tchrm() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (copts.cplusplus && !strcmp("mutable", tkidentifier->name)) {
+        if (copts.ecplusplus)
+            CPrep_Warning(339);
+        return TK_MUTABLE;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrn(short t) {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (copts.cplusplus && t == 'n') {
+        if (!strcmp("new", tkidentifier->name))
+            return TK_NEW;
+        if (!strcmp("namespace", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_NAMESPACE;
+        }
+        if (!in_assembler && !preprocessing_only) {
+            if (!strcmp("not", tkidentifier->name))
+                return '!';
+            if (!strcmp("not_eq", tkidentifier->name))
+                return TK_LOGICAL_NE;
+        }
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchro() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (copts.cplusplus) {
+        if (!strcmp("operator", tkidentifier->name))
+            return TK_OPERATOR;
+        if (!in_assembler && !preprocessing_only) {
+            if (!strcmp("or", tkidentifier->name))
+                return TK_LOGICAL_OR;
+            if (!strcmp("or_eq", tkidentifier->name))
+                return TK_OR_ASSIGN;
+        }
+    }
+    if (copts.objective_c) {
+        if (!strcmp("out", tkidentifier->name))
+            return TK_OUT;
+        if (!strcmp("oneway", tkidentifier->name))
+            return TK_ONEWAY;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrp(short t) {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!copts.only_std_keywords && !strcmp("pascal", tkidentifier->name))
+        return TK_PASCAL;
+    if (copts.cplusplus && t == 'p') {
+        if (!strcmp("private", tkidentifier->name))
+            return TK_PRIVATE;
+        if (!strcmp("protected", tkidentifier->name))
+            return TK_PROTECTED;
+        if (!strcmp("public", tkidentifier->name))
+            return TK_PUBLIC;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrr() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("register", tkidentifier->name))
+        return TK_REGISTER;
+    if (!strcmp("return", tkidentifier->name))
+        return TK_RETURN;
+    if (copts.cplusplus && !strcmp("reinterpret_cast", tkidentifier->name)) {
+        if (copts.ecplusplus)
+            CPrep_Warning(339);
+        return TK_REINTERPRET_CAST;
+    }
+    if (copts.c9x && !strcmp("restrict", tkidentifier->name))
+        return TK_RESTRICT;
+    return TK_IDENTIFIER;
+}
+
+static short tchrs() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("short", tkidentifier->name))
+        return TK_SHORT;
+    if (!strcmp("signed", tkidentifier->name))
+        return TK_SIGNED;
+    if (!strcmp("sizeof", tkidentifier->name))
+        return TK_SIZEOF;
+    if (!strcmp("static", tkidentifier->name))
+        return TK_STATIC;
+    if (!strcmp("struct", tkidentifier->name))
+        return TK_STRUCT;
+    if (!strcmp("switch", tkidentifier->name))
+        return TK_SWITCH;
+    if (copts.cplusplus && !strcmp("static_cast", tkidentifier->name)) {
+        if (copts.ecplusplus)
+            CPrep_Warning(339);
+        return TK_STATIC_CAST;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrt() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("typedef", tkidentifier->name))
+        return TK_TYPEDEF;
+    if (copts.gcc_extensions && !strcmp("typeof", tkidentifier->name))
+        return TK_UU_TYPEOF_UU;
+    if (copts.cplusplus) {
+        if (!strcmp("this", tkidentifier->name))
+            return TK_THIS;
+        if (copts.booltruefalse && !strcmp("true", tkidentifier->name))
+            return TK_TRUE;
+        if (!strcmp("template", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_TEMPLATE;
+        }
+        if (!strcmp("try", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_TRY;
+        }
+        if (!strcmp("throw", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_THROW;
+        }
+        if (!strcmp("typeid", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_TYPEID;
+        }
+        if (!strcmp("typename", tkidentifier->name)) {
+            if (copts.ecplusplus)
+                CPrep_Warning(339);
+            return TK_TYPENAME;
+        }
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchru() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("union", tkidentifier->name))
+        return TK_UNION;
+    if (!strcmp("unsigned", tkidentifier->name))
+        return TK_UNSIGNED;
+    if (copts.cplusplus && !strcmp("using", tkidentifier->name)) {
+        if (copts.ecplusplus)
+            CPrep_Warning(339);
+        return TK_USING;
+    }
+    return TK_IDENTIFIER;
+}
+
+static short tchrv() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("void", tkidentifier->name))
+        return TK_VOID;
+    if (!strcmp("volatile", tkidentifier->name))
+        return TK_VOLATILE;
+    if (copts.cplusplus && !strcmp("virtual", tkidentifier->name))
+        return TK_VIRTUAL;
+    return TK_IDENTIFIER;
+}
+
+static short tchrw() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+    if (!strcmp("while", tkidentifier->name))
+        return TK_WHILE;
+    if (copts.cplusplus && copts.wchar_type && !strcmp("wchar_t", tkidentifier->name))
+        return TK_WCHAR_T;
+    return TK_IDENTIFIER;
+}
+
+static short tchrx(short t) {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+
+    if (copts.cplusplus && t == 'x' && !in_assembler && !preprocessing_only) {
+        if (!strcmp("xor", tkidentifier->name))
+            return '^';
+        if (!strcmp("xor_eq", tkidentifier->name))
+            return TK_XOR_ASSIGN;
+    }
+
+    return TK_IDENTIFIER;
+}
+
+static short tchr_() {
+    at_linestart = 0;
+    pos = ReadIdentifier(pos - 1);
+    if (macrocheck && macrotest())
+        return 0;
+
+    if (!strcmp("__stdcall", tkidentifier->name))
+        return TK_UU_STDCALL;
+    if (!strcmp("__cdecl", tkidentifier->name))
+        return TK_UU_CDECL;
+    if (!strcmp("__fastcall", tkidentifier->name))
+        return TK_UU_FASTCALL;
+    if (!strcmp("__declspec", tkidentifier->name))
+        return TK_UU_DECLSPEC;
+    if (!strcmp("__asm", tkidentifier->name))
+        return TK_ASM;
+    if (!strcmp("__asm__", tkidentifier->name))
+        return TK_ASM;
+    if (!strcmp("__inline", tkidentifier->name))
+        return TK_INLINE;
+    if (!strcmp("__inline__", tkidentifier->name))
+        return TK_INLINE;
+    if (!strcmp("__restrict", tkidentifier->name))
+        return TK_RESTRICT;
+    if (!strcmp("__restrict__", tkidentifier->name))
+        return TK_RESTRICT;
+    if (!strcmp("__attribute__", tkidentifier->name))
+        return TK_UU_ATTRIBUTE_UU;
+    if (!strcmp("__far", tkidentifier->name))
+        return TK_UU_FAR;
+    if (!strcmp("__alignof__", tkidentifier->name))
+        return TK_UU_ALIGNOF_UU;
+    if (!strcmp("__typeof__", tkidentifier->name))
+        return TK_UU_TYPEOF_UU;
+    if (copts.c9x) {
+        if (!strcmp("_Bool", tkidentifier->name))
+            return TK_BOOL;
+        if (!strcmp("_Complex", tkidentifier->name))
+            return TK_U_COMPLEX;
+        if (!strcmp("_Imaginary", tkidentifier->name))
+            return TK_U_IMAGINARY;
+    }
+    if (copts.altivec_model && !strcmp("__vector", tkidentifier->name))
+        return TK_UU_VECTOR;
+    return TK_IDENTIFIER;
+}
+
+static short token(short t) {
+    at_linestart = 0;
+    return t;
+}
+
+static short tdoll(short t) {
+    at_linestart = 0;
+    return t;
+}
+
+static short tisid() {
+    if (!tokenstacklevel)
+        CError_Error(105);
+    return 0;
+}
+
+static short tnull(short t) {
+    if (tokenstacklevel > 0 || pos >= prep_file_end) {
+        at_linestart = 0;
+        return t;
+    } else {
+        pos[-1] = -64;
+        return TK_NEG6;
+    }
+}
+
+static short t0x1a() {
+    if (*pos == 0)
+        return 0;
+    else
+        return TK_NEG6;
+}
+
+typedef short (*TokenizePtr)(short);
+#define F(func) ((TokenizePtr) (&func))
+
+static TokenizePtr cprep_tokenize[256] = {
+        F(tnull), F(tille), F(tille), F(tille), F(tisid), F(tisid), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tcret), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(t0x1a), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(texcl), F(tquot), F(thash), F(tdoll), F(tperc), F(tampe), F(tapos),
+        F(token), F(token), F(tmult), F(tplus), F(token), F(tminu), F(tpoin), F(tdivi),
+        F(tzero), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb), F(tnumb),
+        F(tnumb), F(tnumb), F(tcolo), F(token), F(tless), F(tequa), F(tgrea), F(token),
+        F(tatsg), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden),
+        F(tiden), F(tiden), F(tiden), F(tiden), F(tchrL), F(tiden), F(tchrn), F(tiden),
+        F(tchrp), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden), F(tiden),
+        F(tchrx), F(tiden), F(tiden), F(token), F(token), F(token), F(tpowe), F(tchr_),
+        F(token), F(tchra), F(tchrb), F(tchrc), F(tchrd), F(tchre), F(tchrf), F(tchrg),
+        F(tiden), F(tchri), F(tiden), F(tiden), F(tchrl), F(tchrm), F(tchrn), F(tchro),
+        F(tchrp), F(tiden), F(tchrr), F(tchrs), F(tchrt), F(tchru), F(tchrv), F(tchrw),
+        F(tchrx), F(tiden), F(tiden), F(token), F(torrr), F(token), F(token), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille),
+        F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille), F(tille)
 };
 
 short lookahead() {
+    SInt32 state;
+    short t;
+
+    CPrep_TokenStreamGetState(&state);
+    t = lex();
+    CPrep_TokenStreamSetState(&state);
+
+    return t;
 }
 
 short lookahead_noeol() {
+    SInt32 state;
+    short t;
+
+    CPrep_TokenStreamGetState(&state);
+    do {
+        t = lex();
+    } while (t == TK_NEG7);
+    CPrep_TokenStreamSetState(&state);
+
+    return t;
 }
 
 static void CPrep_StringConCat(Boolean flag) {
@@ -160,7 +1893,45 @@ short lex() {
 }
 
 short plex() {
+    // does not match - global refs in wrong order
+    short t;
+
+    while (1) {
+        t = prepskipnextchar();
+        pos = nextcharpos;
+        if ((t = cprep_tokenize[t](t)))
+            return t;
+
+        if (!*pos) {
+            if (tokenstacklevel > 0) {
+                poptokenseq();
+            } else if (filesp > 0) {
+                popfile();
+            } else {
+                break;
+            }
+        }
+    }
+
+    return 0;
 }
 
 short lexidentifier() {
+    short t;
+    char *save_pos;
+
+    t = prepskipnextchar();
+    pos = nextcharpos;
+    at_linestart = 0;
+    if ((t >= 'a' && t <= 'z') || (t >= 'A' && t <= 'Z') || (t == '_')) {
+        pos = ReadIdentifier(pos - 1);
+        return TK_IDENTIFIER;
+    } else if (t == '.') {
+        save_pos = pos;
+        if (prepnextchar() == '.' && prepnextchar() == '.')
+            return TK_ELLIPSIS;
+        else
+            pos = save_pos;
+    }
+    return t;
 }