From b9266cdf96f410061b08460651669aefc499cab6 Mon Sep 17 00:00:00 2001 From: Ellen Arvidsson Date: Tue, 15 Apr 2025 20:02:25 +0200 Subject: [PATCH] lexing up and running --- CMakeLists.txt | 12 +- include/msph/err.h | 20 ++ src/msph/msph_token.h => include/msph/token.h | 31 +- include/spho/err.h | 51 ++-- src/msph/msph.c | 153 ++++++++-- src/msph/{msph_token.c => token.c} | 269 ++++++++++++++---- 6 files changed, 434 insertions(+), 102 deletions(-) create mode 100644 include/msph/err.h rename src/msph/msph_token.h => include/msph/token.h (80%) rename src/msph/{msph_token.c => token.c} (59%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 34e7f44..a972050 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,14 +46,18 @@ target_compile_definitions(devcheck PRIVATE set(MSPH_SRC - ${SRC_DIR}/msph/msph_token.c + ${SRC_DIR}/msph/token.c ${SRC_DIR}/msph/msph.c ) set(MSPH_HEADER - ${SRC_DIR}/msph/msph_token.h + ${INCLUDE_DIR}/msph/token.h ) -add_executable(msph ${MSPH_HEADER} ${MSPH_SRC}) +add_executable(msph ${MSPH_SRC} ${MSPH_HEADER}) target_include_directories(msph PRIVATE ${INCLUDE_DIR}) -target_link_libraries(devcheck spho) +target_link_libraries(msph spho) +target_compile_definitions(msph PRIVATE + $<$:SPHO_DEBUG> + $<$:SPHO_USE_STRLCPY> +) diff --git a/include/msph/err.h b/include/msph/err.h new file mode 100644 index 0000000..8fa7795 --- /dev/null +++ b/include/msph/err.h @@ -0,0 +1,20 @@ +#ifndef _MSPH_ERR_H +#define _MSPH_ERR_H + +#ifdef SPHO_DEBUG +// #define SPHO_ENABLE_DEBUG_PRINT +#endif + +#include "spho/err.h" + + +#define MSPH_ERR_SYS 0x0001 + +#define MSPH_ERR_INVAL 0x1001 + +#define MSPH_ERR_TOOLONG 0x2001 + +#define MSPH_ERR(ctx, e) SPHO_ERR(ctx, e) +#define MSPH_TOKS_ERR(toks, e) MSPH_ERR((toks)->ctx, e) + +#endif diff --git a/src/msph/msph_token.h b/include/msph/token.h similarity index 80% rename from src/msph/msph_token.h rename to include/msph/token.h index 7207052..93dc21a 100644 --- a/src/msph/msph_token.h +++ b/include/msph/token.h @@ -80,12 +80,17 @@ */ struct msph_ctx { - int err; - int err_info; + int err; + int err_info; + char errbuf[SPHO_ERR_BUF_LEN]; + +#ifdef SPHO_DEBUG + char filebuf[SPHO_ERR_FILEBUF_LEN]; + int errline; +#endif }; enum msph_tok_type { - TOK_START, TOK_LBRACE, // { TOK_RBRACE, // } TOK_LBRAK, // [ @@ -111,13 +116,14 @@ enum msph_tok_type { TOK_CONST_FALSE, // False TOK_IDENT, // identifiers - TOK_END + TOK_WSPACE, // special: whitespace + TOK_END // special: denote end of array }; -#define MSPH_TOKEN_BUF_SZ 128 +#define MSPH_TOKEN_BUF_LEN 128 struct token_s { - char buf[MSPH_TOKEN_BUF_SZ]; + char buf[MSPH_TOKEN_BUF_LEN]; }; union token_data { @@ -129,7 +135,7 @@ struct msph_token { union token_data d; }; -#define MSPH_PATH_LEN 1024 +#define MSPH_FILE_NAME_LEN 1024 #define MSPH_FILE_BUF_LEN 4096 #define MSPH_TOKEN_SRC_FILE 1 @@ -142,7 +148,7 @@ struct msph_token_src_file { char buf[MSPH_FILE_BUF_LEN]; /* file path */ - char name[MSPH_PATH_LEN]; + char name[MSPH_FILE_NAME_LEN]; }; #define MSPH_TOKEN_SRC_STR 2 @@ -168,8 +174,10 @@ struct msph_token_stream { struct msph_token_src src; }; -struct msph_token_stream *msph_token_stream_fopen(struct msph_ctx *, - const char *); +void msph_ctx_init(struct msph_ctx *); + +struct msph_token_stream *msph_token_stream_file(struct msph_ctx *, + FILE *, const char *); struct msph_token_stream *msph_token_stream_frombuf(struct msph_ctx *, const char *, size_t); @@ -177,6 +185,9 @@ int msph_token_stream_close(struct msph_token_stream*); ssize_t msph_token_stream_read_tok(struct msph_token *, size_t, struct msph_token_stream *); +int msph_token_stream_print(struct msph_token_stream *, FILE *); + +size_t msph_token_str(char *buf, size_t len, struct msph_token *tok); #endif /* _MSPH_EXPR_H */ diff --git a/include/spho/err.h b/include/spho/err.h index b4dcc33..0dd651d 100644 --- a/include/spho/err.h +++ b/include/spho/err.h @@ -16,8 +16,14 @@ #define SPHO_ERR_IS_SYSERR(err) (SPHO_ERR_SYS == err) +#define SPHO_ERR_BUF_LEN 2048 + + + #ifdef SPHO_DEBUG +#define SPHO_ERR_FILEBUF_LEN 128 + #define SPHO_ERR(ctx, e) \ do { \ (ctx)->err = (e); \ @@ -28,20 +34,6 @@ (ctx)->errline = __LINE__; \ } while (0) -#else /* SPHO_DEBUG */ - -#define SPHO_ERR(ctx, e) \ - do { \ - (ctx)->err = (e); \ - if ((e) == SPHO_ERR_SYS) \ - (ctx)->err_info = errno; \ - } while (0) - -#endif /* SPHO_DEBUG */ - -/* debug stuff */ -#ifdef SPHO_DEBUG - #define SPHO_STRINGIFY(a) #a #define SPHO_MACRO_STR(b) SPHO_STRINGIFY(b) @@ -77,10 +69,35 @@ } \ } while (0) -#else + +#ifdef SPHO_ENABLE_DEBUG_PRINT + +#define SPHO_DEBUG_PRINT(fmt, ...) \ + do { \ + fprintf(stderr, fmt __VA_OPT__(,) __VA_ARGS__); \ + } while (0) + +#else /* SPHO_ENABLE_DEBUG_PRINT */ + +#define SPHO_DEBUG_PRINT(fmt, ...) + +#endif /* SPHO_ENABLE_DEBUG_PRINT */ + +#else /* SPHO_DEBUG */ + +#define SPHO_ERR(ctx, e) \ + do { \ + (ctx)->err = (e); \ + if ((e) == SPHO_ERR_SYS) \ + (ctx)->err_info = errno; \ + } while (0) + #define SPHO_PRECOND(cond) #define SPHO_ASSERT(cond) #define SPHO_POSTCOND(cond) -#endif -#endif +#define SPHO_DBG_PRINT(fmt, ...) + +#endif /* SPHO_DEBUG */ + +#endif /* _SPHO_ERR_H */ diff --git a/src/msph/msph.c b/src/msph/msph.c index 8643a46..b85945c 100644 --- a/src/msph/msph.c +++ b/src/msph/msph.c @@ -1,40 +1,147 @@ +#include #include +#include - - -#include "msph_token.h" +#include "msph/err.h" +#include "msph/token.h" #define PRINTERR(ctx) \ do { \ fprintf(stderr, "msph_err: %d (%d)\n", (ctx).err, (ctx).err_info); \ } while (0) +#ifdef SPHO_DEBUG +#define DEBUGFLAG "d" +#endif + +struct msph_opts { + int tokenize_only; +#ifdef DEBUGFLAG + int debug; +#endif + const char *in_path; + const char *out_path; +} run_opts = { + 0, +#ifdef DEBUGFLAG + 0, +#endif + NULL, + NULL +}; + + +#define USAGE \ + "msph: micro sappho\n" \ + "\tUsage: msph [-t] [-o out_path] [in_path]\n" \ + "\n" \ + "\t-t - print tokenization\n" \ + "\t-o - output path\n" + + +void +print_usage(void) +{ + printf("%s", USAGE); +} + +int run(struct msph_opts *); int -main(int argc, char **argv) +main(int argc, char *argv[]) { - int ret; - char *path; - struct msph_ctx ctx; - struct msph_token_stream *s; + int opt; - - if (argc != 2) - return (-1); - - - path = argv[1]; - - ctx.err = 0; - ctx.err_info = 0; - - if ((s = msph_token_stream_fopen(&ctx, path)) == NULL) { - PRINTERR(ctx); - return (-1); + while ((opt = getopt(argc, argv, DEBUGFLAG "to:")) != -1) { + switch (opt) { + case 't': + run_opts.tokenize_only = 1; + break; + case 'o': + run_opts.out_path = optarg; + break; +#ifdef DEBUGFLAG + case 'd': + run_opts.debug = 1; + break; +#endif + case '?': + default: + print_usage(); + exit(EXIT_FAILURE); + } } + argc -= optind; + argv += optind; - ret = msph_token_stream_close(s); + if (argc > 1) { + print_usage(); + exit(EXIT_FAILURE); + } - return (ret); + if (argc == 1) + run_opts.in_path = argv[0]; + + if (run(&run_opts)) + exit(EXIT_FAILURE); +} + + +int +run(struct msph_opts *opts) +{ + int ret; + struct msph_ctx ctx; + struct msph_token_stream *s; + FILE *in, *out; + + in = out = NULL; + + msph_ctx_init(&ctx); + + if (opts->in_path == NULL) { + in = stdin; opts->in_path = "stdin"; + } else { + if ((in = fopen(opts->in_path, "r")) == NULL) { + MSPH_ERR(&ctx, MSPH_ERR_SYS); + goto err; + } + } + + if (opts->out_path == NULL) { + out = stdout; + } else { + if ((out = fopen(opts->out_path, "w")) == NULL) { + MSPH_ERR(&ctx, MSPH_ERR_SYS); + goto err; + } + } + + if ((s = msph_token_stream_file(&ctx, in, opts->in_path)) == NULL) { + goto err; + } + + printf("msph v0, parsing %s\n", opts->in_path); + + if ((ret = msph_token_stream_print(s, out)) == -1) + goto err; + + if (opts->tokenize_only) + goto exit; + +exit: + if (msph_token_stream_close(s) == -1) + goto err; + + return (0); + +err: + PRINTERR(ctx); + + if (in) + fclose(in); + if (out) + fclose(out); + return (-1); } diff --git a/src/msph/msph_token.c b/src/msph/token.c similarity index 59% rename from src/msph/msph_token.c rename to src/msph/token.c index f69c8b6..959546f 100644 --- a/src/msph/msph_token.c +++ b/src/msph/token.c @@ -5,34 +5,94 @@ #include #include -#include "spho/err.h" - -#include "msph_token.h" +#include "msph/err.h" +#include "msph/token.h" -#define MSPH_ERR_SYS 0x0001 -#define MSPH_ERR_TOOLONG 0x0002 - -#define MSPH_ERR(ctx, e) SPHO_ERR(ctx, e) -#define MSPH_TOKS_ERR(toks, e) MSPH_ERR((toks)->ctx, e) - -struct msph_token_matcher { +struct msph_matcher { size_t off; size_t matchlen; const int type; -} msph_matcher[] = { - { 0, 0, TOK_START }, +}; + +struct msph_matcher token_matcher[] = { + { 0, 0, -1 }, + { 0, 0, TOK_LBRACE }, + { 0, 0, TOK_RBRACE }, + { 0, 0, TOK_LBRAK }, + { 0, 0, TOK_RBRAK }, + { 0, 0, TOK_LPAREN }, + { 0, 0, TOK_RPAREN }, + { 0, 0, TOK_COLON }, + { 0, 0, TOK_EQUALS }, + + { 0, 0, TOK_AMP }, + { 0, 0, TOK_PIPE }, + { 0, 0, TOK_RARROW }, + { 0, 0, TOK_SUB }, + + { 0, 0, TOK_KW_TYPE }, + { 0, 0, TOK_KW_NOMINAL }, + { 0, 0, TOK_KW_MEMBER }, + { 0, 0, TOK_KW_CHECK }, + { 0, 0, TOK_KW_BOX }, + { 0, 0, TOK_KW_FORALL }, + + { 0, 0, TOK_CONST_TRUE }, + { 0, 0, TOK_CONST_FALSE }, + { 0, 0, TOK_IDENT }, { 0, 0, TOK_END } }; +struct msph_matcher wspace = { 0, 0, TOK_WSPACE }; + +struct msph_token_info { + const int type; + const char *dbg_str; + const char *str; +} token_info[] = { +#define TOK_INFO(tok, s) { tok , #tok, s } + TOK_INFO(TOK_LBRACE, "{"), + TOK_INFO(TOK_RBRACE, "}"), + TOK_INFO(TOK_LBRAK, "["), + TOK_INFO(TOK_RBRAK, "]"), + TOK_INFO(TOK_LPAREN, "("), + TOK_INFO(TOK_RPAREN, ")"), + TOK_INFO(TOK_COLON, ":"), + TOK_INFO(TOK_EQUALS, "="), + + TOK_INFO(TOK_AMP, "&"), + TOK_INFO(TOK_PIPE, "|"), + TOK_INFO(TOK_RARROW, "=>"), + TOK_INFO(TOK_SUB, "<:"), + + TOK_INFO(TOK_KW_TYPE, "type"), + TOK_INFO(TOK_KW_NOMINAL, "nominal"), + TOK_INFO(TOK_KW_MEMBER, "member"), + TOK_INFO(TOK_KW_CHECK, "check"), + TOK_INFO(TOK_KW_BOX, "box"), + TOK_INFO(TOK_KW_FORALL, "forall"), + + TOK_INFO(TOK_CONST_TRUE, "True"), + TOK_INFO(TOK_CONST_FALSE, "False"), + + TOK_INFO(TOK_IDENT, NULL), + TOK_INFO(TOK_WSPACE, NULL), + { TOK_END , NULL, NULL } +#undef TOK_INFO +}; + + + +#define BUF_LEN(b) ((sizeof(b) / sizeof((b)[0]))) static ssize_t src_file_fill_buf(struct msph_ctx *, struct msph_token_src_file *); static int tok_match(struct msph_ctx *, struct msph_token_src *, - struct msph_token_matcher *); + struct msph_matcher *); static int tok_commit(struct msph_ctx *, struct msph_token_src *, - struct msph_token_matcher *, struct msph_token *); + struct msph_matcher *, struct msph_token *); static int char_at(struct msph_ctx *, struct msph_token_src *, size_t, char *); static int fromcbuf_charcpy(char *, const char *, size_t, size_t, size_t); @@ -41,18 +101,27 @@ static int file_char_at(struct msph_ctx *, struct msph_token_src *, size_t, static int read_single_tok(struct msph_token *, struct msph_token_stream *); -struct msph_token_stream * -msph_token_stream_fopen(struct msph_ctx *ctx, const char *path) +static const char *tok_base_str(struct msph_token *); + + +void msph_ctx_init(struct msph_ctx *ctx) +{ + ctx->err = 0; + ctx->err_info = 0; +} + +struct msph_token_stream * +msph_token_stream_file(struct msph_ctx *ctx, FILE *f, const char *name) { - FILE *f; size_t res; struct msph_token_stream *ret; - if ((f = fopen(path, "r")) == NULL) { - MSPH_ERR(ctx, MSPH_ERR_SYS); + if (ctx == NULL || f == NULL || name == NULL) { + MSPH_ERR(ctx, MSPH_ERR_INVAL); return (NULL); } + if ((ret = calloc(1, sizeof(struct msph_token_stream))) == NULL) { MSPH_ERR(ctx, MSPH_ERR_SYS); goto err; @@ -64,7 +133,7 @@ msph_token_stream_fopen(struct msph_ctx *ctx, const char *path) ret->src.inner.file.pos = 0; ret->src.inner.file.end = 0; - res = strlcpy(ret->src.inner.file.name, path, + res = strlcpy(ret->src.inner.file.name, name, sizeof(ret->src.inner.file.name)); if (res >= sizeof(ret->src.inner.file.name)) { MSPH_ERR(ctx, MSPH_ERR_TOOLONG); @@ -101,6 +170,53 @@ msph_token_stream_frombuf(struct msph_ctx *ctx, const char *buf, size_t len) return (ret); } +size_t +msph_token_str(char *buf, size_t len, struct msph_token *tok) +{ + size_t ret; + + ret = snprintf(buf, len, "%s", tok_base_str(tok)); + + if (ret > len) + return (ret); + + len -= ret; + buf += ret; + + switch (tok->type) { + case TOK_IDENT: + ret += snprintf(buf, len, "(%s)", tok->d.s.buf); + break; + default: + break; + } + + return (ret); +} + +#define MSPH_TOKEN_PRINT_BUF_LEN 2 * MSPH_TOKEN_BUF_LEN +int +msph_token_stream_print(struct msph_token_stream *s, FILE *out) +{ + ssize_t ret; + ssize_t i; + struct msph_token tok; + char tokstr[MSPH_TOKEN_PRINT_BUF_LEN]; + + while ((ret = msph_token_stream_read_tok( &tok, 1, s)) > 0) { + SPHO_DEBUG_PRINT("msph_token_stream_print: ret=%zd\n", ret); + if (msph_token_str( + tokstr, BUF_LEN(tokstr), &tok) > BUF_LEN(tokstr)) { + tokstr[BUF_LEN(tokstr) - 1] = '\0'; + } + fprintf(out, "%s\n", tokstr); + } + + SPHO_DEBUG_PRINT("msph_token_stream_print: ret=%zd\n", ret); + + return ((int)ret); +} + int msph_token_stream_close(struct msph_token_stream *s) @@ -128,24 +244,21 @@ ssize_t msph_token_stream_read_tok(struct msph_token *ptr, size_t n, struct msph_token_stream *s) { - ssize_t ret; - size_t i; + size_t ret; int res; ret = 0; res = -1; - while ((res = read_single_tok(&ptr[i], s)) == 1) { - if (res == -1) { - ret = -1; - break; - } + while (ret < n && (res = read_single_tok(&ptr[ret], s)) != 0) { + if (res == -1) + return (-1); ret++; } - return (ret); + return ((ssize_t)ret); } -/* 1: success, 0: failed match, -1: error */ +/* 1: matched token, 0: failed to match token, -1: error */ static int read_single_tok(struct msph_token *ptr, struct msph_token_stream *s) { @@ -158,15 +271,23 @@ read_single_tok(struct msph_token *ptr, struct msph_token_stream *s) ctx = s->ctx; src = &s->src; + /* Skipping whitespace */ + if (tok_match(ctx, src, &wspace) == -1) + return (-1); + if (wspace.matchlen > 0 && + tok_commit(ctx, src, &wspace, NULL) == -1) + return (-1); + max_m = 0; - for (m = 0; msph_matcher[m].type != TOK_END; m++) { - res = tok_match(ctx, src, &msph_matcher[m]); + for (m = 1; token_matcher[m].type != TOK_END; m++) { + res = tok_match(ctx, src, &token_matcher[m]); + SPHO_DEBUG_PRINT("read_single_tok: tok_match=%d\n", res); if (res == -1) return (-1); if (res == 0 && - msph_matcher[m].matchlen > msph_matcher[max_m].matchlen) { + token_matcher[m].matchlen > token_matcher[max_m].matchlen) { max_m = m; } } @@ -174,15 +295,15 @@ read_single_tok(struct msph_token *ptr, struct msph_token_stream *s) if (max_m == 0) return (0); - if (tok_commit(ctx, src, &msph_matcher[max_m], ptr) == -1) + SPHO_DEBUG_PRINT("read_single_tok: commit=%zu\n", max_m); + if (tok_commit(ctx, src, &token_matcher[max_m], ptr) == -1) return (-1); + SPHO_DEBUG_PRINT("read_single_tok: committed\n"); return (1); } -#define BUF_LEN(b) (sizeof(b) / sizeof((b)[0])) - static ssize_t src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file) { @@ -207,6 +328,9 @@ src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file) ret += nread; file->end = (file->end + nread) % BUF_LEN(file->buf); + SPHO_DEBUG_PRINT("src_file_fill_buf: valid range (%zu, %zu)\n", + file->pos, file->end); + if (nread < maxread) { if (ferror(file->f)) { MSPH_ERR(ctx, MSPH_ERR_SYS); @@ -217,6 +341,8 @@ src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file) } while (file->end != file->pos); + SPHO_DEBUG_PRINT("src_file_fill_buf: read %zd\n", ret); + return (ret); } @@ -229,13 +355,16 @@ file_char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i, ssize_t fill; struct msph_token_src_file *file; - SPHO_PRECOND(s != NULL); - SPHO_PRECOND(s->src.type == MSPH_TOKEN_SRC_FILE); + SPHO_PRECOND(src != NULL); + SPHO_PRECOND(src->type == MSPH_TOKEN_SRC_FILE); ret = 0; file = &src->inner.file; + fill = 0; do { + SPHO_DEBUG_PRINT("want to read %zu, valid range (%zu, %zu)\n", + (file->pos + i) % BUF_LEN(file->buf), file->pos, file->end); /* simplest case */ if (file->pos + i < file->end) { *out = file->buf[file->pos + i]; @@ -250,11 +379,12 @@ file_char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i, if (feof(file->f)) return (0); - if (src_file_fill_buf(ctx, file) == -1) + if ((fill = src_file_fill_buf(ctx, file)) == -1) return (-1); - } while (ret++); - return (-1); + } while (fill > 0 && ret++); + + return (ret); } static int @@ -265,18 +395,22 @@ char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i, char *out) switch (src->type) { case MSPH_TOKEN_SRC_FILE: - return (file_char_at(ctx, src, i, out)); + ret = file_char_at(ctx, src, i, out); + break; case MSPH_TOKEN_SRC_STR: str = &src->inner.str; if (str->pos + i < str->len) { *out = str->s[str->pos + i]; - return (1); + ret = 1; + } else { + ret = 0; } - return (0); + break; default: break; } + SPHO_DEBUG_PRINT("char_at: ret=%d, *out=%c\n", ret, *out); return (ret); } @@ -289,9 +423,11 @@ fromcbuf_charcpy(char *dst, const char *src, size_t src_len, size_t src_pos, return (-1); } - cpy1 = src_pos + ncpy < src_len ? ncpy : src_len - src_pos; + cpy1 = (src_pos + ncpy < src_len) ? ncpy : src_len - src_pos; cpy2 = ncpy - cpy1; + SPHO_DEBUG_PRINT("fromcbuf_charcpy: cpy1=%zu cpy2=%zu\n", cpy1, cpy2); + memcpy(dst, &src[src_pos], cpy1 * sizeof(src[0])); if (! cpy2) @@ -304,27 +440,30 @@ fromcbuf_charcpy(char *dst, const char *src, size_t src_len, size_t src_pos, static int tok_match(struct msph_ctx *ctx, struct msph_token_src *src, - struct msph_token_matcher *m) + struct msph_matcher *m) { int res; int more; char chr; const char *match_str; size_t off, len; - SPHO_PRECOND(m != NULL && s != NULL); + SPHO_PRECOND(m != NULL && src != NULL); m->matchlen = 0; #define MATCH_CHAR(c) \ do { \ + SPHO_DEBUG_PRINT("tok_match: '%c'\n", c); \ if ((res = char_at(ctx, src, 0, &chr)) == -1) \ return (-1); \ else if (res == 0) \ return (0); \ + SPHO_DEBUG_PRINT("tok_match: char_at(0)='%c'\n", c); \ \ if (chr == (c)) { \ m->matchlen = 1; \ } \ + SPHO_DEBUG_PRINT("tok_match: matchlen=%zu\n", m->matchlen); \ return (0); \ } while (0) @@ -397,6 +536,16 @@ tok_match(struct msph_ctx *ctx, struct msph_token_src *src, if (res == -1) return (-1); return (0); + case TOK_WSPACE: + off = 0; + while((res = char_at(ctx, src, off++, &chr)) == 1) { + if (! isspace(chr)) + break; + m->matchlen++; + } + if (res == -1) + return (-1); + return (0); default: SPHO_ASSERT(0); return (-1); @@ -409,40 +558,50 @@ tok_match(struct msph_ctx *ctx, struct msph_token_src *src, #define TOK_HAS_DATA(type) (type == TOK_IDENT) static int tok_commit(struct msph_ctx *ctx, struct msph_token_src *src, - struct msph_token_matcher *m, struct msph_token *ptr) + struct msph_matcher *m, struct msph_token *ptr) { size_t pos_old; struct msph_token_src_str *str; struct msph_token_src_file *file; - SPHO_PRECOND(p != NULL && m != NULL); + SPHO_PRECOND(ctx != NULL && m != NULL); SPHO_PRECOND(m->matchlen != 0); + SPHO_DEBUG_PRINT("committing\n"); + switch (src->type) { case MSPH_TOKEN_SRC_FILE: file = &src->inner.file; pos_old = file->pos; + SPHO_DEBUG_PRINT("committing\n"); file->pos += m->matchlen; file->pos %= BUF_LEN(file->buf); SPHO_ASSERT(file->pos < BUF_LEN(file->buf) || file->pos < pos_old); + SPHO_DEBUG_PRINT("committing\n"); + if (ptr == NULL) + return (0); + ptr->type = m->type; if (! TOK_HAS_DATA(ptr->type)) return (0); + SPHO_DEBUG_PRINT("committing\n"); if (m->matchlen >= sizeof(ptr->d.s.buf)) { MSPH_ERR(ctx, MSPH_ERR_TOOLONG); return (-1); } + SPHO_DEBUG_PRINT("committing\n"); if (fromcbuf_charcpy(ptr->d.s.buf, file->buf, sizeof(file->buf), pos_old, m->matchlen) == -1) { MSPH_ERR(ctx, MSPH_ERR_TOOLONG); return (-1); } + SPHO_DEBUG_PRINT("committing\n"); ptr->d.s.buf[m->matchlen] = '\0'; return (0); @@ -453,6 +612,9 @@ tok_commit(struct msph_ctx *ctx, struct msph_token_src *src, str->pos += m->matchlen; SPHO_ASSERT(str->pos <= str->len); + if (ptr == NULL) + return (0); + ptr->type = m->type; if (! TOK_HAS_DATA(ptr->type)) return (0); @@ -474,6 +636,17 @@ tok_commit(struct msph_ctx *ctx, struct msph_token_src *src, +static const char * +tok_base_str(struct msph_token *tok) +{ + size_t i; + for (i = 0; token_info[i].type != TOK_END; i++) { + if (token_info[i].type == tok->type) + return (token_info[i].dbg_str); + } + + return (NULL); +}