lnsol/log-e-sappho
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lexing up and running

Browse source
This commit is contained in:
Ellen Arvidsson 2025-04-15 20:02:25 +02:00
parent 10e16147ba
commit b9266cdf96
6 changed files with 434 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

12
CMakeLists.txt
View file

@ -46,14 +46,18 @@ target_compile_definitions(devcheck PRIVATE
set(MSPH_SRC
${SRC_DIR}/msph/msph_token.c
${SRC_DIR}/msph/token.c
${SRC_DIR}/msph/msph.c
)
set(MSPH_HEADER
${SRC_DIR}/msph/msph_token.h
${INCLUDE_DIR}/msph/token.h
)
add_executable(msph ${MSPH_HEADER} ${MSPH_SRC})
add_executable(msph ${MSPH_SRC} ${MSPH_HEADER})
target_include_directories(msph PRIVATE ${INCLUDE_DIR})
target_link_libraries(devcheck spho)
target_link_libraries(msph spho)
target_compile_definitions(msph PRIVATE
$<$<CONFIG:Debug>:SPHO_DEBUG>
$<$<BOOL:${HAVE_STRLCPY}>:SPHO_USE_STRLCPY>
)

20
include/msph/err.h Normal file
View file

@ -0,0 +1,20 @@
#ifndef _MSPH_ERR_H
#define _MSPH_ERR_H
#ifdef SPHO_DEBUG
// #define SPHO_ENABLE_DEBUG_PRINT
#endif
#include "spho/err.h"
#define MSPH_ERR_SYS 0x0001
#define MSPH_ERR_INVAL 0x1001
#define MSPH_ERR_TOOLONG 0x2001
#define MSPH_ERR(ctx, e) SPHO_ERR(ctx, e)
#define MSPH_TOKS_ERR(toks, e) MSPH_ERR((toks)->ctx, e)
#endif

31
src/msph/msph_token.h → include/msph/token.h
View file

@ -80,12 +80,17 @@
*/
struct msph_ctx {
int err;
int err_info;
int err;
int err_info;
char errbuf[SPHO_ERR_BUF_LEN];
#ifdef SPHO_DEBUG
char filebuf[SPHO_ERR_FILEBUF_LEN];
int errline;
#endif
};
enum msph_tok_type {
TOK_START,
TOK_LBRACE, // {
TOK_RBRACE, // }
TOK_LBRAK, // [
@ -111,13 +116,14 @@ enum msph_tok_type {
TOK_CONST_FALSE, // False
TOK_IDENT, // identifiers
TOK_END
TOK_WSPACE, // special: whitespace
TOK_END // special: denote end of array
};
#define MSPH_TOKEN_BUF_SZ 128
#define MSPH_TOKEN_BUF_LEN 128
struct token_s {
char buf[MSPH_TOKEN_BUF_SZ];
char buf[MSPH_TOKEN_BUF_LEN];
};
union token_data {
@ -129,7 +135,7 @@ struct msph_token {
union token_data d;
};
#define MSPH_PATH_LEN 1024
#define MSPH_FILE_NAME_LEN 1024
#define MSPH_FILE_BUF_LEN 4096
#define MSPH_TOKEN_SRC_FILE 1
@ -142,7 +148,7 @@ struct msph_token_src_file {
char buf[MSPH_FILE_BUF_LEN];
/* file path */
char name[MSPH_PATH_LEN];
char name[MSPH_FILE_NAME_LEN];
};
#define MSPH_TOKEN_SRC_STR 2
@ -168,8 +174,10 @@ struct msph_token_stream {
struct msph_token_src src;
};
struct msph_token_stream *msph_token_stream_fopen(struct msph_ctx *,
const char *);
void msph_ctx_init(struct msph_ctx *);
struct msph_token_stream *msph_token_stream_file(struct msph_ctx *,
FILE *, const char *);
struct msph_token_stream *msph_token_stream_frombuf(struct msph_ctx *,
const char *, size_t);
@ -177,6 +185,9 @@ int msph_token_stream_close(struct msph_token_stream*);
ssize_t msph_token_stream_read_tok(struct msph_token *, size_t,
struct msph_token_stream *);
int msph_token_stream_print(struct msph_token_stream *, FILE *);
size_t msph_token_str(char *buf, size_t len, struct msph_token *tok);
#endif /* _MSPH_EXPR_H */

51
include/spho/err.h
View file

@ -16,8 +16,14 @@
#define SPHO_ERR_IS_SYSERR(err) (SPHO_ERR_SYS == err)
#define SPHO_ERR_BUF_LEN 2048
#ifdef SPHO_DEBUG
#define SPHO_ERR_FILEBUF_LEN 128
#define SPHO_ERR(ctx, e) \
do { \
(ctx)->err = (e); \
@ -28,20 +34,6 @@
(ctx)->errline = __LINE__; \
} while (0)
#else /* SPHO_DEBUG */
#define SPHO_ERR(ctx, e) \
do { \
(ctx)->err = (e); \
if ((e) == SPHO_ERR_SYS) \
(ctx)->err_info = errno; \
} while (0)
#endif /* SPHO_DEBUG */
/* debug stuff */
#ifdef SPHO_DEBUG
#define SPHO_STRINGIFY(a) #a
#define SPHO_MACRO_STR(b) SPHO_STRINGIFY(b)
@ -77,10 +69,35 @@
} \
} while (0)
#else
#ifdef SPHO_ENABLE_DEBUG_PRINT
#define SPHO_DEBUG_PRINT(fmt, ...) \
do { \
fprintf(stderr, fmt __VA_OPT__(,) __VA_ARGS__); \
} while (0)
#else /* SPHO_ENABLE_DEBUG_PRINT */
#define SPHO_DEBUG_PRINT(fmt, ...)
#endif /* SPHO_ENABLE_DEBUG_PRINT */
#else /* SPHO_DEBUG */
#define SPHO_ERR(ctx, e) \
do { \
(ctx)->err = (e); \
if ((e) == SPHO_ERR_SYS) \
(ctx)->err_info = errno; \
} while (0)
#define SPHO_PRECOND(cond)
#define SPHO_ASSERT(cond)
#define SPHO_POSTCOND(cond)
#endif
#endif
#define SPHO_DBG_PRINT(fmt, ...)
#endif /* SPHO_DEBUG */
#endif /* _SPHO_ERR_H */

153
src/msph/msph.c
View file

@ -1,40 +1,147 @@
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include "msph_token.h"
#include "msph/err.h"
#include "msph/token.h"
#define PRINTERR(ctx) \
do { \
fprintf(stderr, "msph_err: %d (%d)\n", (ctx).err, (ctx).err_info); \
} while (0)
#ifdef SPHO_DEBUG
#define DEBUGFLAG "d"
#endif
struct msph_opts {
int tokenize_only;
#ifdef DEBUGFLAG
int debug;
#endif
const char *in_path;
const char *out_path;
} run_opts = {
0,
#ifdef DEBUGFLAG
0,
#endif
NULL,
NULL
};
#define USAGE \
"msph: micro sappho\n" \
"\tUsage: msph [-t] [-o out_path] [in_path]\n" \
"\n" \
"\t-t - print tokenization\n" \
"\t-o - output path\n"
void
print_usage(void)
{
printf("%s", USAGE);
}
int run(struct msph_opts *);
int
main(int argc, char **argv)
main(int argc, char *argv[])
{
int ret;
char *path;
struct msph_ctx ctx;
struct msph_token_stream *s;
int opt;
if (argc != 2)
return (-1);
path = argv[1];
ctx.err = 0;
ctx.err_info = 0;
if ((s = msph_token_stream_fopen(&ctx, path)) == NULL) {
PRINTERR(ctx);
return (-1);
while ((opt = getopt(argc, argv, DEBUGFLAG "to:")) != -1) {
switch (opt) {
case 't':
run_opts.tokenize_only = 1;
break;
case 'o':
run_opts.out_path = optarg;
break;
#ifdef DEBUGFLAG
case 'd':
run_opts.debug = 1;
break;
#endif
case '?':
default:
print_usage();
exit(EXIT_FAILURE);
}
}
argc -= optind;
argv += optind;
ret = msph_token_stream_close(s);
if (argc > 1) {
print_usage();
exit(EXIT_FAILURE);
}
return (ret);
if (argc == 1)
run_opts.in_path = argv[0];
if (run(&run_opts))
exit(EXIT_FAILURE);
}
int
run(struct msph_opts *opts)
{
int ret;
struct msph_ctx ctx;
struct msph_token_stream *s;
FILE *in, *out;
in = out = NULL;
msph_ctx_init(&ctx);
if (opts->in_path == NULL) {
in = stdin; opts->in_path = "stdin";
} else {
if ((in = fopen(opts->in_path, "r")) == NULL) {
MSPH_ERR(&ctx, MSPH_ERR_SYS);
goto err;
}
}
if (opts->out_path == NULL) {
out = stdout;
} else {
if ((out = fopen(opts->out_path, "w")) == NULL) {
MSPH_ERR(&ctx, MSPH_ERR_SYS);
goto err;
}
}
if ((s = msph_token_stream_file(&ctx, in, opts->in_path)) == NULL) {
goto err;
}
printf("msph v0, parsing %s\n", opts->in_path);
if ((ret = msph_token_stream_print(s, out)) == -1)
goto err;
if (opts->tokenize_only)
goto exit;
exit:
if (msph_token_stream_close(s) == -1)
goto err;
return (0);
err:
PRINTERR(ctx);
if (in)
fclose(in);
if (out)
fclose(out);
return (-1);
}

269
src/msph/msph_token.c → src/msph/token.c
View file

@ -5,34 +5,94 @@
#include <string.h>
#include <ctype.h>
#include "spho/err.h"
#include "msph_token.h"
#include "msph/err.h"
#include "msph/token.h"
#define MSPH_ERR_SYS 0x0001
#define MSPH_ERR_TOOLONG 0x0002
#define MSPH_ERR(ctx, e) SPHO_ERR(ctx, e)
#define MSPH_TOKS_ERR(toks, e) MSPH_ERR((toks)->ctx, e)
struct msph_token_matcher {
struct msph_matcher {
size_t off;
size_t matchlen;
const int type;
} msph_matcher[] = {
{ 0, 0, TOK_START },
};
struct msph_matcher token_matcher[] = {
{ 0, 0, -1 },
{ 0, 0, TOK_LBRACE },
{ 0, 0, TOK_RBRACE },
{ 0, 0, TOK_LBRAK },
{ 0, 0, TOK_RBRAK },
{ 0, 0, TOK_LPAREN },
{ 0, 0, TOK_RPAREN },
{ 0, 0, TOK_COLON },
{ 0, 0, TOK_EQUALS },
{ 0, 0, TOK_AMP },
{ 0, 0, TOK_PIPE },
{ 0, 0, TOK_RARROW },
{ 0, 0, TOK_SUB },
{ 0, 0, TOK_KW_TYPE },
{ 0, 0, TOK_KW_NOMINAL },
{ 0, 0, TOK_KW_MEMBER },
{ 0, 0, TOK_KW_CHECK },
{ 0, 0, TOK_KW_BOX },
{ 0, 0, TOK_KW_FORALL },
{ 0, 0, TOK_CONST_TRUE },
{ 0, 0, TOK_CONST_FALSE },
{ 0, 0, TOK_IDENT },
{ 0, 0, TOK_END }
};
struct msph_matcher wspace = { 0, 0, TOK_WSPACE };
struct msph_token_info {
const int type;
const char *dbg_str;
const char *str;
} token_info[] = {
#define TOK_INFO(tok, s) { tok , #tok, s }
TOK_INFO(TOK_LBRACE, "{"),
TOK_INFO(TOK_RBRACE, "}"),
TOK_INFO(TOK_LBRAK, "["),
TOK_INFO(TOK_RBRAK, "]"),
TOK_INFO(TOK_LPAREN, "("),
TOK_INFO(TOK_RPAREN, ")"),
TOK_INFO(TOK_COLON, ":"),
TOK_INFO(TOK_EQUALS, "="),
TOK_INFO(TOK_AMP, "&"),
TOK_INFO(TOK_PIPE, "|"),
TOK_INFO(TOK_RARROW, "=>"),
TOK_INFO(TOK_SUB, "<:"),
TOK_INFO(TOK_KW_TYPE, "type"),
TOK_INFO(TOK_KW_NOMINAL, "nominal"),
TOK_INFO(TOK_KW_MEMBER, "member"),
TOK_INFO(TOK_KW_CHECK, "check"),
TOK_INFO(TOK_KW_BOX, "box"),
TOK_INFO(TOK_KW_FORALL, "forall"),
TOK_INFO(TOK_CONST_TRUE, "True"),
TOK_INFO(TOK_CONST_FALSE, "False"),
TOK_INFO(TOK_IDENT, NULL),
TOK_INFO(TOK_WSPACE, NULL),
{ TOK_END , NULL, NULL }
#undef TOK_INFO
};
#define BUF_LEN(b) ((sizeof(b) / sizeof((b)[0])))
static ssize_t src_file_fill_buf(struct msph_ctx *,
struct msph_token_src_file *);
static int tok_match(struct msph_ctx *, struct msph_token_src *,
struct msph_token_matcher *);
struct msph_matcher *);
static int tok_commit(struct msph_ctx *, struct msph_token_src *,
struct msph_token_matcher *, struct msph_token *);
struct msph_matcher *, struct msph_token *);
static int char_at(struct msph_ctx *, struct msph_token_src *, size_t,
char *);
static int fromcbuf_charcpy(char *, const char *, size_t, size_t, size_t);
@ -41,18 +101,27 @@ static int file_char_at(struct msph_ctx *, struct msph_token_src *, size_t,
static int read_single_tok(struct msph_token *, struct msph_token_stream *);
struct msph_token_stream *
msph_token_stream_fopen(struct msph_ctx *ctx, const char *path)
static const char *tok_base_str(struct msph_token *);
void msph_ctx_init(struct msph_ctx *ctx)
{
ctx->err = 0;
ctx->err_info = 0;
}
struct msph_token_stream *
msph_token_stream_file(struct msph_ctx *ctx, FILE *f, const char *name)
{
FILE *f;
size_t res;
struct msph_token_stream *ret;
if ((f = fopen(path, "r")) == NULL) {
MSPH_ERR(ctx, MSPH_ERR_SYS);
if (ctx == NULL || f == NULL || name == NULL) {
MSPH_ERR(ctx, MSPH_ERR_INVAL);
return (NULL);
}
if ((ret = calloc(1, sizeof(struct msph_token_stream))) == NULL) {
MSPH_ERR(ctx, MSPH_ERR_SYS);
goto err;
@ -64,7 +133,7 @@ msph_token_stream_fopen(struct msph_ctx *ctx, const char *path)
ret->src.inner.file.pos = 0;
ret->src.inner.file.end = 0;
res = strlcpy(ret->src.inner.file.name, path,
res = strlcpy(ret->src.inner.file.name, name,
sizeof(ret->src.inner.file.name));
if (res >= sizeof(ret->src.inner.file.name)) {
MSPH_ERR(ctx, MSPH_ERR_TOOLONG);
@ -101,6 +170,53 @@ msph_token_stream_frombuf(struct msph_ctx *ctx, const char *buf, size_t len)
return (ret);
}
size_t
msph_token_str(char *buf, size_t len, struct msph_token *tok)
{
size_t ret;
ret = snprintf(buf, len, "%s", tok_base_str(tok));
if (ret > len)
return (ret);
len -= ret;
buf += ret;
switch (tok->type) {
case TOK_IDENT:
ret += snprintf(buf, len, "(%s)", tok->d.s.buf);
break;
default:
break;
}
return (ret);
}
#define MSPH_TOKEN_PRINT_BUF_LEN 2 * MSPH_TOKEN_BUF_LEN
int
msph_token_stream_print(struct msph_token_stream *s, FILE *out)
{
ssize_t ret;
ssize_t i;
struct msph_token tok;
char tokstr[MSPH_TOKEN_PRINT_BUF_LEN];
while ((ret = msph_token_stream_read_tok( &tok, 1, s)) > 0) {
SPHO_DEBUG_PRINT("msph_token_stream_print: ret=%zd\n", ret);
if (msph_token_str(
tokstr, BUF_LEN(tokstr), &tok) > BUF_LEN(tokstr)) {
tokstr[BUF_LEN(tokstr) - 1] = '\0';
}
fprintf(out, "%s\n", tokstr);
}
SPHO_DEBUG_PRINT("msph_token_stream_print: ret=%zd\n", ret);
return ((int)ret);
}
int
msph_token_stream_close(struct msph_token_stream *s)
@ -128,24 +244,21 @@ ssize_t
msph_token_stream_read_tok(struct msph_token *ptr, size_t n,
struct msph_token_stream *s)
{
ssize_t ret;
size_t i;
size_t ret;
int res;
ret = 0;
res = -1;
while ((res = read_single_tok(&ptr[i], s)) == 1) {
if (res == -1) {
ret = -1;
break;
}
while (ret < n && (res = read_single_tok(&ptr[ret], s)) != 0) {
if (res == -1)
return (-1);
ret++;
}
return (ret);
return ((ssize_t)ret);
}
/* 1: success, 0: failed match, -1: error */
/* 1: matched token, 0: failed to match token, -1: error */
static int
read_single_tok(struct msph_token *ptr, struct msph_token_stream *s)
{
@ -158,15 +271,23 @@ read_single_tok(struct msph_token *ptr, struct msph_token_stream *s)
ctx = s->ctx;
src = &s->src;
/* Skipping whitespace */
if (tok_match(ctx, src, &wspace) == -1)
return (-1);
if (wspace.matchlen > 0 &&
tok_commit(ctx, src, &wspace, NULL) == -1)
return (-1);
max_m = 0;
for (m = 0; msph_matcher[m].type != TOK_END; m++) {
res = tok_match(ctx, src, &msph_matcher[m]);
for (m = 1; token_matcher[m].type != TOK_END; m++) {
res = tok_match(ctx, src, &token_matcher[m]);
SPHO_DEBUG_PRINT("read_single_tok: tok_match=%d\n", res);
if (res == -1)
return (-1);
if (res == 0 &&
msph_matcher[m].matchlen > msph_matcher[max_m].matchlen) {
token_matcher[m].matchlen > token_matcher[max_m].matchlen) {
max_m = m;
}
}
@ -174,15 +295,15 @@ read_single_tok(struct msph_token *ptr, struct msph_token_stream *s)
if (max_m == 0)
return (0);
if (tok_commit(ctx, src, &msph_matcher[max_m], ptr) == -1)
SPHO_DEBUG_PRINT("read_single_tok: commit=%zu\n", max_m);
if (tok_commit(ctx, src, &token_matcher[max_m], ptr) == -1)
return (-1);
SPHO_DEBUG_PRINT("read_single_tok: committed\n");
return (1);
}
#define BUF_LEN(b) (sizeof(b) / sizeof((b)[0]))
static ssize_t
src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file)
{
@ -207,6 +328,9 @@ src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file)
ret += nread;
file->end = (file->end + nread) % BUF_LEN(file->buf);
SPHO_DEBUG_PRINT("src_file_fill_buf: valid range (%zu, %zu)\n",
file->pos, file->end);
if (nread < maxread) {
if (ferror(file->f)) {
MSPH_ERR(ctx, MSPH_ERR_SYS);
@ -217,6 +341,8 @@ src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file)
} while (file->end != file->pos);
SPHO_DEBUG_PRINT("src_file_fill_buf: read %zd\n", ret);
return (ret);
}
@ -229,13 +355,16 @@ file_char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i,
ssize_t fill;
struct msph_token_src_file *file;
SPHO_PRECOND(s != NULL);
SPHO_PRECOND(s->src.type == MSPH_TOKEN_SRC_FILE);
SPHO_PRECOND(src != NULL);
SPHO_PRECOND(src->type == MSPH_TOKEN_SRC_FILE);
ret = 0;
file = &src->inner.file;
fill = 0;
do {
SPHO_DEBUG_PRINT("want to read %zu, valid range (%zu, %zu)\n",
(file->pos + i) % BUF_LEN(file->buf), file->pos, file->end);
/* simplest case */
if (file->pos + i < file->end) {
*out = file->buf[file->pos + i];
@ -250,11 +379,12 @@ file_char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i,
if (feof(file->f))
return (0);
if (src_file_fill_buf(ctx, file) == -1)
if ((fill = src_file_fill_buf(ctx, file)) == -1)
return (-1);
} while (ret++);
return (-1);
} while (fill > 0 && ret++);
return (ret);
}
static int
@ -265,18 +395,22 @@ char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i, char *out)
switch (src->type) {
case MSPH_TOKEN_SRC_FILE:
return (file_char_at(ctx, src, i, out));
ret = file_char_at(ctx, src, i, out);
break;
case MSPH_TOKEN_SRC_STR:
str = &src->inner.str;
if (str->pos + i < str->len) {
*out = str->s[str->pos + i];
return (1);
ret = 1;
} else {
ret = 0;
}
return (0);
break;
default:
break;
}
SPHO_DEBUG_PRINT("char_at: ret=%d, *out=%c\n", ret, *out);
return (ret);
}
@ -289,9 +423,11 @@ fromcbuf_charcpy(char *dst, const char *src, size_t src_len, size_t src_pos,
return (-1);
}
cpy1 = src_pos + ncpy < src_len ? ncpy : src_len - src_pos;
cpy1 = (src_pos + ncpy < src_len) ? ncpy : src_len - src_pos;
cpy2 = ncpy - cpy1;
SPHO_DEBUG_PRINT("fromcbuf_charcpy: cpy1=%zu cpy2=%zu\n", cpy1, cpy2);
memcpy(dst, &src[src_pos], cpy1 * sizeof(src[0]));
if (! cpy2)
@ -304,27 +440,30 @@ fromcbuf_charcpy(char *dst, const char *src, size_t src_len, size_t src_pos,
static int
tok_match(struct msph_ctx *ctx, struct msph_token_src *src,
struct msph_token_matcher *m)
struct msph_matcher *m)
{
int res;
int more;
char chr;
const char *match_str;
size_t off, len;
SPHO_PRECOND(m != NULL && s != NULL);
SPHO_PRECOND(m != NULL && src != NULL);
m->matchlen = 0;
#define MATCH_CHAR(c) \
do { \
SPHO_DEBUG_PRINT("tok_match: '%c'\n", c); \
if ((res = char_at(ctx, src, 0, &chr)) == -1) \
return (-1); \
else if (res == 0) \
return (0); \
SPHO_DEBUG_PRINT("tok_match: char_at(0)='%c'\n", c); \
\
if (chr == (c)) { \
m->matchlen = 1; \
} \
SPHO_DEBUG_PRINT("tok_match: matchlen=%zu\n", m->matchlen); \
return (0); \
} while (0)
@ -397,6 +536,16 @@ tok_match(struct msph_ctx *ctx, struct msph_token_src *src,
if (res == -1)
return (-1);
return (0);
case TOK_WSPACE:
off = 0;
while((res = char_at(ctx, src, off++, &chr)) == 1) {
if (! isspace(chr))
break;
m->matchlen++;
}
if (res == -1)
return (-1);
return (0);
default:
SPHO_ASSERT(0);
return (-1);
@ -409,40 +558,50 @@ tok_match(struct msph_ctx *ctx, struct msph_token_src *src,
#define TOK_HAS_DATA(type) (type == TOK_IDENT)
static int
tok_commit(struct msph_ctx *ctx, struct msph_token_src *src,
struct msph_token_matcher *m, struct msph_token *ptr)
struct msph_matcher *m, struct msph_token *ptr)
{
size_t pos_old;
struct msph_token_src_str *str;
struct msph_token_src_file *file;
SPHO_PRECOND(p != NULL && m != NULL);
SPHO_PRECOND(ctx != NULL && m != NULL);
SPHO_PRECOND(m->matchlen != 0);
SPHO_DEBUG_PRINT("committing\n");
switch (src->type) {
case MSPH_TOKEN_SRC_FILE:
file = &src->inner.file;
pos_old = file->pos;
SPHO_DEBUG_PRINT("committing\n");
file->pos += m->matchlen;
file->pos %= BUF_LEN(file->buf);
SPHO_ASSERT(file->pos < BUF_LEN(file->buf) ||
file->pos < pos_old);
SPHO_DEBUG_PRINT("committing\n");
if (ptr == NULL)
return (0);
ptr->type = m->type;
if (! TOK_HAS_DATA(ptr->type))
return (0);
SPHO_DEBUG_PRINT("committing\n");
if (m->matchlen >= sizeof(ptr->d.s.buf)) {
MSPH_ERR(ctx, MSPH_ERR_TOOLONG);
return (-1);
}
SPHO_DEBUG_PRINT("committing\n");
if (fromcbuf_charcpy(ptr->d.s.buf, file->buf, sizeof(file->buf),
pos_old, m->matchlen) == -1) {
MSPH_ERR(ctx, MSPH_ERR_TOOLONG);
return (-1);
}
SPHO_DEBUG_PRINT("committing\n");
ptr->d.s.buf[m->matchlen] = '\0';
return (0);
@ -453,6 +612,9 @@ tok_commit(struct msph_ctx *ctx, struct msph_token_src *src,
str->pos += m->matchlen;
SPHO_ASSERT(str->pos <= str->len);
if (ptr == NULL)
return (0);
ptr->type = m->type;
if (! TOK_HAS_DATA(ptr->type))
return (0);
@ -474,6 +636,17 @@ tok_commit(struct msph_ctx *ctx, struct msph_token_src *src,
static const char *
tok_base_str(struct msph_token *tok)
{
size_t i;
for (i = 0; token_info[i].type != TOK_END; i++) {
if (token_info[i].type == tok->type)
return (token_info[i].dbg_str);
}
return (NULL);
}