lexing up and running

CMakeLists.txt

@@ -46,14 +46,18 @@ target_compile_definitions(devcheck PRIVATE
 
 
 set(MSPH_SRC
-	${SRC_DIR}/msph/msph_token.c
+	${SRC_DIR}/msph/token.c
 	${SRC_DIR}/msph/msph.c
 )
 
 set(MSPH_HEADER
-	${SRC_DIR}/msph/msph_token.h
+	${INCLUDE_DIR}/msph/token.h
 )
 
-add_executable(msph ${MSPH_HEADER} ${MSPH_SRC})
+add_executable(msph ${MSPH_SRC} ${MSPH_HEADER})
 target_include_directories(msph PRIVATE ${INCLUDE_DIR})
-target_link_libraries(devcheck spho)
+target_link_libraries(msph spho)
+target_compile_definitions(msph PRIVATE
+	$<$<CONFIG:Debug>:SPHO_DEBUG>
+	$<$<BOOL:${HAVE_STRLCPY}>:SPHO_USE_STRLCPY>
+)

include/msph/err.h

@@ -0,0 +1,20 @@
+#ifndef _MSPH_ERR_H
+#define _MSPH_ERR_H
+
+#ifdef SPHO_DEBUG
+// #define SPHO_ENABLE_DEBUG_PRINT
+#endif
+
+#include "spho/err.h"
+
+
+#define MSPH_ERR_SYS		0x0001
+
+#define MSPH_ERR_INVAL		0x1001
+
+#define MSPH_ERR_TOOLONG	0x2001
+
+#define MSPH_ERR(ctx, e)	SPHO_ERR(ctx, e)
+#define MSPH_TOKS_ERR(toks, e)	MSPH_ERR((toks)->ctx, e)
+
+#endif

include/msph/token.h

@@ -82,10 +82,15 @@
 struct msph_ctx {
 	int	err;
 	int	err_info;
+	char	errbuf[SPHO_ERR_BUF_LEN];
+
+#ifdef SPHO_DEBUG
+	char	filebuf[SPHO_ERR_FILEBUF_LEN];
+	int	errline;
+#endif
 };
 
 enum msph_tok_type {
-	TOK_START,
 	TOK_LBRACE,		// {
 	TOK_RBRACE,		// }
 	TOK_LBRAK,		// [
@@ -111,13 +116,14 @@ enum msph_tok_type {
 	TOK_CONST_FALSE,	// False
 
 	TOK_IDENT,		// identifiers
-	TOK_END
+	TOK_WSPACE,		// special: whitespace
+	TOK_END			// special: denote end of array
 };
 
-#define MSPH_TOKEN_BUF_SZ	128
+#define MSPH_TOKEN_BUF_LEN	128
 
 struct token_s {
-	char	buf[MSPH_TOKEN_BUF_SZ];
+	char	buf[MSPH_TOKEN_BUF_LEN];
 };
 
 union token_data {
@@ -129,7 +135,7 @@ struct msph_token {
 	union token_data	d;
 };
 
-#define MSPH_PATH_LEN		1024
+#define MSPH_FILE_NAME_LEN	1024
 #define MSPH_FILE_BUF_LEN	4096
 #define MSPH_TOKEN_SRC_FILE	1
 
@@ -142,7 +148,7 @@ struct msph_token_src_file {
 	char		buf[MSPH_FILE_BUF_LEN];
 
 	/* file path */
-	char		name[MSPH_PATH_LEN];
+	char		name[MSPH_FILE_NAME_LEN];
 };
 
 #define MSPH_TOKEN_SRC_STR	2
@@ -168,8 +174,10 @@ struct msph_token_stream {
 	struct msph_token_src	src;
 };
 
-struct msph_token_stream	*msph_token_stream_fopen(struct msph_ctx *,
+void msph_ctx_init(struct msph_ctx *);
-				    const char *);
+
+struct msph_token_stream	*msph_token_stream_file(struct msph_ctx *,
+				    FILE *, const char *);
 struct msph_token_stream	*msph_token_stream_frombuf(struct msph_ctx *,
 				    const char *, size_t);
 
@@ -177,6 +185,9 @@ int	msph_token_stream_close(struct msph_token_stream*);
 
 ssize_t	msph_token_stream_read_tok(struct msph_token *, size_t,
 	    struct msph_token_stream *);
+int	msph_token_stream_print(struct msph_token_stream *, FILE *);
+
+size_t	msph_token_str(char *buf, size_t len, struct msph_token *tok);
 
 
 #endif /* _MSPH_EXPR_H */

include/spho/err.h

@@ -16,8 +16,14 @@
 
 #define SPHO_ERR_IS_SYSERR(err)		(SPHO_ERR_SYS == err)
 
+#define SPHO_ERR_BUF_LEN	2048
+
+
+
 #ifdef SPHO_DEBUG
 
+#define SPHO_ERR_FILEBUF_LEN	128
+
 #define SPHO_ERR(ctx, e)						\
 	do {								\
 		(ctx)->err = (e);					\
@@ -28,20 +34,6 @@
 		(ctx)->errline = __LINE__;				\
 	} while (0)
 
-#else /* SPHO_DEBUG */
-
-#define SPHO_ERR(ctx, e)						\
-	do {								\
-		(ctx)->err = (e);					\
-		if ((e) == SPHO_ERR_SYS)				\
-			(ctx)->err_info = errno;			\
-	} while (0)
-
-#endif /* SPHO_DEBUG */
-
-/* debug stuff */
-#ifdef SPHO_DEBUG
-
 #define SPHO_STRINGIFY(a)	#a
 #define SPHO_MACRO_STR(b)	SPHO_STRINGIFY(b)
 
@@ -77,10 +69,35 @@
 		}							\
 	} while (0)
 
-#else
+
+#ifdef SPHO_ENABLE_DEBUG_PRINT
+
+#define SPHO_DEBUG_PRINT(fmt, ...) \
+	do { \
+		fprintf(stderr, fmt __VA_OPT__(,) __VA_ARGS__); \
+	} while (0)
+
+#else /* SPHO_ENABLE_DEBUG_PRINT */
+
+#define SPHO_DEBUG_PRINT(fmt, ...)
+
+#endif /* SPHO_ENABLE_DEBUG_PRINT */
+
+#else /* SPHO_DEBUG */
+
+#define SPHO_ERR(ctx, e)						\
+	do {								\
+		(ctx)->err = (e);					\
+		if ((e) == SPHO_ERR_SYS)				\
+			(ctx)->err_info = errno;			\
+	} while (0)
+
 #define SPHO_PRECOND(cond)
 #define SPHO_ASSERT(cond)
 #define SPHO_POSTCOND(cond)
-#endif
 
-#endif
+#define SPHO_DBG_PRINT(fmt, ...)
+
+#endif /* SPHO_DEBUG */
+
+#endif /* _SPHO_ERR_H */

src/msph/msph.c

@@ -1,40 +1,147 @@
+#include <stdlib.h>
 #include <stdio.h>
+#include <unistd.h>
 
-
+#include "msph/err.h"
-
+#include "msph/token.h"
-#include "msph_token.h"
 
 #define PRINTERR(ctx) \
 	do { \
 		fprintf(stderr, "msph_err: %d (%d)\n", (ctx).err, (ctx).err_info); \
 	} while (0)
 
+#ifdef SPHO_DEBUG
+#define DEBUGFLAG "d"
+#endif
+
+struct msph_opts {
+	int		tokenize_only;
+#ifdef DEBUGFLAG
+	int		debug;
+#endif
+	const char	*in_path;
+	const char	*out_path;
+} run_opts = {
+	0,
+#ifdef DEBUGFLAG
+	0,
+#endif
+	NULL,
+	NULL
+};
+
+
+#define USAGE \
+	"msph: micro sappho\n" \
+	"\tUsage: msph [-t] [-o out_path] [in_path]\n" \
+	"\n" \
+	"\t-t	- print tokenization\n" \
+	"\t-o	- output path\n"
+
+
+void
+print_usage(void)
+{
+	printf("%s", USAGE);
+}
+
+int run(struct msph_opts *);
 
 int
-main(int argc, char **argv)
+main(int argc, char *argv[])
+{
+	int opt;
+
+	while ((opt = getopt(argc, argv, DEBUGFLAG "to:")) != -1) {
+		switch (opt) {
+		case 't':
+			run_opts.tokenize_only = 1;
+			break;
+		case 'o':
+			run_opts.out_path = optarg;
+			break;
+#ifdef DEBUGFLAG
+		case 'd':
+			run_opts.debug = 1;
+			break;
+#endif
+		case '?':
+		default:
+			print_usage();
+			exit(EXIT_FAILURE);
+		}
+	}
+
+	argc -= optind;
+	argv += optind;
+
+	if (argc > 1) {
+		print_usage();
+		exit(EXIT_FAILURE);
+	}
+
+	if (argc == 1)
+		run_opts.in_path = argv[0];
+
+	if (run(&run_opts))
+		exit(EXIT_FAILURE);
+}
+
+
+int
+run(struct msph_opts *opts)
 {
 	int ret;
-	char *path;
 	struct msph_ctx ctx;
 	struct msph_token_stream *s;
+	FILE *in, *out;
 
+	in = out = NULL;
 
-	if (argc != 2)
+	msph_ctx_init(&ctx);
-		return (-1);
 
+	if (opts->in_path == NULL) {
+		in = stdin; opts->in_path = "stdin";
+	} else {
+		if ((in = fopen(opts->in_path, "r")) == NULL) {
+			MSPH_ERR(&ctx, MSPH_ERR_SYS);
+			goto err;
+		}
+	}
 
-	path = argv[1];
+	if (opts->out_path == NULL) {
+		out = stdout;
+	} else {
+		if ((out = fopen(opts->out_path, "w")) == NULL) {
+			MSPH_ERR(&ctx, MSPH_ERR_SYS);
+			goto err;
+		}
+	}
 
-	ctx.err = 0;
+	if ((s = msph_token_stream_file(&ctx, in, opts->in_path)) == NULL) {
-	ctx.err_info = 0;
+		goto err;
+	}
 
-	if ((s = msph_token_stream_fopen(&ctx, path)) == NULL) {
+	printf("msph v0, parsing %s\n", opts->in_path);
+
+	if ((ret = msph_token_stream_print(s, out)) == -1)
+		goto err;
+
+	if (opts->tokenize_only)
+		goto exit;
+
+exit:
+	if (msph_token_stream_close(s) == -1)
+		goto err;
+
+	return (0);
+
+err:
 	PRINTERR(ctx);
+
+	if (in)
+		fclose(in);
+	if (out)
+		fclose(out);
 	return (-1);
 }
-
-
-	ret = msph_token_stream_close(s);
-
-	return (ret);
-}

src/msph/token.c

@@ -5,34 +5,94 @@
 #include <string.h>
 #include <ctype.h>
 
-#include "spho/err.h"
+#include "msph/err.h"
-
+#include "msph/token.h"
-#include "msph_token.h"
 
 
-#define MSPH_ERR_SYS		0x0001
+struct msph_matcher {
-#define MSPH_ERR_TOOLONG	0x0002
-
-#define MSPH_ERR(ctx, e)	SPHO_ERR(ctx, e)
-#define MSPH_TOKS_ERR(toks, e)	MSPH_ERR((toks)->ctx, e)
-
-struct msph_token_matcher {
 	size_t		off;
 	size_t		matchlen;
 
 	const int	type;
-} msph_matcher[] = {
+};
-	{ 0, 0, TOK_START },
+
+struct msph_matcher token_matcher[] = {
+	{ 0, 0, -1 },
+	{ 0, 0, TOK_LBRACE },
+	{ 0, 0, TOK_RBRACE },
+	{ 0, 0, TOK_LBRAK },
+	{ 0, 0, TOK_RBRAK },
+	{ 0, 0, TOK_LPAREN },
+	{ 0, 0, TOK_RPAREN },
+	{ 0, 0, TOK_COLON },
+	{ 0, 0, TOK_EQUALS },
+
+	{ 0, 0, TOK_AMP },
+	{ 0, 0, TOK_PIPE },
+	{ 0, 0, TOK_RARROW },
+	{ 0, 0, TOK_SUB },
+
+	{ 0, 0, TOK_KW_TYPE },
+	{ 0, 0, TOK_KW_NOMINAL },
+	{ 0, 0, TOK_KW_MEMBER },
+	{ 0, 0, TOK_KW_CHECK },
+	{ 0, 0, TOK_KW_BOX },
+	{ 0, 0, TOK_KW_FORALL },
+
+	{ 0, 0, TOK_CONST_TRUE },
+	{ 0, 0, TOK_CONST_FALSE },
+
 	{ 0, 0, TOK_IDENT },
 	{ 0, 0, TOK_END }
 };
+struct msph_matcher wspace = { 0, 0, TOK_WSPACE };
+
+struct msph_token_info {
+	const int type;
+	const char *dbg_str;
+	const char *str;
+} token_info[] = {
+#define TOK_INFO(tok, s)	{ tok , #tok, s }
+	TOK_INFO(TOK_LBRACE, "{"),
+	TOK_INFO(TOK_RBRACE, "}"),
+	TOK_INFO(TOK_LBRAK, "["),
+	TOK_INFO(TOK_RBRAK, "]"),
+	TOK_INFO(TOK_LPAREN, "("),
+	TOK_INFO(TOK_RPAREN, ")"),
+	TOK_INFO(TOK_COLON, ":"),
+	TOK_INFO(TOK_EQUALS, "="),
+
+	TOK_INFO(TOK_AMP, "&"),
+	TOK_INFO(TOK_PIPE, "|"),
+	TOK_INFO(TOK_RARROW, "=>"),
+	TOK_INFO(TOK_SUB, "<:"),
+
+	TOK_INFO(TOK_KW_TYPE, "type"),
+	TOK_INFO(TOK_KW_NOMINAL, "nominal"),
+	TOK_INFO(TOK_KW_MEMBER, "member"),
+	TOK_INFO(TOK_KW_CHECK, "check"),
+	TOK_INFO(TOK_KW_BOX, "box"),
+	TOK_INFO(TOK_KW_FORALL, "forall"),
+
+	TOK_INFO(TOK_CONST_TRUE, "True"),
+	TOK_INFO(TOK_CONST_FALSE, "False"),
+
+	TOK_INFO(TOK_IDENT, NULL),
+	TOK_INFO(TOK_WSPACE, NULL),
+	{ TOK_END , NULL, NULL }
+#undef TOK_INFO
+};
+
+
+
+#define BUF_LEN(b)	((sizeof(b) / sizeof((b)[0])))
 
 static ssize_t	src_file_fill_buf(struct msph_ctx *,
 		    struct msph_token_src_file *);
 static int	tok_match(struct msph_ctx *, struct msph_token_src *,
-		    struct msph_token_matcher *);
+		    struct msph_matcher *);
 static int	tok_commit(struct msph_ctx *, struct msph_token_src *, 
-		    struct msph_token_matcher *, struct msph_token *);
+		    struct msph_matcher *, struct msph_token *);
 static int	char_at(struct msph_ctx *, struct msph_token_src *, size_t,
 		    char *);
 static int	fromcbuf_charcpy(char *, const char *, size_t, size_t, size_t);
@@ -41,18 +101,27 @@ static int	file_char_at(struct msph_ctx *, struct msph_token_src *, size_t,
 static int	read_single_tok(struct msph_token *, struct msph_token_stream *);
 
 
-struct msph_token_stream *
+static const char	*tok_base_str(struct msph_token *);
-msph_token_stream_fopen(struct msph_ctx *ctx, const char *path)
+
+
+void msph_ctx_init(struct msph_ctx *ctx)
+{
+	ctx->err = 0;
+	ctx->err_info = 0;
+}
+
+struct msph_token_stream *
+msph_token_stream_file(struct msph_ctx *ctx, FILE *f, const char *name)
 {
-	FILE *f;
 	size_t res;
 	struct msph_token_stream *ret;
 
-	if ((f = fopen(path, "r")) == NULL) {
+	if (ctx == NULL || f == NULL || name == NULL) {
-		MSPH_ERR(ctx, MSPH_ERR_SYS);
+		MSPH_ERR(ctx, MSPH_ERR_INVAL);
 		return (NULL);
 	}
 
+
 	if ((ret = calloc(1, sizeof(struct msph_token_stream))) == NULL) {
 		MSPH_ERR(ctx, MSPH_ERR_SYS);
 		goto err;
@@ -64,7 +133,7 @@ msph_token_stream_fopen(struct msph_ctx *ctx, const char *path)
 	ret->src.inner.file.pos = 0;
 	ret->src.inner.file.end = 0;
 
-	res = strlcpy(ret->src.inner.file.name, path,
+	res = strlcpy(ret->src.inner.file.name, name,
 	    sizeof(ret->src.inner.file.name));
 	if (res >= sizeof(ret->src.inner.file.name)) {
 		MSPH_ERR(ctx, MSPH_ERR_TOOLONG);
@@ -101,6 +170,53 @@ msph_token_stream_frombuf(struct msph_ctx *ctx, const char *buf, size_t len)
 	return (ret);
 }
 
+size_t
+msph_token_str(char *buf, size_t len, struct msph_token *tok)
+{
+	size_t ret;
+
+	ret = snprintf(buf, len, "%s", tok_base_str(tok));
+
+	if (ret > len)
+		return (ret);
+
+	len -= ret;
+	buf += ret;
+
+	switch (tok->type) {
+	case TOK_IDENT:
+		ret += snprintf(buf, len, "(%s)", tok->d.s.buf);
+		break;
+	default:
+		break;
+	}
+
+	return (ret);
+}
+
+#define MSPH_TOKEN_PRINT_BUF_LEN	2 * MSPH_TOKEN_BUF_LEN
+int
+msph_token_stream_print(struct msph_token_stream *s, FILE *out)
+{
+	ssize_t ret;
+	ssize_t i;
+	struct msph_token tok;
+	char tokstr[MSPH_TOKEN_PRINT_BUF_LEN];
+
+	while ((ret = msph_token_stream_read_tok( &tok, 1, s)) > 0) {
+		SPHO_DEBUG_PRINT("msph_token_stream_print: ret=%zd\n", ret);
+		if (msph_token_str(
+			    tokstr, BUF_LEN(tokstr), &tok) > BUF_LEN(tokstr)) {
+			tokstr[BUF_LEN(tokstr) - 1] = '\0';
+		}
+		fprintf(out, "%s\n", tokstr);
+	}
+
+	SPHO_DEBUG_PRINT("msph_token_stream_print: ret=%zd\n", ret);
+
+	return ((int)ret);
+}
+
 
 int
 msph_token_stream_close(struct msph_token_stream *s)
@@ -128,24 +244,21 @@ ssize_t
 msph_token_stream_read_tok(struct msph_token *ptr, size_t n,
     struct msph_token_stream *s)
 {
-	ssize_t ret;
+	size_t ret;
-	size_t i;
 	int res;
 
 	ret = 0;
 	res = -1;
-	while ((res = read_single_tok(&ptr[i], s)) == 1) {
+	while (ret < n && (res = read_single_tok(&ptr[ret], s)) != 0) {
-		if (res == -1) {
+		if (res == -1)
-			ret = -1;
+			return (-1);
-			break;
-		}
 		ret++;
 	}
 
-	return (ret);
+	return ((ssize_t)ret);
 }
 
-/* 1: success, 0: failed match, -1: error */
+/* 1: matched token, 0: failed to match token, -1: error */
 static int
 read_single_tok(struct msph_token *ptr, struct msph_token_stream *s)
 {
@@ -158,15 +271,23 @@ read_single_tok(struct msph_token *ptr, struct msph_token_stream *s)
 	ctx = s->ctx;
 	src = &s->src;
 
+	/* Skipping whitespace */
+	if (tok_match(ctx, src, &wspace) == -1)
+		return (-1);
+	if (wspace.matchlen > 0 &&
+	    tok_commit(ctx, src, &wspace, NULL) == -1)
+		return (-1);
+
 	max_m = 0;
-	for (m = 0; msph_matcher[m].type != TOK_END; m++) {
+	for (m = 1; token_matcher[m].type != TOK_END; m++) {
-		res = tok_match(ctx, src, &msph_matcher[m]);
+		res = tok_match(ctx, src, &token_matcher[m]);
+		SPHO_DEBUG_PRINT("read_single_tok: tok_match=%d\n", res);
 
 		if (res == -1)
 			return (-1);
 
 		if (res == 0 &&
-		    msph_matcher[m].matchlen > msph_matcher[max_m].matchlen) {
+		    token_matcher[m].matchlen > token_matcher[max_m].matchlen) {
 			max_m = m;
 		}
 	}
@@ -174,15 +295,15 @@ read_single_tok(struct msph_token *ptr, struct msph_token_stream *s)
 	if (max_m == 0)
 		return (0);
 
-	if (tok_commit(ctx, src, &msph_matcher[max_m], ptr) == -1)
+	SPHO_DEBUG_PRINT("read_single_tok: commit=%zu\n", max_m);
+	if (tok_commit(ctx, src, &token_matcher[max_m], ptr) == -1)
 		return (-1);
+	SPHO_DEBUG_PRINT("read_single_tok: committed\n");
 
 	return (1);
 }
 
 
-#define BUF_LEN(b)	(sizeof(b) / sizeof((b)[0]))
-
 static ssize_t
 src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file)
 {
@@ -207,6 +328,9 @@ src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file)
 		ret += nread;
 		file->end = (file->end + nread) % BUF_LEN(file->buf);
 
+		SPHO_DEBUG_PRINT("src_file_fill_buf: valid range (%zu, %zu)\n",
+		    file->pos, file->end);
+
 		if (nread < maxread) {
 			if (ferror(file->f)) {
 				MSPH_ERR(ctx, MSPH_ERR_SYS);
@@ -217,6 +341,8 @@ src_file_fill_buf(struct msph_ctx *ctx, struct msph_token_src_file *file)
 
 	} while (file->end != file->pos);
 
+	SPHO_DEBUG_PRINT("src_file_fill_buf: read %zd\n", ret);
+
 	return (ret);
 }
 
@@ -229,13 +355,16 @@ file_char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i,
 	ssize_t fill;
 	struct msph_token_src_file *file;
 
-	SPHO_PRECOND(s != NULL);
+	SPHO_PRECOND(src != NULL);
-	SPHO_PRECOND(s->src.type == MSPH_TOKEN_SRC_FILE);
+	SPHO_PRECOND(src->type == MSPH_TOKEN_SRC_FILE);
 
 	ret = 0;
 	file = &src->inner.file;
+	fill = 0;
 
 	do {
+		SPHO_DEBUG_PRINT("want to read %zu, valid range (%zu, %zu)\n",
+		    (file->pos + i) % BUF_LEN(file->buf), file->pos, file->end);
 		/* simplest case */
 		if (file->pos + i < file->end) {
 			*out = file->buf[file->pos + i];
@@ -250,11 +379,12 @@ file_char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i,
 
 		if (feof(file->f))
 			return (0);
-		if (src_file_fill_buf(ctx, file) == -1)
+		if ((fill = src_file_fill_buf(ctx, file)) == -1)
 			return (-1);
-	} while (ret++);
 
-	return (-1);
+	} while (fill > 0 && ret++);
+
+	return (ret);
 }
 
 static int
@@ -265,18 +395,22 @@ char_at(struct msph_ctx *ctx, struct msph_token_src *src, size_t i, char *out)
 
 	switch (src->type) {
 	case MSPH_TOKEN_SRC_FILE:
-		return (file_char_at(ctx, src, i, out));
+		ret = file_char_at(ctx, src, i, out);
+		break;
 	case MSPH_TOKEN_SRC_STR:
 		str = &src->inner.str;
 		if (str->pos + i < str->len) {
 			*out = str->s[str->pos + i];
-			return (1);
+			ret = 1;
+		} else {
+			ret = 0;
 		}
-		return (0);
+		break;
 	default:
 		break;
 	}
 
+	SPHO_DEBUG_PRINT("char_at: ret=%d, *out=%c\n", ret, *out);
 	return (ret);
 }
 
@@ -289,9 +423,11 @@ fromcbuf_charcpy(char *dst, const char *src, size_t src_len, size_t src_pos,
 		return (-1);
 	}
 
-	cpy1 = src_pos + ncpy < src_len ? ncpy : src_len - src_pos;
+	cpy1 = (src_pos + ncpy < src_len) ? ncpy : src_len - src_pos;
 	cpy2 = ncpy - cpy1;
 
+	SPHO_DEBUG_PRINT("fromcbuf_charcpy: cpy1=%zu cpy2=%zu\n", cpy1, cpy2);
+
 	memcpy(dst, &src[src_pos], cpy1 * sizeof(src[0]));
 
 	if (! cpy2)
@@ -304,27 +440,30 @@ fromcbuf_charcpy(char *dst, const char *src, size_t src_len, size_t src_pos,
 
 static int
 tok_match(struct msph_ctx *ctx, struct msph_token_src *src,
-    struct msph_token_matcher *m)
+    struct msph_matcher *m)
 {
 	int res;
 	int more;
 	char chr;
 	const char *match_str;
 	size_t off, len;
-	SPHO_PRECOND(m != NULL && s != NULL);
+	SPHO_PRECOND(m != NULL && src != NULL);
 
 	m->matchlen = 0;
 
 #define MATCH_CHAR(c) \
 	do { \
+		SPHO_DEBUG_PRINT("tok_match: '%c'\n", c); \
 		if ((res = char_at(ctx, src, 0, &chr)) == -1) \
 			return (-1); \
 		else if (res == 0) \
 			return (0); \
+		SPHO_DEBUG_PRINT("tok_match: char_at(0)='%c'\n", c); \
 \
 		if (chr == (c)) { \
 			m->matchlen = 1; \
 		} \
+		SPHO_DEBUG_PRINT("tok_match: matchlen=%zu\n", m->matchlen); \
 		return (0); \
 	} while (0)
 
@@ -397,6 +536,16 @@ tok_match(struct msph_ctx *ctx, struct msph_token_src *src,
 		if (res == -1)
 			return (-1);
 		return (0);
+	case TOK_WSPACE:
+		off = 0;
+		while((res = char_at(ctx, src, off++, &chr)) == 1) {
+			if (! isspace(chr))
+				break;
+			m->matchlen++;
+		}
+		if (res == -1)
+			return (-1);
+		return (0);
 	default:
 		SPHO_ASSERT(0);
 		return (-1);
@@ -409,40 +558,50 @@ tok_match(struct msph_ctx *ctx, struct msph_token_src *src,
 #define TOK_HAS_DATA(type)	(type == TOK_IDENT)
 static int
 tok_commit(struct msph_ctx *ctx, struct msph_token_src *src,
-    struct msph_token_matcher *m, struct msph_token *ptr)
+    struct msph_matcher *m, struct msph_token *ptr)
 {
 	size_t pos_old;
 	struct msph_token_src_str *str;
 	struct msph_token_src_file *file;
 
-	SPHO_PRECOND(p != NULL && m != NULL);
+	SPHO_PRECOND(ctx != NULL && m != NULL);
 	SPHO_PRECOND(m->matchlen != 0);
 
+	SPHO_DEBUG_PRINT("committing\n");
+
 	switch (src->type) {
 	case MSPH_TOKEN_SRC_FILE:
 		file = &src->inner.file;
 		pos_old = file->pos;
 
+		SPHO_DEBUG_PRINT("committing\n");
 		file->pos += m->matchlen;
 		file->pos %= BUF_LEN(file->buf);
 		SPHO_ASSERT(file->pos < BUF_LEN(file->buf) ||
 		    file->pos < pos_old);
 
+		SPHO_DEBUG_PRINT("committing\n");
+		if (ptr == NULL)
+			return (0);
+
 		ptr->type = m->type;
 		if (! TOK_HAS_DATA(ptr->type))
 			return (0);
 
+		SPHO_DEBUG_PRINT("committing\n");
 		if (m->matchlen >= sizeof(ptr->d.s.buf)) {
 			MSPH_ERR(ctx, MSPH_ERR_TOOLONG);
 			return (-1);
 		}
 
+		SPHO_DEBUG_PRINT("committing\n");
 		if (fromcbuf_charcpy(ptr->d.s.buf, file->buf, sizeof(file->buf),
 			    pos_old, m->matchlen) == -1) {
 			MSPH_ERR(ctx, MSPH_ERR_TOOLONG);
 			return (-1);
 		}
 
+		SPHO_DEBUG_PRINT("committing\n");
 		ptr->d.s.buf[m->matchlen] = '\0';
 		return (0);
 
@@ -453,6 +612,9 @@ tok_commit(struct msph_ctx *ctx, struct msph_token_src *src,
 		str->pos += m->matchlen;
 		SPHO_ASSERT(str->pos <= str->len);
 
+		if (ptr == NULL)
+			return (0);
+
 		ptr->type = m->type;
 		if (! TOK_HAS_DATA(ptr->type))
 			return (0);
@@ -474,6 +636,17 @@ tok_commit(struct msph_ctx *ctx, struct msph_token_src *src,
 
 
 
+static const char *
+tok_base_str(struct msph_token *tok)
+{
+	size_t i;
+	for (i = 0; token_info[i].type != TOK_END; i++) {
+		if (token_info[i].type == tok->type)
+			return (token_info[i].dbg_str);
+	}
+
+	return (NULL);
+}