/*  ibicc - The Ibi C Compiler
    Copyright (C) 2004 Antti-Juhani Kaijanaho

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions
    are met:

    1. Redistributions of source code must retain the above copyright
       notice, this list of conditions and the following disclaimer.

    2. Redistributions in binary form must reproduce the above
       copyright notice, this list of conditions and the following
       disclaimer in the documentation and/or other materials provided
       with the distribution.

    THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
    OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
    WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
    ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
    DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
    DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
    GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "asprintf.h"
#include "arena.h"
#include "diag.h"
#include "file.h"
#include "ibicpp.h"
#include "ibi_symtab.h"
#include "strutil.h"

#include <errno.h>
#include <limits.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <time.h>

/*#define BUFFER_SIZE 16384*/

struct ibicpp_token const ibicpp_token_init = { .tcat = -2,
						.semval = { 0, 0, 0 },
						.file = 0,
						.line = 0 };

struct dirvec {
	char **vec;
	size_t n;
	size_t max;
};

static struct file *find_file(struct dirvec * dv, char const *fn, struct file * parent,
			      int system_file)
{
	size_t i, n, fnlen;
	struct file *file = 0;
	if (fn[0] == '/' || fn[0] == '.') return new_file(fn, parent, system_file);
	fnlen = strlen(fn);
	n = dv->n;
	for (i = 0; file == 0 && i < n; i++) {
		char *tmpstr = 0;
		char const *dn = dv->vec[n-i-1];
		size_t dnlen = strlen(dn);
		int need_slash;
		size_t rvlen;
		char *rf;

		if (dnlen == 0 && parent != 0) {
			char *p;
			/* find the parent file's directory */
			tmpstr = strdup(parent->af->fname);
			if (tmpstr == 0) enomem();
			p = strrchr(tmpstr, '/');
			if (p != 0) {
				*p = '\0';
				dn = tmpstr;
			}
		}

		/*fprintf(stderr, "[try_dir=%s]\n", dn);*/
		need_slash = dnlen > 0 && dn[dnlen-1] != '/';
		rvlen = dnlen + need_slash + fnlen + 1;
		rf = malloc(rvlen);

		strcpy(rf, dn);
		if (need_slash) strcat(rf, "/");
		strcat(rf, fn);

		/*fprintf(stderr, "[try_file=%s]\n", rf);*/

		file = new_file(rf, parent, system_file);

		free(rf);
		free(tmpstr);
	}
	return file;
}

static void append_dir(struct dirvec *dv, char const *dn)
{
	char *dn2;
	assert(dv != 0);
	assert(dv->n <= dv->max);
	if (dv->n == dv->max) {
		size_t nmax = dv->max == 0 ? 16 : 2*dv->max;
		char **nvec = realloc(dv->vec, nmax * sizeof *nvec);
		if (nvec == 0) enomem();
		dv->max = nmax;
		dv->vec = nvec;
	}
	assert(dv->n < dv->max);
	dn2 = strdup(dn);
	if (dn2 == 0) enomem();
	dv->vec[dv->n++] = dn2;
}

static void dirvec_clear(struct dirvec *dv)
{
	size_t i;
	for (i = 0; i < dv->n; i++) free(dv->vec[i]);
	free(dv->vec);
	dv->n = 0;
	dv->max = 0;
}

/* token string */
struct tokstr {
	char *ident; /* must be malloced or 0 (will be freed by tokstr_free) */
	size_t n, maxn;
	size_t i; /* used as an index to toks by tok_peek */
	struct ibicpp_token * toks;
};

static struct tokstr const tokstr_init = {
	.i = 0, .ident = 0, .n = 0, .maxn = 0, .toks = 0
};

static void tokstr_free(struct tokstr *ts)
{
	free(ts->ident);
	free(ts->toks);
	ts->toks = 0;
	ts->maxn = 0;
	ts->n = 0;
}

static void tokstr_append(struct tokstr *ts, struct ibicpp_token tok)
{
	assert(ts->n <= ts->maxn);
	if (ts->n == ts->maxn) {
		size_t nmax = ts->maxn == 0 ? 4 : 2 * ts->maxn;
		struct ibicpp_token * ntoks = realloc(ts->toks, nmax * sizeof *ntoks);
		if (ntoks == 0) enomem();
		ts->maxn = nmax;
		ts->toks = ntoks;
	}
	assert(ts->n < ts->maxn);
	tok.semval = tok.semval;
	ts->toks[ts->n++] = tok;
}

static void tokstr_append_ts(struct tokstr *r, struct tokstr const *s)
{
	size_t i;
	if (r->n + s->n > r->maxn) {
		size_t nmax = r->maxn == 0 ? 4 : 2 * r->maxn;
		while (nmax < r->n + s->n) nmax *= 2;
		struct ibicpp_token * ntoks = realloc(r->toks, nmax * sizeof *ntoks);
		if (ntoks == 0) enomem();
		r->maxn = nmax;
		r->toks = ntoks;
	}
	assert(r->n + s->n <= r->maxn);
	for (i = 0; i < s->n; i++) {
		r->toks[r->n+i] = s->toks[i];
	}
	r->n += s->n;
	assert(r->n <= r->maxn);
}

static int tokstr_eq(struct tokstr const *ts1, struct tokstr const *ts2)
{
	size_t i;
	if (ts1->n != ts2->n) return 0;
	for (i = 0; i < ts1->n; i++) {
		if (ts1->toks[i].tcat != ts2->toks[i].tcat ||
		    sstrcmp(ts1->toks[i].semval, ts2->toks[i].semval) != 0) {
			return 0;
		}
	}
	return 1;
}

typedef unsigned char byte_t;

/* compilation unit */
struct ibicpp_source {
	arena_t arena;
	ibi_symtab_t symtab;
	int in_include_directive;
	struct file * file;
	int num_files;
	unsigned new_file : 1;
	unsigned reissue_line_marker : 1;
	short curr_tcat;
	substr_t curr_semval;
	char const *curr_fname;
	size_t curr_line;
	int curr_bol;
	size_t if_depth;
	struct tokstr **tokstr_stack;
	size_t tokstr_max;
	size_t tokstr_top;
	struct dirvec sys_include_dirs;
	struct dirvec usr_include_dirs;
	/*byte_t */
};

arena_t ibicpp_get_arena(ibicpp_source_t s)
{
	return s->arena;
}

void ibicpp_append_system_include_dir(ibicpp_source_t s, char const *dirname)
{
	append_dir(&s->sys_include_dirs, dirname);
}

void ibicpp_append_user_include_dir(ibicpp_source_t s, char const *dirname)
{
	append_dir(&s->usr_include_dirs, dirname);
}

static ibicpp_source_t open_source(struct file *f)
{
	arena_t arena = arena_init();
	struct ibicpp_source *is;
	new(is,arena);
	is->arena = arena;
	is->file = f;
	is->symtab = ibi_symtab_new();
	if (is->symtab == 0) goto fail_clean_is;
	is->new_file = 1;
	is->reissue_line_marker = 0;
	is->num_files = 1;
	is->in_include_directive = 0;
	is->curr_tcat = -2;
	is->curr_semval = empty_substr;
	is->curr_fname = 0;
	is->curr_line = 0;
	is->curr_bol = 1;
	is->if_depth = 0;
	is->tokstr_stack = 0;
	is->tokstr_max = 0;
	is->tokstr_top = 0;
	is->sys_include_dirs = (struct dirvec){ .vec = 0, .n = 0, .max = 0 };
	is->usr_include_dirs = (struct dirvec){ .vec = 0, .n = 0, .max = 0 };
	append_dir(&is->usr_include_dirs, "");
	return is;

fail_clean_is:
	free(is);
	return 0;
}

static void introduce_standard_defines(struct ibicpp_source *s)
{
	static char const months[][4] = { "Jan", "Feb", "Mar", "Apr",
					  "May", "Jun", "Jul", "Aug",
					  "Sep", "Oct", "Nov", "Dec" };
	time_t t;
	struct tm *tm;
	char date[14], tim[11];
	time(&t);
	tm = localtime(&t);
	sprintf(date, "\"%s %2d %4d\"", months[tm->tm_mon],
		tm->tm_mday, tm->tm_year+1900);
	sprintf(tim, "\"%.2d:%.2d:%.2d\"", tm->tm_hour, tm->tm_min, tm->tm_sec);
	ibicpp_define_symbol(s, "__DATE__", date, "<standard>");
	ibicpp_define_symbol(s, "__TIME__", tim, "<standard>");
}

ibicpp_source_t ibicpp_open_source(char const *filename)
{
	struct file *file;
	ibicpp_source_t is;
	file = new_file(filename, 0, 0);
	if (file == 0) return 0;
	is = open_source(file);
	if (is == 0) {
		close_file(file);
		return 0;
	}
	introduce_standard_defines(is);
	return is;
}

void ibicpp_close_source(ibicpp_source_t s)
{
	struct file *file = s->file;
	while (file != 0) {
		struct file *nf = file->parent;
		close_file(file);
		free(file);
		file = nf;
	}
	ibi_symtab_free(s->symtab);
	s->curr_semval = empty_substr;
	dirvec_clear(&s->sys_include_dirs);
	dirvec_clear(&s->usr_include_dirs);
	arena_fini(s->arena);
}

static void eat_comment(struct ibicpp_source * src)
{
	assert(src != 0);
	while (1) {
		while (file_peek(src->file) != '*' && file_peek(src->file) != '/') {
			file_get(src->file);
		}
		if (file_peek(src->file) == '/') {
			file_get(src->file);
			if (file_peek(src->file) == '*') {
				warn(src->file->af->fname, src->file->current_line,
				     "comments nest");
			}
			continue;
		}
		file_eat(src->file, '*');
		if (file_peek(src->file) == '/') {
			file_get(src->file);
			return;
		}
	}
}

static void eat_line_comment(struct ibicpp_source * src)
{
	while (!src->file->is_beginning_of_logical_line) {
		if (file_peek(src->file) == '/') {
			file_get(src->file);
			if (file_peek(src->file) == '*') {
				warn(src->file->af->fname, src->file->current_line,
				     "comments nest");
			}
		}
		file_get(src->file);
	}
}

static substr_t pp_num(struct ibicpp_source * src, int decimal_point_read)
{
	substr_t st;
	size_t n;
	get_substr(&st, src->file);
	if (decimal_point_read) {
		--st.buf;
		++st.len;
		assert(st.buf[0] == '.');
	}
	do {
		n = st.len;
		while ('0' <= file_peek(src->file) && file_peek(src->file) <= '9') {
			file_substr_add(&st, src->file);
			file_get(src->file);
		}
		if (file_peek(src->file) == 'e' || file_peek(src->file) == 'E' ||
		    file_peek(src->file) == 'p' || file_peek(src->file) == 'P') {
			file_substr_add(&st, src->file);
			file_get(src->file);
			if (file_peek(src->file) == '+' ||
			    file_peek(src->file) == '-') {
				file_substr_add(&st, src->file);
				file_get(src->file);
			}
		} else if (file_peek(src->file) == '.' ||
			   ('a' <= file_peek(src->file) && file_peek(src->file) <= 'z') ||
			   ('A' <= file_peek(src->file) && file_peek(src->file) <= 'Z') ||
			   file_peek(src->file) == '_') {
			file_substr_add(&st, src->file);
			file_get(src->file);
/*
		} else if (file_peek(src->file) == '\\') {
			universal_character_name();
			dynstr_append(&ds, file_peek(src->file));
			file_get(src->file);
*/
		}
	} while (st.len > n);
	return st;
}

static substr_t sc_char_seq(struct ibicpp_source * src, char delimiter)
{
	int esc = 0;
	substr_t st;
	get_substr(&st, src->file);
	while (esc || file_peek(src->file) != delimiter) {
		esc = file_peek(src->file) == '\\';
		if (file_peek(src->file) == -1 || file_peek(src->file) == '\n') {
			err(src->file->af->fname, src->file->current_line, 
			    "unterminated character constant");
			break;
		}
		file_substr_add(&st, src->file);
		file_get(src->file);
	}
	file_get(src->file);
	return st;
}

static void install_token(struct ibicpp_source * src)
{
	char c;
	if (src->file == 0) { src->curr_tcat = -1; return; }
	get_substr(&src->curr_semval, src->file);
	src->curr_fname = src->file->af->fname;
	src->curr_line = src->file->current_line;
	src->curr_bol = src->file->is_beginning_of_logical_line;
	if (src->new_file || src->reissue_line_marker) {
		int r;
		char *s;
		char *s2;
		src->curr_tcat = IBICPP_LINE_MARKER;
		r = asprintf(&s, "1 \"%s\" %s%s", src->file->af->fname,
			     src->new_file ? " 1" : "",
			     src->file->is_system_file ? " 3" : "");
		if (r == -1) enomem();
		src->new_file = 0;
		src->reissue_line_marker = 0;
		newa(s2, (size_t)(r+1), src->arena);
		strcpy(s2, s);
		free(s);
		cstr2substr(&src->curr_semval, s2);
		return;
	}
	c = file_peek(src->file);
	switch (c) {
	case -1:
	{
		char *s, *s2;
		int r;
		struct file *of = src->file;
		src->file = of->parent;
		if (of->af->real_file) --src->num_files;
		close_file(of);
		if (src->file == 0) {
			src->curr_tcat = -1;
			break;
		}
		src->curr_tcat = IBICPP_LINE_MARKER;
		r = asprintf(&s, "%d \"%s\" 2%s",
			     src->file->current_line, src->file->af->fname,
			     src->file->is_system_file ? " 3" : "");
		if (r == -1) enomem();
		newa(s2, (size_t)(r+1), src->arena);
		strcpy(s2, s);
		free(s);
		cstr2substr(&src->curr_semval, s2);
		break;
	}
	case '[': case ']': case '(': case ')':
	case '{': case '}': case '~': case ';':
	case '?': case ',': case ' ': case ':':
	case '\n':
		src->curr_tcat = file_get(src->file);
		break;
	case '\t':
		file_get(src->file);
		src->curr_tcat = ' ';
		break;
	case '.':
		file_get(src->file);
		if (file_peek(src->file) == '.') {
			file_get(src->file);
			if (file_peek(src->file) == '.') {
				src->curr_tcat = IBICPP_TOK_ELLIPSIS;
				break;
			}
			err(src->file->af->fname, src->file->current_line, "`.' expected");
			break;
		}
		if ('0' <= file_peek(src->file) && file_peek(src->file) <= '9') {
			src->curr_semval = pp_num(src, 1);
			src->curr_tcat = IBICPP_PP_NUMBER;
			break;
		}
		src->curr_tcat = '.';
		break;
	case '-':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '-':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_MINUSMINUS;
			break;
		case '>':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_RARROW;
			break;
		case '=':
			file_get(src->file);
			src->curr_tcat = '-' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '-';
		}
		break;
	case '+':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '+':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_PLUSPLUS;
			break;
		case '=':
			file_get(src->file);
			src->curr_tcat = '+' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '+';
		}
		break;
	case '&':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '&':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_ETET;
			break;
		case '=':
			file_get(src->file);
			src->curr_tcat = '&' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '&';
		}
		break;
	case '*':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '=':
			file_get(src->file);
			src->curr_tcat = '*' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '*';
		}
		break;
	case '!':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '=':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_NEQ;
			break;
		default:
			src->curr_tcat = '!';
			break;
		}
		break;
	case '/':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '*':
			file_get(src->file);
			eat_comment(src);
			src->curr_tcat = ' ';
			break;
		case '/':
			file_get(src->file);
			eat_line_comment(src);
			src->curr_tcat = '\n';
			break;
		case '=':
			src->curr_tcat = '/' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '/';
			break;
		}
		break;
	case '%':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '=':
			file_get(src->file);
			src->curr_tcat = '%' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '%';
			break;
		}
		break;
	case '<':
		file_get(src->file);
		if (src->in_include_directive) {
			/* parse a <header.name> */
			get_substr(&src->curr_semval, src->file);
			while (file_peek(src->file) != '>') {
				if (file_peek(src->file) == '&' ||
				    file_peek(src->file) == '\\' ||
				    file_peek(src->file) == '?' ||
				    (src->curr_semval.len > 0 &&
				     src->curr_semval.buf[
					     src->curr_semval.start +
					     src->curr_semval.len - 1
					     ] == '/' && 
				     (file_peek(src->file) == '/' ||
				      file_peek(src->file) == '*'))) {
					warn(src->file->af->fname, src->file->current_line, 
					     "invokes undefined behaviour (6.4.7:3)");
				}
				if (file_peek(src->file) == -1 || 
				    file_peek(src->file) == '\n') {
					err(src->file->af->fname, src->file->current_line,
					    "unterminated header name");
					break;
				}
				file_substr_add(&src->curr_semval, src->file);
				file_get(src->file);
			}
			file_get(src->file);
			src->curr_tcat = IBICPP_SYS_HDR_NAME;
			break;
		}
		switch (file_peek(src->file)) {
		case '<':
			file_get(src->file);
			if (file_peek(src->file) == '=') {
				file_get(src->file);
				src->curr_tcat = IBICPP_TOK_LTLT + IBICPP_OPEQ_SHIFT;
				break;
			}
			src->curr_tcat = IBICPP_TOK_LTLT;
			break;
		case '=':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_LEQ;
			break;
		default:
			src->curr_tcat = '<';
		}
		break;
	case '>':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '>':
			file_get(src->file);
			if (file_peek(src->file) == '=') {
				file_get(src->file);
				src->curr_tcat = IBICPP_TOK_GTGT + IBICPP_OPEQ_SHIFT;
				break;
			}
			src->curr_tcat = IBICPP_TOK_GTGT;
			break;
		case '=':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_GEQ;
			break;
		default:
			src->curr_tcat = '>';
		}
		break;
	case '^':
		file_get(src->file);
	        switch (file_peek(src->file)) {
		case '=':
			file_get(src->file);
			src->curr_tcat = '^' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '^';
		}
		break;
	case '|':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '|':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_VLVL;
			break;
		case '=':
			file_get(src->file);
			src->curr_tcat = '|' + IBICPP_OPEQ_SHIFT;
			break;
		default:
			src->curr_tcat = '|';
		}
		break;
	case '=':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '=':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_EQEQ;
			break;
		default:
			src->curr_tcat = '=';
		}
		break;
	case '#':
		file_get(src->file);
		switch (file_peek(src->file)) {
		case '#':
			file_get(src->file);
			src->curr_tcat = IBICPP_TOK_HASHASH;
			break;
		default:
			src->curr_tcat = '#';
		}
		break;
	case '"':
		file_get(src->file);
		if (src->in_include_directive) {
			/* parse a "header.name" */
			get_substr(&src->curr_semval, src->file);
			while (file_peek(src->file) != '"') {
				if (file_peek(src->file) == '\'' ||
				    file_peek(src->file) == '\\' ||
				    (src->curr_semval.len > 0 &&
				     src->curr_semval.buf[
					     src->curr_semval.start +
					     src->curr_semval.len - 1
					     ] == '/' &&
				     (file_peek(src->file) == '/' ||
				      file_peek(src->file) == '*'))) {
					warn(src->file->af->fname, src->file->current_line,
					     "invokes undefined behaviour (6.4.7:3)");
				}
				if (file_peek(src->file) == -1 ||
				    file_peek(src->file) == '\n') {
					err(src->file->af->fname, src->file->current_line,
					    "unterminated header name");
					break;
				}
				     
				file_substr_add(&src->curr_semval, src->file);
				file_get(src->file);
			}
			file_get(src->file);
			src->curr_tcat = IBICPP_USR_HDR_NAME;
			break;
		}
		src->curr_semval = sc_char_seq(src, '"');
		src->curr_tcat = IBICPP_STR_LIT;
		break;
	case '\'':
		file_get(src->file);
		src->curr_semval = sc_char_seq(src, '\'');
		src->curr_tcat = IBICPP_CHAR_CONST;
		break;
	default:
		if ('0' <= c && c <= '9') {
			src->curr_semval = pp_num(src, 0);
			src->curr_tcat = IBICPP_PP_NUMBER;
			break;
		}
		if (('a' <= file_peek(src->file) && file_peek(src->file) <= 'z') ||
		    ('A' <= file_peek(src->file) && file_peek(src->file) <= 'Z') ||
		    file_peek(src->file) == '_') {
			get_substr(&src->curr_semval, src->file);
			do {
				file_substr_add(&src->curr_semval, src->file);
				file_get(src->file);
			} while (('a' <= file_peek(src->file) && file_peek(src->file) <= 'z') ||
				 ('A' <= file_peek(src->file) && file_peek(src->file) <= 'Z') ||
				 ('0' <= file_peek(src->file) && file_peek(src->file) <= '9') ||
				 file_peek(src->file) == '_');
			src->curr_tcat = IBICPP_IDENTIFIER;
			break;
		}
		src->curr_tcat = file_get(src->file);
	}
}

/*
#define tok_peek(is) (((is)->curr_tcat == -2 ? \
                         (is)->tokstr_top > 0 ? \
                           (void)tok_peek_((is)) : \
                           (void)install_token((is)) : \
                          (void)0), \
                       (is)->curr_tcat)
*/

static short tok_peek(struct ibicpp_source *is)
{
	while  (is->curr_tcat == -2) {
		if (is->tokstr_top > 0) {
			assert(is->tokstr_stack != 0);
			assert(is->tokstr_top <= is->tokstr_max);
			struct tokstr * ts = is->tokstr_stack[is->tokstr_top-1];
			if (ts->i < ts->n) {
				is->curr_tcat = ts->toks[ts->i].tcat;
				is->curr_semval = ts->toks[ts->i].semval;
				is->curr_fname = ts->toks[ts->i].file;
				is->curr_line = ts->toks[ts->i].line;
				++ts->i;
				continue;
			}
			--is->tokstr_top;
			continue;
		}
		install_token(is);
	}
	return is->curr_tcat;
}

static void tok_eat(struct ibicpp_source *s)
{
	if (s->curr_tcat == -2) tok_peek(s);
	s->curr_tcat = -2;
}

static short tok_get(struct ibicpp_source *s)
{
	short rv = tok_peek(s);
	tok_eat(s);
	return rv;
}

static void eat_spaces(struct ibicpp_source *s)
{
	while (tok_peek(s) == ' ') tok_eat(s);
}

static void tok_peek_copy(struct ibicpp_source *s, struct ibicpp_token *t)
{
	t->tcat = tok_peek(s);
	t->semval = s->curr_semval;
	t->file = s->curr_fname;
	t->line = s->curr_line;
}

static struct tokstr *find_arg(struct ibicpp_token *ptoks, struct tokstr *args,
			       size_t nargs, substr_t *arg)
{
	size_t i;
	substr_t va_args;
	cstr2substr(&va_args, "VA_ARGS");
	for (i = 0; i < nargs; i++) {
		if ((ptoks[i].tcat == IBICPP_TOK_ELLIPSIS &&
		     sstrcmp(*arg, va_args) == 0) ||
		    (ptoks[i].tcat == IBICPP_IDENTIFIER &&
		     sstrcmp(ptoks[i].semval, *arg) == 0)) {
			return &args[i];
		}			
	}
	return 0;
}

#define static_strdup(s) static_strdup_((s), &buf, &blen);

static void buf_reserve(char **buf, size_t *len, size_t minlen)
{
	if (*len < minlen) {
		if (*len == 0) *len = 16;
		while (*len < minlen) *len *= 2;
		*buf = realloc(*buf, *len);
		if (*buf == 0) enomem();
	}
}

static char *static_strdup_(char const *s, char **buf, size_t *len)
{
	size_t n = (s == 0 ? *len : strlen(s)) + 1;
	buf_reserve(buf, len, n);
	assert(n < *len);
	if (s != 0) strcpy(*buf, s);
	return *buf;
}

char *tok2str(struct ibicpp_token t)
{
	static char *buf = 0;
	static size_t blen = 0;
	short tc = t.tcat;
	char *s = 0;
	/*fprintf(stderr, "[tok2str:%d]\n", tc);*/
	switch (tc) {
	case IBICPP_TOK_PLUSPLUS:
		s = static_strdup("++");
		break;
	case IBICPP_TOK_MINUSMINUS:
		s = static_strdup("--");
		break;
	case IBICPP_TOK_LTLT:
		s = static_strdup("<<");
		break;
	case IBICPP_TOK_GTGT:
		s = static_strdup(">>");
		break;
	case IBICPP_TOK_LEQ:
		s = static_strdup("<=");
		break;
	case IBICPP_TOK_GEQ:
		s = static_strdup(">=");
		break;
	case IBICPP_TOK_EQEQ:
		s = static_strdup("==");
		break;
	case IBICPP_TOK_NEQ:
		s = static_strdup("!=");
		break;
	case IBICPP_TOK_ETET:
		s = static_strdup("&&");
		break;
	case IBICPP_TOK_VLVL:
		s = static_strdup("&&");
		break;
	case IBICPP_TOK_ELLIPSIS:
		s = static_strdup("...");
		break;
	case IBICPP_TOK_HASHASH:
		s = static_strdup("##");
		break;
	case IBICPP_TOK_RARROW:
		s = static_strdup("->");
		break;
	case IBICPP_IDENTIFIER: case IBICPP_PP_NUMBER:
		buf_reserve(&buf, &blen, t.semval.len+1);
		memcpy(buf, t.semval.buf + t.semval.start, t.semval.len);
		buf[t.semval.len] = 0;
		s = buf;
		break;
	case IBICPP_STR_LIT: case IBICPP_USR_HDR_NAME:
	{
		size_t n = t.semval.len;
		/*fprintf(stderr, "[tok2str/STR_LIT:%s]\n", t.semval);*/
		buf_reserve(&buf, &blen, n+3);
		s = buf;
		s[0] = '"';
		memcpy(s+1, t.semval.buf+t.semval.start, n);
		s[n+1] = '"';
		s[n+2] = '\0';
		/*fprintf(stderr, "[tok2str/STR_LIT:%s]\n", s);*/
		break;
	}
	case IBICPP_CHAR_CONST:
	{
		size_t n = t.semval.len;
		buf_reserve(&buf, &blen, n+3);
		s = buf;
		s[0] = '\'';
		memcpy(s+1, t.semval.buf+t.semval.start, n);
		s[n+1] = '\'';
		s[n+2] = '\0';
		break;
	}
	case IBICPP_SYS_HDR_NAME:
	{
		size_t n = t.semval.len;
		buf_reserve(&buf, &blen, n+3);
		s = buf;
		s[0] = '<';
		memcpy(s+1, t.semval.buf+t.semval.start, n);
		s[n+1] = '>';
		s[n+2] = '\0';
		break;
	}
	case IBICPP_LINE_MARKER:
	{
		size_t n = t.semval.len;
		buf_reserve(&buf, &blen, n+4);
		s = buf;
		s[0] = '#';
		s[1] = ' ';
		memcpy(s+2, t.semval.buf+t.semval.start, n);
		s[n+2] = '\n';
		s[n+3] = '\0';
		break;
	}
	case IBICPP_PLACEMARKER:
		s = static_strdup("");
		break;
	default:
		if (0 <= tc && tc < 256) {
			s = static_strdup("  ");
			s[0] = tc;
			break;
		}
		if (IBICPP_OPEQ_SHIFT <= tc && tc < IBICPP_IDENTIFIER) {
			size_t n;
			t.tcat -= IBICPP_OPEQ_SHIFT;
			s = tok2str(t);
			n = strlen(s);
			s = static_strdup(0);
			s[n+0] = '=';
			s[n+1] = 0;
			break;
		}
		/*fprintf(stderr, "[[[%d]]]\n", tc);*/
		assert(0);
	}
	return s;
}



static char *stringify(struct tokstr *ts)
{
	char *rvs;
	size_t i;
	dynstr_t rv = empty_str;
	for (i = 0; i < ts->n; i++) {
		char *s = tok2str(ts->toks[i]);
		size_t j;
		size_t n = strlen(s);
		for (j = 0; j < n; j++) {
			if (strchr("\"\'\\", s[j]) != 0) dynstr_append(&rv, '\\');
			dynstr_append(&rv, s[j]);
		}
	}
	rvs = dynstr_cstr(&rv);
	dynstr_clear(&rv);
	return rvs;
}


static struct tokstr *push_tokstr(struct ibicpp_source *s, char const *id)
{
	struct tokstr *ts;
	/* add the current invocation to the top of the tokstr stack */
	assert(s->tokstr_top <= s->tokstr_max);
	if (s->tokstr_top == s->tokstr_max) {
		s->tokstr_max = s->tokstr_max == 0 ? 4 : 2 * s->tokstr_max;
		s->tokstr_stack = realloc(s->tokstr_stack,
					  s->tokstr_max * sizeof *s->tokstr_stack);
		if (s->tokstr_stack == 0) enomem();
	}
	assert(s->tokstr_top <= s->tokstr_max);
	
	s->tokstr_stack[s->tokstr_top] = malloc(sizeof *s->tokstr_stack[s->tokstr_top]);
	ts = s->tokstr_stack[s->tokstr_top++];
	if (ts == 0) enomem();
	*ts = tokstr_init;
	ts->ident = strdup(id);
	if (ts->ident == 0) enomem();
	return ts;
}

/* macro definition */
struct macro {
	struct tokstr params;
	struct tokstr repl;
};

static struct macro *magic_macro(struct ibicpp_source *s, char const *id)
{
	if (strcmp(id, "__FILE__") == 0) {
		char * str;
		struct macro *m;
		struct ibicpp_token t;
		new(m, s->arena);
		m->params = tokstr_init;
		m->repl = tokstr_init;
		t.tcat = IBICPP_USR_HDR_NAME;
		cstr2substr(&t.semval, s->curr_fname);
		t.file = s->curr_fname;
		t.line = s->curr_line;
		str = tok2str(t);
	        cstr2substr(&t.semval, str);
		t.tcat = IBICPP_STR_LIT;
		tokstr_append(&m->repl, t);
		return m;
	} else if (strcmp(id, "__LINE__") == 0) {
		int r;
		char *str1, *str2;
		size_t str1l;
		struct macro *m;
		struct ibicpp_token t;
		new(m, s->arena);
		m->params = tokstr_init;
		m->repl = tokstr_init;
		t.tcat = IBICPP_STR_LIT;
		t.file = s->curr_fname;
		t.line = s->curr_line;
		r = asprintf(&str1, "%d\n", s->curr_line);
		if (r == -1) enomem();
		str1l = strlen(str1);
		newa(str2, str1l+1, s->arena);
		strcpy(str2, str1);
		free(str1);
		cstr2substr(&t.semval, str2);
		tokstr_append(&m->repl, t);
		return m;
	} else {
		return 0;
	}
}

static int expand_macro(struct ibicpp_source *s)
{
	static unsigned char argument_sentinel;
	size_t i, j;
	substr_t ids;
	char *id;
	char const *fname;
	int line;
	struct macro *md = 0;
	struct tokstr *ts;
	struct tokstr *args = 0;
	size_t nargs = 0;
	int rv = 0;
	if (tok_peek(s) != IBICPP_IDENTIFIER) return 0;
	fname = s->curr_fname;
	line = s->curr_line;
	ids = s->curr_semval;
	id = substr_cstr(&ids);
	md = magic_macro(s, id);
	if (md == 0) md = ibi_symtab_lookup(s->symtab, id);
	if (md == 0) goto exit;

	/* Detect and ignore macro recursion */
	for (i = 0; i < s->tokstr_top; i++) {
		if (strcmp(s->tokstr_stack[i]->ident, id) == 0) goto exit;
	}
	tok_eat(s);

	/* Scan the argument list. */
	nargs = md->params.n;
	if (nargs > 0) {
		size_t pardepth = 0;
		args = malloc(nargs * sizeof *args);
		if (args == 0) enomem();
		for (i = 0; i < nargs; i++) args[i] = tokstr_init;
		if (tok_peek(s) != '(') {
			struct ibicpp_token t;
			struct tokstr *ts = push_tokstr(s, id);
			t.tcat = IBICPP_IDENTIFIER;
			t.semval = ids;
			t.file = fname;
			t.line = line;
			tokstr_append(ts, t);
			tok_peek_copy(s, &t);
			tokstr_append(ts, t);
			tok_eat(s);
			goto exit;
		}
		tok_eat(s);
		i = 0;
		while (pardepth > 0 || tok_peek(s) != ')') {
			struct ibicpp_token t;
			/*
			fprintf(stderr, "[expand_macro/args:%d:%d ('%c'):%s]\n",
				pardepth, tok_peek(s), (char)tok_peek(s),
				dynstr_cstr(&s->curr_semval));
			*/
			switch (tok_peek(s)) {
			case -1:
				err(fname, line,
				    "macro invocation extends to the end of file");
				goto exit;
			case ',':
				if (pardepth == 0 &&
				    md->params.toks[i].tcat != IBICPP_TOK_ELLIPSIS) {
					++i;
					if (i == md->params.n) {
						err(s->curr_fname, s->curr_line,
						    "too many macro arguments");
						goto exit;
					}
					tok_eat(s);
					continue;
				}
				break;
			case '(': ++pardepth; break;
			case ')': --pardepth; break;
			}
			tok_peek_copy(s, &t);
			if (t.tcat == IBICPP_TOK_HASHASH) {
				/* mark this ## as coming from the
				 * argument (normally the t.semval is
				 * 0) */
				t.semval.buf = &argument_sentinel;
			}
			tokstr_append(&args[i], t);
			tok_eat(s);
		}
		tok_eat(s);
		if (args[i].n > 0) i++;
		if (i < nargs) {
			err(s->curr_fname, s->curr_line, "too few macro arguments");
			goto exit;
		}
	}

	ts = push_tokstr(s, id);
		
	/*fprintf(stderr, "[expand_macro]\n");*/
	/* build the tokstr */
	for (i = 0; i < md->repl.n; i++) {
		struct ibicpp_token t = ibicpp_token_init;
		struct tokstr *a;
		int done = 0;
		switch (md->repl.toks[i].tcat) {
		case '#':
			++i;
			/* we assume that do_define has already
			 * checked the constraint (6.10.3.2:1) */
			assert(i < md->repl.n);
			assert(md->repl.toks[i].tcat == IBICPP_IDENTIFIER);
			a = find_arg(md->params.toks, args, nargs,
				     &md->repl.toks[i].semval);
			assert(a != 0);
			t.tcat = IBICPP_STR_LIT;
			t.file = fname;
			t.line = line;
			/* FIXME: memory leak */
			cstr2substr(&t.semval, stringify(a));
			tokstr_append(ts, t);
			done = 1;
			break;
		case IBICPP_TOK_HASHASH: /* ## */
			/* we assume that do_define has already
			 * checked the constraint (6.10.3.3:1) */
			assert(i > 0);
			assert(i + 1 < md->repl.n);
			a = find_arg(md->params.toks, args, nargs,
				     &md->repl.toks[i-1].semval);
			t.tcat = IBICPP_PLACEMARKER;
			t.file = 0;
			t.line = 0;
			t.semval = empty_substr;
			if (a != 0 && a->n == 0) tokstr_append(ts, t);
			a = find_arg(md->params.toks, args, nargs,
				     &md->repl.toks[i+1].semval);
			tokstr_append(ts, md->repl.toks[i]);
			if (a != 0 && a->n == 0) tokstr_append(ts, t);
			done = 1;
			break;
		case IBICPP_IDENTIFIER:
			a = find_arg(md->params.toks, args, nargs,
				     &md->repl.toks[i].semval);
			if (a != 0) {
				tokstr_append_ts(ts, a);
				done = 1;
			}
			break;
		}
		if (!done) tokstr_append(ts, md->repl.toks[i]);
	}
	rv = 1;

	for (i = 0, j = 0; j < ts->n; j++) {
		if (ts->toks[j].tcat == IBICPP_TOK_HASHASH &&
		    ts->toks[j].semval.buf != &argument_sentinel) {
			/* the semval test checked that the ## token
			 * is not from arguments (6.10.3.3:3) */
			struct ibicpp_token t = ibicpp_token_init;
			struct file *file;
			struct ibicpp_source *is;
			char *s1, *s2, *s3;
			size_t sn;
			assert(i > 0);
			assert(j + 1 < ts->n);
			s1 = tok2str(ts->toks[i-1]);
			s2 = tok2str(ts->toks[j+1]);
			sn = strlen(s1) + strlen(s2) + 2;
			s3 = malloc(sn);
			if (s3 == 0) enomem();
			strcpy(s3, s1);
			strcat(s3, s2);
			/*fprintf(stderr, "[expand_macro/##:%s]\n", s3);*/
			s3[sn-2] = '\n';
			s3[sn-1] = '\0';
			file = new_string_file(ts->toks[j].file,
					       ts->toks[j].line,
					       0, s3);
			if (file == 0) enomem();
			is = open_source(file);
			if (is == 0) enomem();
			assert(tok_peek(is) == IBICPP_LINE_MARKER);
			tok_eat(is);
			tok_peek_copy(is, &t);
			tok_eat(is);
			if (tok_get(is) != '\n' || tok_peek(is) != -1) {
				err(ts->toks[j].file, ts->toks[j].line,
				    "invalid token paste");
			}
			ibicpp_close_source(is);
			free(s1);
			free(s2);
			free(s3);
			ts->toks[i-1] = t;
			j++;
			continue;
		}
		ts->toks[i++] = ts->toks[j];
	}
	ts->n = i;
exit:
	/*fprintf(stderr, "[expand_macro:%d]\n", s->tokstr_top);*/
	for (i = 0; i < nargs; i++) tokstr_free(&args[i]);
	free(args);
	free(id);
	return rv;
}

static void eat_until_eol(struct ibicpp_source *s)
{
	while (tok_peek(s) != '\n' && tok_peek(s) != -1) tok_eat(s);
}

static void do_define(struct ibicpp_source *s)
{
	char const *fname;
	size_t i, j;
	int line;
	substr_t id;
	char * cid;
	struct macro m = { tokstr_init, tokstr_init };
	struct macro *md;
	tok_eat(s);
	eat_spaces(s);

	if (tok_peek(s) != IBICPP_IDENTIFIER) {
		err(s->curr_fname, s->curr_line, "macro name identifier expected");
		eat_until_eol(s);
		return;
	}
	id = s->curr_semval;
	cid = substr_cstr(&id);
	fname = s->curr_fname;
	line = s->curr_line;
	tok_eat(s);
	if (tok_peek(s) == '(') {
		/* parameters */
		tok_eat(s);
		eat_spaces(s);
		if (tok_peek(s) != ')') {
			do {
				struct ibicpp_token t = ibicpp_token_init;
				eat_spaces(s);
				tok_peek_copy(s, &t);
				tokstr_append(&m.params, t);
				tok_eat(s);
				eat_spaces(s);
			} while (tok_get(s) == ',');
		}
	}

	for (i = 0; i < m.params.n; i++) {
		struct ibicpp_token * tp = &m.params.toks[i];
		if (tp->tcat == IBICPP_TOK_ELLIPSIS) {
			if (i != m.params.n - 1) {
				err(tp->file, tp->line, 
				    "an ellipsis ('...') should be last "
				    "item on a parameter list");
				goto fail_params;
			}
		}
	}

	eat_spaces(s);
	while (tok_peek(s) != '\n') {
		struct ibicpp_token t = ibicpp_token_init;
		if (tok_peek(s) == -1) {
			err(s->curr_fname, s->curr_line, "invalid #define");
			goto fail_repl;
		}
		tok_peek_copy(s, &t);
		tokstr_append(&m.repl, t);
		tok_eat(s);
	}


	/* Canonicalize replacement list (combine adjacent space
	 * tokens) */
	for (i = 0, j = 0; j < m.repl.n; j++) {
		struct ibicpp_token *tp1 = &m.repl.toks[i];
		struct ibicpp_token *tp2 = &m.repl.toks[j];
		if (tp1 != tp2 && tp1->tcat == ' ' && tp2->tcat == ' ') continue;
		*tp2 = *tp1;
		i++;
	}
	m.repl.n = i;

	/* Check if already defined; that is allowed if the parameter
	 * lists and the canonical replacement lists are identical
	 * (6.10.3:2) */
	md = ibi_symtab_lookup(s->symtab, cid);
	if (md != 0) {
		if (!tokstr_eq(&m.params, &md->params) ||
		    !tokstr_eq(&m.repl, &md->repl)) {
			err(fname, line, "macro %s already defined (differently)",
			    cid);
		}
		goto fail_repl;
	}

	md = malloc(sizeof *md);
	if (md == 0) enomem();

	*md = m;
	ibi_symtab_bind(s->symtab, cid, md);
	return;

fail_repl:
	tokstr_free(&m.repl);
fail_params:
	tokstr_free(&m.params);

	free(cid);
}

static void do_undef(struct ibicpp_source *s)
{
	char *id;
	tok_eat(s);
	eat_spaces(s);
	if (tok_peek(s) != IBICPP_IDENTIFIER) {
		err(s->curr_fname, s->curr_line, "malformed #undef directive");
		eat_until_eol(s);
		return;
	}
	id = substr_cstr(&s->curr_semval);
	ibi_symtab_remove(s->symtab, id);
	free(id);
	tok_eat(s);
	eat_spaces(s);
	if (tok_get(s) != '\n') {
		err(s->curr_fname, s->curr_line, "garbage at end of #undef directive");
		eat_until_eol(s);
		return;
	}
}

static void do_include(struct ibicpp_source *s)
{
	char *hdr;
	struct file * file = 0;
	char const *cfn; int cli;

	if (s->num_files > 512) {
		err(s->curr_fname, s->curr_line,
		    "include nesting too deep (baseless recursion?)");
		eat_until_eol(s);
		return;
	}

	s->in_include_directive = 1;
	tok_eat(s);
	eat_spaces(s);
	hdr = substr_cstr(&s->curr_semval);
	switch (tok_peek(s)) {
	case IBICPP_USR_HDR_NAME:
		file = find_file(&s->usr_include_dirs, hdr, s->file, 0);
		/* fallthrough */
	case IBICPP_SYS_HDR_NAME:
		if (file == 0) {
			file = find_file(&s->sys_include_dirs, hdr, s->file, 1);
		}
		cfn = s->curr_fname;
		cli = s->curr_line;
		tok_eat(s);
		if (file == 0) {
			err(s->curr_fname, s->curr_line,
			    "cannot find header file '%s'", hdr);
			break;
		}
		while (tok_peek(s) != '\n' && tok_peek(s) != -1) {
			if (tok_peek(s) != ' ') err(cfn, cli, "garbage at end of line");
			tok_eat(s);
		}
		s->file = file;
		s->new_file = 1;
		++s->num_files;
		break;
	default:
		err(s->curr_fname, s->curr_line, "malformed #include directive");
	}
	free(hdr);
	s->in_include_directive = 0;
}

static void do_line(struct ibicpp_source *s)
{
	unsigned long line;
	char *ds = 0, *fn = 0;
	char *ep;
	tok_eat(s);
	eat_spaces(s);
	if (tok_peek(s) != IBICPP_PP_NUMBER) {
		err(s->curr_fname, s->curr_line, "number expected after #line");
		goto fail;
	}
	ds = substr_cstr(&s->curr_semval);
	tok_eat(s);
	eat_spaces(s);
	if (tok_peek(s) != '\n') {
		if (tok_peek(s) == IBICPP_STR_LIT) {
			fn = substr_cstr(&s->curr_semval);
			tok_eat(s);
		} else {
			err(s->curr_fname, s->curr_line, "garbage at the end of #line");
		}
	}
	eat_spaces(s);
	if (tok_peek(s) != '\n') {
		err(s->curr_fname, s->curr_line, "garbage at the end of #line");
		eat_until_eol(s);
	}

	errno = 0;
	line = strtoul(ds, &ep, 10);
	if (line == ULONG_MAX && errno != 0) {
		err(s->curr_fname, s->curr_line, "specified line number is broken: %s",
		    strerror(errno));
		goto fail;
	}
	if (line >= (unsigned long)INT_MAX) {
		err(s->curr_fname, s->curr_line, "specified line number is too large");
		goto fail;
	}
	s->file->current_line = (int)line;
	if (fn != 0) s->file->af->fname = fn;
	s->reissue_line_marker = 1;
	free(ds);
	return;

fail:
	free(fn);
	free(ds);
	eat_until_eol(s);
}

static void do_err_warn(err_warn_t ew, struct ibicpp_source *s)
{
	dynstr_t msg = empty_str;
	char *str;
	tok_eat(s);
	eat_spaces(s);
	while (tok_peek(s) != -1 && tok_peek(s) != '\n') {
		struct ibicpp_token t;
/*
		fprintf(stderr, "[do_err_warn:%d (%c):%s]\n",
			s->curr_tcat, s->curr_tcat, dynstr_cstr(&s->curr_semval));
*/
		tok_peek_copy(s, &t);
		str = tok2str(t);
		dynstr_append_s(&msg, str);
		free(str);
		tok_eat(s);
	}
	str = dynstr_cstr(&msg);
	ew(s->curr_fname, s->curr_line, "%s", str);
	free(str);
	dynstr_clear(&msg);
}

enum directive { D_NONE, D_DEFINE, D_ELIF, D_ELSE, D_ENDIF, D_ERROR,
		 D_IF, D_IFDEF, D_IFNDEF, D_INCLUDE, D_LINE, D_PRAGMA,
		 D_WARNING, D_UNDEF };

static enum directive intern_directive(struct ibicpp_source *s)
{
	if (tok_peek(s) != IBICPP_IDENTIFIER) return D_NONE;
	switch (substr_at(&s->curr_semval, 0)) {
	case 'd':
		if (scstrcmp(s->curr_semval, "define") != 0) return D_NONE;
		return D_DEFINE;
	case 'e':
		switch (substr_at(&s->curr_semval, 1)) {
		case 'l':
			if (scstrcmp(s->curr_semval, "else") == 0) {
				return D_ELSE;
			} if (scstrcmp(s->curr_semval, "elif") == 0) {
				return D_ELIF;
			} else {
				return D_NONE;
			}
		case 'n':
			if (scstrcmp(s->curr_semval, "endif") != 0) return D_NONE;
			return D_ENDIF;
		case 'r':
			if (scstrcmp(s->curr_semval, "error") != 0) return D_NONE;
			return D_ERROR;
		default:
			return D_NONE;
		}
	case 'i':
		switch (substr_at(&s->curr_semval, 1)) {
		case 'f':
			if (s->curr_semval.len == 2) {
				return D_IF;
			} else if (scstrcmp(s->curr_semval, "ifdef") == 0) {
				return D_IFDEF;
			} else if (scstrcmp(s->curr_semval, "ifndef") == 0) {
				return D_IFNDEF;
			} else {
				return D_NONE;
			}
		case 'n':
			if (scstrcmp(s->curr_semval, "include") != 0) return D_NONE;
			return D_INCLUDE;
		default:
			return D_NONE;
		}
	case 'l':
		if (scstrcmp(s->curr_semval, "line") != 0) return D_NONE;
		return D_LINE;
	case 'p':
		if (scstrcmp(s->curr_semval, "pragma") != 0) return D_NONE;
		return D_PRAGMA;
	case 'w':
		if (scstrcmp(s->curr_semval, "warning") != 0) return D_NONE;
		return D_WARNING;
	case 'u':
		if (scstrcmp(s->curr_semval, "undef") != 0) return D_NONE;
		return D_UNDEF;
	default:
		return D_NONE;
	}
}

static void do_skipping(struct ibicpp_source *s)
{
	size_t level = 1;
	while (level > 0) {
		tok_peek(s);
		while (tok_peek(s) != -1 && !s->curr_bol) tok_eat(s);

		if (tok_peek(s) == -1) {
			err(s->curr_fname, s->curr_line, "missing #endif");
			s->if_depth = 0;
			return;
		}

		assert(s->curr_bol);

		eat_spaces(s);

		/*fprintf(stderr, "[do_skipping:%d (%c)]\n",
			tok_peek(s), (char)tok_peek(s));*/
		if (tok_get(s) != '#') continue;

		eat_spaces(s);

		/*fprintf(stderr, "[do_skipping/dir:%d (%c):%s]\n",
		  tok_peek(s), (char)tok_peek(s), dynstr_cstr(&s->curr_semval));*/
		switch (intern_directive(s)) {
		case D_IF: case D_IFDEF: case D_IFNDEF:
			++level;
			break;
		case D_ELIF: case D_ELSE:
			if (level == 1) return;
			break;
		case D_ENDIF:
			--level;
			break;
		default:
			;
		}
	}
}

struct value {
	enum { SIGNED, UNSIGNED, NAN } tag;
	union {
		intmax_t signd;
		uintmax_t unsignd;
		struct {
			char const *file;
			int line;
			char const *msg;
		} err;
	} u;
};

static void vals_promote(struct value *v1, struct value *v2)
{
	if (v1->tag == v2->tag) return;
	if (v1->tag == NAN) return;
	if (v2->tag == NAN) return;
	if (v1->tag == SIGNED) {
		assert(v2->tag == UNSIGNED);
		v1->u.unsignd = v1->u.signd;
		v1->tag = UNSIGNED;
		return;
	}
	assert(v1->tag == UNSIGNED);
	assert(v2->tag == SIGNED);
	v2->u.unsignd = v2->u.signd;
	v2->tag = UNSIGNED;
}

static int is_nonzero(struct value v)
{
	switch (v.tag) {
	case SIGNED:   return v.u.signd != 0;
	case UNSIGNED: return v.u.unsignd != 0;
	case NAN:
		err(v.u.err.file, v.u.err.line, "%s", v.u.err.msg);
	}
	return 0;
}

static struct ibicpp_token ts_peek(struct tokstr *ts)
{
	if (ts->i < ts->n) return ts->toks[ts->i];
	return (struct ibicpp_token){ .tcat = -1, .semval = {0,0,0}, .file = 0, .line = 0 };
}

static void ts_eat(struct tokstr *ts)
{
	if (ts->i == ts->n) return;
	++ts->i;
}

static void ts_get(struct tokstr *ts, short tcat)
{
	if (ts_peek(ts).tcat != tcat) {
		struct ibicpp_token t = ibicpp_token_init;
		t.tcat = tcat;
		char *s = tok2str(t);
		err(ts_peek(ts).file, ts_peek(ts).line, "'%s' expected", s);
		free(s);
	}
	ts_eat(ts);
}

static struct value eval_expr(struct tokstr *ts);

static struct value primary_expr(struct tokstr *ts)
{
	struct value v;
	v.tag = NAN;
	v.u.err.file = ts_peek(ts).file;
	v.u.err.line = ts_peek(ts).line;
	v.u.err.msg = "syntax error in preprocessor expression";
	switch (ts_peek(ts).tcat) {
	case '(':
		ts_eat(ts);
		v = eval_expr(ts);
		ts_get(ts, ')');
		break;
	case IBICPP_IDENTIFIER:
		v.tag = SIGNED;
		v.u.signd = 0;
		ts_eat(ts);
		break;
	case IBICPP_PP_NUMBER:
	{
		uintmax_t rv = 0;
		substr_t semval = ts_peek(ts).semval;
		size_t n = semval.len;
		size_t i;
		int radix;
		int unsignd = 0;
		if (substr_at(&semval, 0) == '0') {
			if (substr_at(&semval, 1) == 'x' ||
			    substr_at(&semval, 1) == 'X') {
				radix = 16;
			} else {
				radix = 8;
			}
		} else {
			radix = 10;
		}
		for (i = radix == 16 ? 2 : 0; i < n; i++) {
			char c = substr_at(&semval, i);
			if (('0' <= c && c <= '7') ||
			    (radix != 8 && (c == '8' || c == '9'))) {
				rv = rv * radix + (c - '0');
			} else if (radix == 16 && 'a' <= c && c <= 'f') {
				rv = rv * radix + (c - 'a' + 10);
			} else if (radix == 16 && 'A' <= c && c <= 'F') {
				rv = rv * radix + (c - 'A' + 10);
			} else {
				break;
			}
		}
		for (; i < n; i++) {
			char c = substr_at(&semval, i);
			if (c == 'u' || c == 'U') {
				unsignd = 1;
			} else if (c != 'l' && c != 'L') {
				err(ts_peek(ts).file, ts_peek(ts).line,
				      "malformed integer constant");
			}
		}
		if (rv > INTMAX_MAX) unsignd = 1;
		if (unsignd) {
			v.tag = UNSIGNED;
			v.u.unsignd = rv;
		} else {
			v.tag = SIGNED;
			v.u.signd = rv;
		}
		ts_eat(ts);
		break;
	}
	default:
		ts_eat(ts);
	}
	return v;
}

static struct value unary_expr(struct tokstr *ts)
{
	struct value v;
	switch (ts_peek(ts).tcat) {
	case '+':
		ts_eat(ts);
		return unary_expr(ts);
	case '-':
		ts_eat(ts);
		v = unary_expr(ts);
		switch (v.tag) {
		case SIGNED:
			v.u.signd = -v.u.signd;
			break;
		case UNSIGNED:
			v.u.unsignd = -v.u.unsignd;
			break;
		case NAN: ;
		}
		return v;
	case '~':
		ts_eat(ts);
		v = unary_expr(ts);
		switch (v.tag) {
		case SIGNED:
			v.u.signd = ~v.u.signd;
			break;
		case UNSIGNED:
			v.u.unsignd = ~v.u.unsignd;
			break;
		case NAN: ;
		}
		return v;
	case '!':
		ts_eat(ts);
		v = unary_expr(ts);
		switch (v.tag) {
		case SIGNED:
			v.u.signd = !v.u.signd;
			break;
		case UNSIGNED:
			v.u.unsignd = !v.u.unsignd;
			break;
		case NAN: ;
		}
		return v;
	default:
		return primary_expr(ts);
	}

}

static struct value mult_expr(struct tokstr *ts)
{
	struct value v = unary_expr(ts);
	while (ts_peek(ts).tcat == '*' ||
	       ts_peek(ts).tcat == '/' ||
	       ts_peek(ts).tcat == '%') {
		struct value vr;
		short tc = ts_peek(ts).tcat;
		char const *file = ts_peek(ts).file;
		int line = ts_peek(ts).line;
		ts_eat(ts);
		vr = unary_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED:
			switch (tc) {
			case '*':
				v.u.signd = v.u.signd * vr.u.signd;
				break;
			case '/':
				if (vr.u.signd == 0) goto dbz;
				v.u.signd = v.u.signd / vr.u.signd;
				break;
			case '%': 
				if (vr.u.signd == 0) goto dbz;
				v.u.signd = v.u.signd % vr.u.signd;
				break;
			}
			break;
		case UNSIGNED:
			switch (tc) {
			case '*':
				v.u.unsignd = v.u.unsignd * vr.u.unsignd;
				break;
			case '/':
				if (vr.u.signd == 0) goto dbz;
				v.u.unsignd = v.u.unsignd / vr.u.unsignd;
				break;
			case '%': 
				if (vr.u.signd == 0) goto dbz;
				v.u.unsignd = v.u.unsignd % vr.u.unsignd;
				break;
			}
			break;
		case NAN: ;
		}
		continue;
	dbz:
		v.tag = NAN;
		v.u.err.file = file;
		v.u.err.line = line;
		v.u.err.msg = "division by zero";
	}
	return v;
}

static struct value additive_expr(struct tokstr *ts)
{
	struct value v = mult_expr(ts);
	while (ts_peek(ts).tcat == '+' || ts_peek(ts).tcat == '-') {
		struct value vr;
		int plus = ts_peek(ts).tcat == '+';
		ts_eat(ts);
		vr = mult_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED:
			if (plus) {
				v.u.signd = v.u.signd + vr.u.signd;
			} else {
				v.u.signd = v.u.signd - vr.u.signd;
			}
			break;
		case UNSIGNED:
			if (plus) {
				v.u.unsignd = v.u.unsignd + vr.u.unsignd;
			} else {
				v.u.unsignd = v.u.unsignd - vr.u.unsignd;
			}
			break;
		case NAN: ;
		}
	}
	return v;
}

static struct value shift_expr(struct tokstr *ts)
{
	struct value v = additive_expr(ts);
	while (ts_peek(ts).tcat == IBICPP_TOK_LTLT ||
	       ts_peek(ts).tcat == IBICPP_TOK_GTGT) {
		struct value vr;
		int ltlt = ts_peek(ts).tcat == IBICPP_TOK_LTLT;
		ts_eat(ts);
		vr = additive_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED:
			if (ltlt) {
				v.u.signd = v.u.signd << vr.u.signd;
			} else {
				v.u.signd = v.u.signd >> vr.u.signd;
			}
			break;
		case UNSIGNED:
			if (ltlt) {
				v.u.unsignd = v.u.unsignd << vr.u.unsignd;
			} else {
				v.u.unsignd = v.u.unsignd >> vr.u.unsignd;
			}
			break;
		case NAN: ;
		}
	}
	return v;
}

static struct value rel_expr(struct tokstr *ts)
{
	struct value v = shift_expr(ts);
	while (ts_peek(ts).tcat == '<' ||
	       ts_peek(ts).tcat == '>' ||
	       ts_peek(ts).tcat == IBICPP_TOK_GEQ ||
	       ts_peek(ts).tcat == IBICPP_TOK_LEQ) {
		struct value vr;
		short tc = ts_peek(ts).tcat;
		ts_eat(ts);
		vr = shift_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED:
			switch (tc) {
			case '<':
				v.u.signd = v.u.signd < vr.u.signd; break;
			case '>':
				v.u.signd = v.u.signd > vr.u.signd; break;
			case IBICPP_TOK_LEQ: 
				v.u.signd = v.u.signd <= vr.u.signd; break;
			case IBICPP_TOK_GEQ:
				v.u.signd = v.u.signd >= vr.u.signd; break;
			}
			break;
		case UNSIGNED:
			switch (tc) {
			case '<':
				v.u.unsignd = v.u.unsignd < vr.u.unsignd; break;
			case '>':
				v.u.unsignd = v.u.unsignd > vr.u.unsignd; break;
			case IBICPP_TOK_LEQ: 
				v.u.unsignd = v.u.unsignd <= vr.u.unsignd; break;
			case IBICPP_TOK_GEQ:
				v.u.unsignd = v.u.unsignd >= vr.u.unsignd; break;
			}
			break;
		case NAN: ;
		}
	}
	return v;
}

static struct value eq_expr(struct tokstr *ts)
{
	struct value v = rel_expr(ts);
	while (ts_peek(ts).tcat == IBICPP_TOK_EQEQ ||
	       ts_peek(ts).tcat == IBICPP_TOK_NEQ) {
		struct value vr;
		int neq = ts_peek(ts).tcat == IBICPP_TOK_NEQ;
		ts_eat(ts);
		vr = rel_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED:
			if (neq) {
				v.u.signd = v.u.signd != vr.u.signd;
			} else {
				v.u.signd = v.u.signd == vr.u.signd;
			}
			break;
		case UNSIGNED:
			if (neq) {
				v.u.unsignd = v.u.unsignd != vr.u.unsignd;
			} else {
				v.u.unsignd = v.u.unsignd == vr.u.unsignd;
			}
			break;
		case NAN: ;
		}
	}
	return v;
}

static struct value and_expr(struct tokstr *ts)
{
	struct value v = eq_expr(ts);
	while (ts_peek(ts).tcat == '&') {
		struct value vr;
		ts_eat(ts);
		vr = eq_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED: v.u.signd &= vr.u.signd; break;
		case UNSIGNED: v.u.unsignd &= vr.u.unsignd; break;
		case NAN: ;
		}
	}
	return v;
}

static struct value xor_expr(struct tokstr *ts)
{
	struct value v = and_expr(ts);
	while (ts_peek(ts).tcat == '^') {
		struct value vr;
		ts_eat(ts);
		vr = and_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED: v.u.signd ^= vr.u.signd; break;
		case UNSIGNED: v.u.unsignd ^= vr.u.unsignd; break;
		case NAN: ;
		}
	}
	return v;
}

static struct value or_expr(struct tokstr *ts)
{
	struct value v = xor_expr(ts);
	while (ts_peek(ts).tcat == '|') {
		struct value vr;
		ts_eat(ts);
		vr = xor_expr(ts);
		vals_promote(&v, &vr);
		if (vr.tag == NAN) v = vr;
		switch (v.tag) {
		case SIGNED: v.u.signd |= vr.u.signd; break;
		case UNSIGNED: v.u.unsignd |= vr.u.unsignd; break;
		case NAN: ;
		}
	}
	return v;
}

static struct value land_expr(struct tokstr *ts)
{
	struct value v = or_expr(ts);
	while (ts_peek(ts).tcat == IBICPP_TOK_ETET) {
		struct value vr;
		ts_eat(ts);
		vr = or_expr(ts);
		vals_promote(&v, &vr);
		if (v.tag == NAN) continue;
		if (is_nonzero(v)) v = vr;
	}
	return v;
}

static struct value lor_expr(struct tokstr *ts)
{
	struct value v = land_expr(ts);
	while (ts_peek(ts).tcat == IBICPP_TOK_VLVL) {
		struct value vr;
		ts_eat(ts);
		vr = land_expr(ts);
		vals_promote(&v, &vr);
		if (v.tag == NAN) continue;
		if (!is_nonzero(v)) v = vr;
	}
	return v;
}

static struct value eval_expr(struct tokstr *ts)
{
	struct value v1, v2, v3;
	v1 = lor_expr(ts);
	if (ts_peek(ts).tcat != '?') return v1;
	ts_eat(ts);
	v2 = eval_expr(ts);
	ts_get(ts, ':');
	v3 = eval_expr(ts);
	return is_nonzero(v1) ? v2 : v3;
}

static int do_if(struct ibicpp_source *s)
{
	int r;
	struct tokstr ts = tokstr_init;
	tok_eat(s);
	eat_spaces(s);
	while (tok_peek(s) != '\n') {
		struct ibicpp_token t = ibicpp_token_init;
		switch (tok_peek(s)) {
		case ' ': tok_eat(s); continue;
		case IBICPP_IDENTIFIER:
			if (scstrcmp(s->curr_semval, "defined") == 0) {
				char *id;
				int parens = 0;
				tok_eat(s);
				eat_spaces(s);
				if (tok_peek(s) == '(') {
					parens = 1;
					tok_eat(s);
				}
				eat_spaces(s);
				if (tok_peek(s) != IBICPP_IDENTIFIER) {
					err(s->curr_fname, s->curr_line,
					    "malformed 'defined' operator application");
				}
				id = substr_cstr(&s->curr_semval);
				t.tcat = IBICPP_PP_NUMBER;
				t.file = s->curr_fname;
				t.line = s->curr_line;
				if (ibi_symtab_lookup(s->symtab, id) != 0) {
					cstr2substr(&t.semval, "1");
				} else {
					cstr2substr(&t.semval, "0");
				}
				free(id);
				tok_eat(s);
				if (parens) {
					eat_spaces(s);
					if (tok_peek(s) != ')') {
					err(s->curr_fname, s->curr_line,
					    "')' expected");
					}
					tok_eat(s);
				}
				break;
			}
			expand_macro(s);
			/* passthrough */
		default:
			tok_peek_copy(s, &t);
			tok_eat(s);
		}
		tokstr_append(&ts, t);
	}
	tok_eat(s);

/*
	{
		char *tmp = stringify(&ts);
		fprintf(stderr, "[do_if:%s]\n", tmp);
		free(tmp);
	}
*/

	r = is_nonzero(eval_expr(&ts));
	tokstr_free(&ts);
	return r;
}

static void directive(struct ibicpp_source *s)
{
	char *id;
	int pos;
	int skipped = 0;
redo:
	pos = 0;
	while (tok_peek(s) == ' ') tok_eat(s);
	if (tok_peek(s) == '\n') return;
/*	fprintf(stderr, "[directive:%s:%d]\n", dynstr_cstr(&s->curr_semval),
	s->if_depth);*/
	switch (intern_directive(s)) {
	case D_IFDEF:
		pos = 1;
		/* passthrough */
	case D_IFNDEF:
		tok_eat(s);
		eat_spaces(s);
		if (tok_peek(s) != IBICPP_IDENTIFIER) {
			err(s->curr_fname, s->curr_line, "invalid #%s directive",
			    pos ? "ifdef" : "ifndef");
			eat_until_eol(s);
			return;
		}
		id = substr_cstr(&s->curr_semval);
		tok_eat(s);
		eat_spaces(s);
		if (tok_peek(s) != '\n') {
			err(s->curr_fname, s->curr_line,
			    "garbage after end of directive");
			eat_until_eol(s);
			free(id);
			return;
		}
		++s->if_depth;
		if (pos == (ibi_symtab_lookup(s->symtab, id) != 0)) {
			free(id);
		} else {
			do_skipping(s);
			skipped = 1;
			free(id);
			goto redo;
		}
		break;
	case D_ELIF:
		if (s->if_depth == 0) {
			err(s->curr_fname, s->curr_line, "stray #elif");
			break;
		}
		/* passthrough */
	case D_IF:
		++s->if_depth;
		if (!do_if(s)) {
			do_skipping(s);
			skipped = 1;
			goto redo;
		}
		break;
	case D_ENDIF:
		/*fprintf(stderr, "[directive/endif]\n");*/
		tok_eat(s);
		eat_spaces(s);
		if (tok_peek(s) != '\n') {
			err(s->curr_fname, s->curr_line,
			    "garbage after directive");
			eat_until_eol(s);
			break;
		}
		if (s->if_depth == 0) {
			err(s->curr_fname, s->curr_line, "stray #endif");
			break;
		} else {
			--s->if_depth;
		}
		/*fprintf(stderr, "[directive/endif:%d]\n", s->if_depth);*/
		break;
	case D_ELSE:
		tok_eat(s);
		eat_spaces(s);
		if (tok_peek(s) != '\n') {
			err(s->curr_fname, s->curr_line, "garbage after directive");
			eat_until_eol(s);
			break;
		}
		if (s->if_depth == 0) {
			err(s->curr_fname, s->curr_line, "stray #else");
			break;
		}
		if (skipped) break;
		do_skipping(s);
		skipped = 1;
		goto redo;
	case D_ERROR:
		do_err_warn(err, s);
		break;
	case D_WARNING:
		do_err_warn(warn, s);
		break;
	case D_INCLUDE:
		do_include(s);
		break;
	case D_DEFINE:
		do_define(s);
		break;
	case D_UNDEF:
		do_undef(s);
		break;
	case D_LINE:
		do_line(s);
		break;
	default:
		eat_until_eol(s);
	}
	return;
}

void ibicpp_define_symbol(ibicpp_source_t s, char const *sym, char const *def,
			  char const *fn)
{
	static char const hashdef[] = "#define ";
	char *str;
	struct file * file;
	str = malloc(sizeof hashdef + strlen(sym) + strlen(def) + 3);
	strcpy(str, hashdef);
	strcat(str, sym);
	strcat(str, " ");
	strcat(str, def);
	strcat(str, "\n");
	
	file = new_string_file(fn, 1, s->file, str);
	if (file == 0) enomem();
	file->is_beginning_of_logical_line = 1;
	s->file = file;
	free(str);
}

void ibicpp_token_print(FILE * fp, struct ibicpp_token const *buf,
			size_t base, size_t count)
{
	size_t i;
	for (i = base; i < base + count; i++) {
		char *s = tok2str(buf[i]);
		fputs(s, fp);
	}
}

void ibicpp_dump_macros(FILE *fp, struct ibicpp_source *s)
{
	size_t i;
	ibi_symtab_iterator_t it;
	for (it = ibi_symtab_begin(s->symtab);
	     ibi_symtab_iterator_valid(it);
	     ibi_symtab_iterator_next(it)) {
		char const *id = ibi_symtab_iterator_key(it);
		struct macro *m = ibi_symtab_iterator_data(it);
		assert(id != 0);
		assert(m != 0);

		if (m->repl.n > 0) {
			fprintf(fp, "/* %s:%d */\n",
				m->repl.toks[0].file,
				m->repl.toks[0].line);
		} else {
			fputs("/* (unknown source) */\n", fp);
		}
		fprintf(fp, "#define %s", id);
		if (m->params.n > 0) {
			fputc('(', fp);
			for (i = 0; i < m->params.n; i++) {
				char *str = tok2str(m->params.toks[i]);
				if (i > 0) fputs(", ", fp);
				fputs(str, fp);
				free(str);
			}
			fputc(')', fp);
		}
		fputc(' ', fp);
		for (i = 0; i < m->repl.n; i++) {
			char *str = tok2str(m->repl.toks[i]);
			fputs(str, fp);
			free(str);
		}		
		fputc('\n', fp);
	}
}

size_t ibicpp_get_tokens(struct ibicpp_source *s,
			 struct ibicpp_token *buf, size_t base, size_t count)
{
	size_t i;
	int just_saw_space = 0;
	for (i = 0; i < count; i++) {
		int r;
		if (just_saw_space && tok_peek(s) == ' ') {
			int bll = s->curr_bol;
			tok_eat(s);
			s->curr_bol = bll;
			--i;
			continue;
		}

		tok_peek(s);
		if (s->curr_bol) {
			while (tok_peek(s) == ' ') tok_eat(s);
			if (tok_peek(s) == '#' && s->tokstr_top == 0) {
				tok_eat(s);
				directive(s);
				--i;
				just_saw_space = 0;
				continue;
			}
		}

		r = expand_macro(s);
		if (r) {
			--i;
			just_saw_space = 0;
			continue;
		}

		just_saw_space = tok_peek(s) == ' ';

		if (tok_peek(s) == -1) break;
		buf[base+i].tcat = s->curr_tcat;
		buf[base+i].semval = s->curr_semval;
		buf[base+i].file = s->curr_fname;
		buf[base+i].line = s->curr_line;
		tok_eat(s);
	}
	return i;
}