token.c File Reference

#include <string.h>
#include <assert.h>
#include "util.h"
#include "token.h"
#include "ezxml.h"
#include "read_xml_util.h"

Include dependency graph for token.c:

Go to the source code of this file.

Functions
enum e_token_type	GetTokenTypeFromChar (INP enum e_token_type cur_token_type, INP char cur)
t_token *	GetTokensFromString (INP const char *inString, OUTP int *num_tokens)
void	freeTokens (INP t_token *tokens, INP int num_tokens)
boolean	checkTokenType (INP t_token token, OUTP enum e_token_type token_type)
void	my_atof_2D (INOUTP float **matrix, INP int max_i, INP int max_j, INP char *instring)

Function Documentation

boolean checkTokenType	(	INP t_token	token,
		OUTP enum e_token_type	token_type
	)

Definition at line 126 of file token.c.

                                                                             {
    if (token.type != token_type) {
        return FALSE;
    }
    return TRUE;
}

Here is the caller graph for this function:

void freeTokens	(	INP t_token *	tokens,
		INP int	num_tokens
	)

Definition at line 93 of file token.c.

                                                         {
    int i;
    for (i = 0; i < num_tokens; i++) {
        free(tokens[i].data);
    }
    free(tokens);
}

Here is the caller graph for this function:

t_token* GetTokensFromString	(	INP const char *	inString,
		OUTP int *	num_tokens
	)

Definition at line 18 of file token.c.

                                                                              {
    const char *cur;
    t_token * tokens;
    int i, in_string_index, prev_in_string_index;
    boolean has_null;
    enum e_token_type cur_token_type, new_token_type;

    *num_tokens = i = 0;
    cur_token_type = TOKEN_NULL;

    if (inString == NULL) {
        return NULL;
    };

    cur = inString;

    /* Count number of tokens */
    while (*cur) {
        new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
        if (new_token_type != cur_token_type) {
            cur_token_type = new_token_type;
            if (new_token_type != TOKEN_NULL) {
                i++;
            }
        }
        ++cur;
    }
    *num_tokens = i;

    if (*num_tokens > 0) {
        tokens = (t_token*)my_calloc(*num_tokens + 1, sizeof(t_token));
    } else {
        return NULL;
    }

    /* populate tokens */
    i = 0;
    in_string_index = 0;
    has_null = TRUE;
    prev_in_string_index = 0;
    cur_token_type = TOKEN_NULL;

    cur = inString;

    while (*cur) {

        new_token_type = GetTokenTypeFromChar(cur_token_type, *cur);
        if (new_token_type != cur_token_type) {
            if (!has_null) {
                tokens[i - 1].data[in_string_index - prev_in_string_index] =
                        '\0'; /* NULL the end of the data string */
                has_null = TRUE;
            }
            if (new_token_type != TOKEN_NULL) {
                tokens[i].type = new_token_type;
                tokens[i].data = my_strdup(inString + in_string_index);
                prev_in_string_index = in_string_index;
                has_null = FALSE;
                i++;
            }
            cur_token_type = new_token_type;
        }
        ++cur;
        in_string_index++;
    }

    assert(i == *num_tokens);

    tokens[*num_tokens].type = TOKEN_NULL;
    tokens[*num_tokens].data = NULL;

    /* Return the list */
    return tokens;
}

Here is the call graph for this function:

Here is the caller graph for this function:

enum e_token_type GetTokenTypeFromChar	(	INP enum e_token_type	cur_token_type,
		INP char	cur
	)

Jason Luu July 22, 2009 Tokenizer

Definition at line 101 of file token.c.

                      {
    if (IsWhitespace(cur)) {
        return TOKEN_NULL;
    } else {
        if (cur == '[') {
            return TOKEN_OPEN_SQUARE_BRACKET;
        } else if (cur == ']') {
            return TOKEN_CLOSE_SQUARE_BRACKET;
        } else if (cur == '{') {
            return TOKEN_OPEN_SQUIG_BRACKET;
        } else if (cur == '}') {
            return TOKEN_CLOSE_SQUIG_BRACKET;
        } else if (cur == ':') {
            return TOKEN_COLON;
        } else if (cur == '.') {
            return TOKEN_DOT;
        } else if (cur >= '0' && cur <= '9' && cur_token_type != TOKEN_STRING) {
            return TOKEN_INT;
        } else {
            return TOKEN_STRING;
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void my_atof_2D	(	INOUTP float **	matrix,
		INP int	max_i,
		INP int	max_j,
		INP char *	instring
	)

Definition at line 133 of file token.c.

                            {
    int i, j;
    char *cur, *cur2, *copy, *final;

    copy = my_strdup(instring);
    final = copy;
    while (*final != '\0') {
        final++;
    }

    cur = copy;
    i = j = 0;
    while (cur != final) {
        while (IsWhitespace(*cur) && cur != final) {
            if (j == max_j) {
                i++;
                j = 0;
            }
            cur++;
        }
        if (cur == final) {
            break;
        }
        cur2 = cur;
        while (!IsWhitespace(*cur2) && cur2 != final) {
            cur2++;
        }
        *cur2 = '\0';
        assert(i < max_i && j < max_j);
        matrix[i][j] = atof(cur);
        j++;
        cur = cur2;
        *cur = ' ';
    }

    assert((i == max_i && j == 0) || (i == max_i - 1 && j == max_j));

    free(copy);
}

Here is the call graph for this function:

Here is the caller graph for this function: