Confronta il testo

Trova la differenza tra due file di testo

Editor live

Comprimi invariate

Senza a capo

Layout

Livello di dettaglio

Evidenziazione sintassi

Diffchecker Desktop Il modo più sicuro per usare Diffchecker. Ottieni l'app Diffchecker Desktop: i tuoi diff non lasciano mai il tuo computer!Ottieni Desktop

lexing_goto_v_musttail

Creato 12 mesi faIl diff non scade mai

47 rimozioni

Linee
Totale
Rimosso

Caratteri
Totale
Rimosso

Per continuare a utilizzare questa funzione, aggiorna a Diffchecker Pro Visualizza prezzi

311 linee

109 aggiunte

Linee
Totale
Aggiunto

Caratteri
Totale
Aggiunto

Per continuare a utilizzare questa funzione, aggiorna a Diffchecker Pro Visualizza prezzi

354 linee

#include "lexer.h"

#include "common.h"

#include "mem.h"

#include "strings.h"

#include <stddef.h>

#include <stdint.h>

#include <stdio.h>

#include <stdlib.h>

#define SINGLE_TOK(t) ((Token){.type = t})

Str TOKEN_TYPE_MAP[] = {[T_DELIMITOR_LEFT] = STRING("T_DELIMITOR_LEFT"),

[T_DELIMITOR_RIGHT] = STRING("T_DELIMITOR_RIGHT"),

[T_BRAKET_LEFT] = STRING("T_BRAKET_LEFT"),

[T_BRAKET_RIGHT] = STRING("T_BRAKET_RIGHT"),

[T_STRING] = STRING("T_STRING"),

[T_TRUE] = STRING("T_TRUE"),

[T_FALSE] = STRING("T_FALSE"),

[T_DOUBLE] = STRING("T_DOUBLE"),

[T_INTEGER] = STRING("T_INTEGER"),

[T_BUILTIN] = STRING("T_BUILTIN"),

[T_IDENT] = STRING("T_IDENT"),

[T_PLUS] = STRING("T_PLUS"),

[T_MINUS] = STRING("T_MINUS"),

[T_ASTERISKS] = STRING("T_ASTERISKS"),

[T_SLASH] = STRING("T_SLASH"),

[T_EQUAL] = STRING("T_EQUAL"),

[T_EOF] = STRING("T_EOF")};

Lexer Lexer_new(Str input) {

return (Lexer){

.input = input,

.pos = 0,

};

}

#define cur(L) (L->input.p[L->pos])

__attribute__((always_inline)) inline static bool is_alphanum(uint8_t cc) {

uint8_t lower = cc | 0x20;

bool is_alpha = (lower >= 'a' && lower <= 'z');

bool is_digit = (cc >= '0' && cc <= '9');

return is_alpha || is_digit || cc == '_' || cc == '-';

}

// we can "intern" these, since all of them are the same, regardless of position

Token *INTERN_DELIMITOR_LEFT = &SINGLE_TOK(T_DELIMITOR_LEFT);

Token *INTERN_DELIMITOR_RIGHT = &SINGLE_TOK(T_DELIMITOR_RIGHT);

Token *INTERN_BRAKET_LEFT = &SINGLE_TOK(T_BRAKET_LEFT);

Token *INTERN_BRAKET_RIGHT = &SINGLE_TOK(T_BRAKET_RIGHT);

Token *INTERN_MINUS = &SINGLE_TOK(T_MINUS);

Token *INTERN_PLUS = &SINGLE_TOK(T_PLUS);

Token *INTERN_ASTERISKS = &SINGLE_TOK(T_ASTERISKS);

Token *INTERN_SLASH = &SINGLE_TOK(T_SLASH);

Token *INTERN_FALSE = &SINGLE_TOK(T_FALSE);

Token *INTERN_TRUE = &SINGLE_TOK(T_TRUE);

Token *INTERN_EQUAL = &SINGLE_TOK(T_EQUAL);

Token *INTERN_EOF = &SINGLE_TOK(T_EOF);

size_t Lexer_all(Lexer *l, Allocator *a, Token **out) {

#define rule(name) size_t name(Lexer *l, Allocator *a, Token **out)

rule(Lexer_all);

rule(delimitor_left);

rule(delimitor_right);

rule(braket_left);

rule(builtin);

rule(plus);

rule(minus);

rule(slash);

rule(equal);

rule(asterisks);

rule(number);

rule(ident);

rule(quoted);

rule(string);

rule(comment);

rule(whitespace);

rule(unknown);

rule(end);

typedef size_t (*rule_t)(Lexer *l, Allocator *a, Token **out);

static rule_t jump_table[256] = {

[0 ... 255] = &unknown,

[' '] = &whitespace,

['\t'] = &whitespace,

['\n'] = &whitespace,

[';'] = &comment,

['('] = &delimitor_left,

[')'] = &delimitor_right,

['@'] = &builtin,

['.'] = &number,

['0' ... '9'] = &number,

['a' ... 'z'] = &ident,

['A' ... 'Z'] = &ident,

['_'] = &ident,

['\''] = &quoted,

['"'] = &string,

['+'] = &plus,

['-'] = &minus,

['/'] = &slash,

['*'] = &asterisks,

['='] = &equal,

['['] = &braket_left,

[']'] = &braket_right,

[0] = &end,

};

#ifdef __clang__

#define musttail [[clang::musttail]]

#elif __GNUC__

#define musttail [[gnu::musttail]]

#else

#define musttail

#endif

#define JUMP_TARGET return musttail jump_table[(int32_t)l->input.p[l->pos]](l, a, out)

rule(Lexer_all) {

ASSERT(out != NULL, "Failed to allocate token list");

// empty input

if (l->input.len == 0) {

out[0] = INTERN_EOF;

return 1;

}

size_t true_hash = Str_hash(&STRING("true"));

size_t false_hash = Str_hash(&STRING("false"));

size_t count = 0;

static void *jump_table[256] = {

[0 ... 255] = &&unknown,

[' '] = &&whitespace,

['\t'] = &&whitespace,

['\n'] = &&whitespace,

[';'] = &&comment,

['('] = &&delimitor_left,

[')'] = &&delimitor_right,

['@'] = &&builtin,

['.'] = &&number,

['0' ... '9'] = &&number,

['a' ... 'z'] = &&ident,

['A' ... 'Z'] = &&ident,

['_'] = &&ident,

['\''] = &&quoted,

['"'] = &&string,

['+'] = &&plus,

['-'] = &&minus,

['/'] = &&slash,

['*'] = &&asterisks,

['='] = &&equal,

['['] = &&braket_left,

[']'] = &&braket_right,

[0] = &&end,

};

#define JUMP_TARGET goto *jump_table[(int32_t)l->input.p[l->pos]]

JUMP_TARGET;

}

delimitor_left:

rule(delimitor_left) {

out[count++] = INTERN_DELIMITOR_LEFT;

l->pos++;

JUMP_TARGET;

}

delimitor_right:

rule(delimitor_right) {

out[count++] = INTERN_DELIMITOR_RIGHT;

l->pos++;

JUMP_TARGET;

}

braket_left:

rule(braket_left) {

out[count++] = INTERN_BRAKET_LEFT;

l->pos++;

JUMP_TARGET;

}

braket_right:

rule(braket_right) {

out[count++] = INTERN_BRAKET_RIGHT;

l->pos++;

JUMP_TARGET;

}

builtin: {

rule(builtin) {

l->pos++;

// not an ident after @, this is shit

if (!is_alphanum(cur(l))) {

out[count++] = INTERN_EOF;

}

size_t start = l->pos;

size_t hash = FNV_OFFSET_BASIS;

for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {

hash ^= cc;

hash *= FNV_PRIME;

}

size_t len = l->pos - start;

Str s = (Str){

.p = l->input.p + start,

.len = len,

.hash = hash,

};

Token *b = CALL(a, request, sizeof(Token));

b->string = s;

b->type = T_BUILTIN;

out[count++] = b;

JUMP_TARGET;

}

plus:

rule(plus) {

out[count++] = INTERN_PLUS;

l->pos++;

JUMP_TARGET;

}

minus:

rule(minus) {

out[count++] = INTERN_MINUS;

l->pos++;

JUMP_TARGET;

}

slash:

rule(slash) {

out[count++] = INTERN_SLASH;

l->pos++;

JUMP_TARGET;

}

equal:

rule(equal) {

out[count++] = INTERN_EQUAL;

l->pos++;

JUMP_TARGET;

}

asterisks:

rule(asterisks) {

out[count++] = INTERN_ASTERISKS;

l->pos++;

JUMP_TARGET;

}

number: {

rule(number) {

size_t start = l->pos;

size_t i = start;

bool is_double = false;

size_t hash = FNV_OFFSET_BASIS;

for (; i < l->input.len; i++) {

char cc = l->input.p[i];

hash ^= cc;

hash *= FNV_PRIME;

if (cc >= '0' && cc <= '9')

continue;

if (cc == '.') {

ASSERT(!is_double, "Two dots in double");

is_double = true;

continue;

}

break;

}

l->pos = i;

Token *n = CALL(a, request, sizeof(Token));

n->string = (Str){

.p = l->input.p + start,

.len = i - start,

.hash = hash,

};

if (is_double) {

n->type = T_DOUBLE;

} else {

n->type = T_INTEGER;

}

out[count++] = n;

JUMP_TARGET;

}

ident: {

rule(ident) {

size_t start = l->pos;

size_t hash = FNV_OFFSET_BASIS;

for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {

hash ^= cc;

hash *= FNV_PRIME;

}

size_t len = l->pos - start;

Token *t;

if (hash == true_hash) {

t = INTERN_TRUE;

} else if (hash == false_hash) {

t = INTERN_FALSE;

} else {

t = CALL(a, request, sizeof(Token));

t->type = T_IDENT;

t->string = (Str){

.p = l->input.p + start,

.len = len,

.hash = hash,

};

}

out[count++] = t;

JUMP_TARGET;

}

// same as string but only with leading '

quoted: {

rule(quoted) {

// skip '

l->pos++;

size_t start = l->pos;

size_t hash = FNV_OFFSET_BASIS;

for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {

hash ^= cc;

hash *= FNV_PRIME;

}

size_t len = l->pos - start;

Token *t;

t = CALL(a, request, sizeof(Token));

t->type = T_STRING;

t->string = (Str){

.p = l->input.p + start,

.len = len,

.hash = hash,

};

out[count++] = t;

JUMP_TARGET;

}

string: {

rule(string) {

// skip "

l->pos++;

size_t start = l->pos;

size_t hash = FNV_OFFSET_BASIS;

for (char cc = cur(l); cc > 0 && cc != '"'; l->pos++, cc = cur(l)) {

hash ^= cc;

hash *= FNV_PRIME;

}

if (UNLIKELY(cur(l) != '"')) {

Str slice = Str_slice(&l->input, l->pos, l->input.len);

fprintf(stderr, "lex: Unterminated string near: '%.*s'", (int)slice.len,

slice.p);

out[count++] = INTERN_EOF;

} else {

Token *t = CALL(a, request, sizeof(Token));

t->type = T_STRING;

t->string = (Str){

.p = l->input.p + start,

.len = l->pos - start,

.hash = hash,

};

out[count++] = t;

// skip "

l->pos++;

}

JUMP_TARGET;

}

comment:

rule(comment) {

for (char cc = cur(l); cc > 0 && cc != '\n'; l->pos++, cc = cur(l)) {

}

JUMP_TARGET;

}

whitespace:

rule(whitespace) {

l->pos++;

JUMP_TARGET;

}

unknown: {

rule(unknown) {

uint8_t c = cur(l);

ASSERT(0, "Unexpected byte '%c' (0x%X) in input", c, c)

}

end:

rule(end) {

out[count++] = INTERN_EOF;

return count;

}

#undef SINGLE_TOK

Diff salvati

Testo originale

Apri file

#include "lexer.h"
#include "common.h"
#include "mem.h"
#include "strings.h"
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>

#define SINGLE_TOK(t) ((Token){.type = t})

Str TOKEN_TYPE_MAP[] = {[T_DELIMITOR_LEFT] = STRING("T_DELIMITOR_LEFT"),
                        [T_DELIMITOR_RIGHT] = STRING("T_DELIMITOR_RIGHT"),
                        [T_BRAKET_LEFT] = STRING("T_BRAKET_LEFT"),
                        [T_BRAKET_RIGHT] = STRING("T_BRAKET_RIGHT"),
                        [T_STRING] = STRING("T_STRING"),
                        [T_TRUE] = STRING("T_TRUE"),
                        [T_FALSE] = STRING("T_FALSE"),
                        [T_DOUBLE] = STRING("T_DOUBLE"),
                        [T_INTEGER] = STRING("T_INTEGER"),
                        [T_BUILTIN] = STRING("T_BUILTIN"),
                        [T_IDENT] = STRING("T_IDENT"),
                        [T_PLUS] = STRING("T_PLUS"),
                        [T_MINUS] = STRING("T_MINUS"),
                        [T_ASTERISKS] = STRING("T_ASTERISKS"),
                        [T_SLASH] = STRING("T_SLASH"),
                        [T_EQUAL] = STRING("T_EQUAL"),
                        [T_EOF] = STRING("T_EOF")};

Lexer Lexer_new(Str input) {
  return (Lexer){
      .input = input,
      .pos = 0,
  };
}

#define cur(L) (L->input.p[L->pos])

__attribute__((always_inline)) inline static bool is_alphanum(uint8_t cc) {
  uint8_t lower = cc | 0x20;
  bool is_alpha = (lower >= 'a' && lower <= 'z');
  bool is_digit = (cc >= '0' && cc <= '9');
  return is_alpha || is_digit || cc == '_' || cc == '-';
}

// we can "intern" these, since all of them are the same, regardless of position
Token *INTERN_DELIMITOR_LEFT = &SINGLE_TOK(T_DELIMITOR_LEFT);
Token *INTERN_DELIMITOR_RIGHT = &SINGLE_TOK(T_DELIMITOR_RIGHT);
Token *INTERN_BRAKET_LEFT = &SINGLE_TOK(T_BRAKET_LEFT);
Token *INTERN_BRAKET_RIGHT = &SINGLE_TOK(T_BRAKET_RIGHT);
Token *INTERN_MINUS = &SINGLE_TOK(T_MINUS);
Token *INTERN_PLUS = &SINGLE_TOK(T_PLUS);
Token *INTERN_ASTERISKS = &SINGLE_TOK(T_ASTERISKS);
Token *INTERN_SLASH = &SINGLE_TOK(T_SLASH);
Token *INTERN_FALSE = &SINGLE_TOK(T_FALSE);
Token *INTERN_TRUE = &SINGLE_TOK(T_TRUE);
Token *INTERN_EQUAL = &SINGLE_TOK(T_EQUAL);
Token *INTERN_EOF = &SINGLE_TOK(T_EOF);

size_t Lexer_all(Lexer *l, Allocator *a, Token **out) {
  ASSERT(out != NULL, "Failed to allocate token list");

// empty input
  if (l->input.len == 0) {
    out[0] = INTERN_EOF;
    return 1;
  }

size_t true_hash = Str_hash(&STRING("true"));
  size_t false_hash = Str_hash(&STRING("false"));

size_t count = 0;
  static void *jump_table[256] = {
      [0 ... 255] = &&unknown,
      [' '] = &&whitespace,
      ['\t'] = &&whitespace,
      ['\n'] = &&whitespace,
      [';'] = &&comment,
      ['('] = &&delimitor_left,
      [')'] = &&delimitor_right,
      ['@'] = &&builtin,
      ['.'] = &&number,
      ['0' ... '9'] = &&number,
      ['a' ... 'z'] = &&ident,
      ['A' ... 'Z'] = &&ident,
      ['_'] = &&ident,
      ['\''] = &&quoted,
      ['"'] = &&string,
      ['+'] = &&plus,
      ['-'] = &&minus,
      ['/'] = &&slash,
      ['*'] = &&asterisks,
      ['='] = &&equal,
      ['['] = &&braket_left,
      [']'] = &&braket_right,
      [0] = &&end,
  };

#define JUMP_TARGET goto *jump_table[(int32_t)l->input.p[l->pos]]

JUMP_TARGET;

delimitor_left:
  out[count++] = INTERN_DELIMITOR_LEFT;
  l->pos++;
  JUMP_TARGET;

delimitor_right:
  out[count++] = INTERN_DELIMITOR_RIGHT;
  l->pos++;
  JUMP_TARGET;

braket_left:
  out[count++] = INTERN_BRAKET_LEFT;
  l->pos++;
  JUMP_TARGET;

braket_right:
  out[count++] = INTERN_BRAKET_RIGHT;
  l->pos++;
  JUMP_TARGET;

builtin: {
  l->pos++;
  // not an ident after @, this is shit
  if (!is_alphanum(cur(l))) {
    out[count++] = INTERN_EOF;
  }
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

size_t len = l->pos - start;
  Str s = (Str){
      .p = l->input.p + start,
      .len = len,
      .hash = hash,
  };
  Token *b = CALL(a, request, sizeof(Token));
  b->string = s;
  b->type = T_BUILTIN;
  out[count++] = b;
  JUMP_TARGET;
}

plus:
  out[count++] = INTERN_PLUS;
  l->pos++;
  JUMP_TARGET;

minus:
  out[count++] = INTERN_MINUS;
  l->pos++;
  JUMP_TARGET;

slash:
  out[count++] = INTERN_SLASH;
  l->pos++;
  JUMP_TARGET;

equal:
  out[count++] = INTERN_EQUAL;
  l->pos++;
  JUMP_TARGET;

asterisks:
  out[count++] = INTERN_ASTERISKS;
  l->pos++;
  JUMP_TARGET;

number: {
  size_t start = l->pos;
  size_t i = start;
  bool is_double = false;
  size_t hash = FNV_OFFSET_BASIS;
  for (; i < l->input.len; i++) {
    char cc = l->input.p[i];
    hash ^= cc;
    hash *= FNV_PRIME;
    if (cc >= '0' && cc <= '9')
      continue;
    if (cc == '.') {
      ASSERT(!is_double, "Two dots in double");
      is_double = true;
      continue;
    }
    break;
  }

l->pos = i;
  Token *n = CALL(a, request, sizeof(Token));
  n->string = (Str){
      .p = l->input.p + start,
      .len = i - start,
      .hash = hash,
  };
  if (is_double) {
    n->type = T_DOUBLE;
  } else {
    n->type = T_INTEGER;
  }

out[count++] = n;
  JUMP_TARGET;
}

ident: {
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

size_t len = l->pos - start;
  Token *t;
  if (hash == true_hash) {
    t = INTERN_TRUE;
  } else if (hash == false_hash) {
    t = INTERN_FALSE;
  } else {
    t = CALL(a, request, sizeof(Token));
    t->type = T_IDENT;
    t->string = (Str){
        .p = l->input.p + start,
        .len = len,
        .hash = hash,
    };
  }
  out[count++] = t;
  JUMP_TARGET;
}

// same as string but only with leading '
quoted: {
  // skip '
  l->pos++;
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

size_t len = l->pos - start;
  Token *t;
  t = CALL(a, request, sizeof(Token));
  t->type = T_STRING;
  t->string = (Str){
      .p = l->input.p + start,
      .len = len,
      .hash = hash,
  };
  out[count++] = t;
  JUMP_TARGET;
}

string: {
  // skip "
  l->pos++;
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && cc != '"'; l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

if (UNLIKELY(cur(l) != '"')) {
    Str slice = Str_slice(&l->input, l->pos, l->input.len);
    fprintf(stderr, "lex: Unterminated string near: '%.*s'", (int)slice.len,
            slice.p);
    out[count++] = INTERN_EOF;
  } else {
    Token *t = CALL(a, request, sizeof(Token));
    t->type = T_STRING;
    t->string = (Str){
        .p = l->input.p + start,
        .len = l->pos - start,
        .hash = hash,
    };
    out[count++] = t;
    // skip "
    l->pos++;
  }
  JUMP_TARGET;
}

comment:
  for (char cc = cur(l); cc > 0 && cc != '\n'; l->pos++, cc = cur(l)) {
  }
  JUMP_TARGET;

whitespace:
  l->pos++;
  JUMP_TARGET;

unknown: {
  uint8_t c = cur(l);
  ASSERT(0, "Unexpected byte '%c' (0x%X) in input", c, c)
}

end:
  out[count++] = INTERN_EOF;
  return count;
}

#undef SINGLE_TOK

Testo modificato

Apri file

#include "lexer.h"
#include "common.h"
#include "mem.h"
#include "strings.h"
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>

#define SINGLE_TOK(t) ((Token){.type = t})

Lexer Lexer_new(Str input) {
  return (Lexer){
      .input = input,
      .pos = 0,
  };
}

#define cur(L) (L->input.p[L->pos])

#define rule(name) size_t name(Lexer *l, Allocator *a, Token **out)

rule(Lexer_all);
rule(delimitor_left);
rule(delimitor_right);
rule(braket_left);
rule(builtin);
rule(plus);
rule(minus);
rule(slash);
rule(equal);
rule(asterisks);
rule(number);
rule(ident);
rule(quoted);
rule(string);
rule(comment);
rule(whitespace);
rule(unknown);
rule(end);

typedef size_t (*rule_t)(Lexer *l, Allocator *a, Token **out);

static rule_t jump_table[256] = {
    [0 ... 255] = &unknown,
    [' '] = &whitespace,
    ['\t'] = &whitespace,
    ['\n'] = &whitespace,
    [';'] = &comment,
    ['('] = &delimitor_left,
    [')'] = &delimitor_right,
    ['@'] = &builtin,
    ['.'] = &number,
    ['0' ... '9'] = &number,
    ['a' ... 'z'] = &ident,
    ['A' ... 'Z'] = &ident,
    ['_'] = &ident,
    ['\''] = &quoted,
    ['"'] = &string,
    ['+'] = &plus,
    ['-'] = &minus,
    ['/'] = &slash,
    ['*'] = &asterisks,
    ['='] = &equal,
    ['['] = &braket_left,
    [']'] = &braket_right,
    [0] = &end,
};

#ifdef __clang__
#define musttail [[clang::musttail]]
#elif __GNUC__
#define musttail [[gnu::musttail]]
#else
#define musttail
#endif
#define JUMP_TARGET return musttail jump_table[(int32_t)l->input.p[l->pos]](l, a, out)

rule(Lexer_all) {
  ASSERT(out != NULL, "Failed to allocate token list");

// empty input
  if (l->input.len == 0) {
    out[0] = INTERN_EOF;
    return 1;
  }

size_t true_hash = Str_hash(&STRING("true"));
  size_t false_hash = Str_hash(&STRING("false"));

size_t count = 0;

JUMP_TARGET;
}

rule(delimitor_left) {
  out[count++] = INTERN_DELIMITOR_LEFT;
  l->pos++;
  JUMP_TARGET;
}

rule(delimitor_right) {
  out[count++] = INTERN_DELIMITOR_RIGHT;
  l->pos++;
  JUMP_TARGET;
}

rule(braket_left) {
  out[count++] = INTERN_BRAKET_LEFT;
  l->pos++;
  JUMP_TARGET;
}

rule(braket_right) {
  out[count++] = INTERN_BRAKET_RIGHT;
  l->pos++;
  JUMP_TARGET;
}

rule(builtin) {
  l->pos++;
  // not an ident after @, this is shit
  if (!is_alphanum(cur(l))) {
    out[count++] = INTERN_EOF;
  }
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

rule(plus) {
  out[count++] = INTERN_PLUS;
  l->pos++;
  JUMP_TARGET;
}

rule(minus) {
  out[count++] = INTERN_MINUS;
  l->pos++;
  JUMP_TARGET;
}

rule(slash) {
  out[count++] = INTERN_SLASH;
  l->pos++;
  JUMP_TARGET;
}

rule(equal) {
  out[count++] = INTERN_EQUAL;
  l->pos++;
  JUMP_TARGET;
}

rule(asterisks) {
  out[count++] = INTERN_ASTERISKS;
  l->pos++;
  JUMP_TARGET;
}

rule(number) {
  size_t start = l->pos;
  size_t i = start;
  bool is_double = false;
  size_t hash = FNV_OFFSET_BASIS;
  for (; i < l->input.len; i++) {
    char cc = l->input.p[i];
    hash ^= cc;
    hash *= FNV_PRIME;
    if (cc >= '0' && cc <= '9')
      continue;
    if (cc == '.') {
      ASSERT(!is_double, "Two dots in double");
      is_double = true;
      continue;
    }
    break;
  }

out[count++] = n;
  JUMP_TARGET;
}

rule(ident) {
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

// same as string but only with leading '
rule(quoted) {
  // skip '
  l->pos++;
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && is_alphanum(cc); l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

rule(string) {
  // skip "
  l->pos++;
  size_t start = l->pos;
  size_t hash = FNV_OFFSET_BASIS;
  for (char cc = cur(l); cc > 0 && cc != '"'; l->pos++, cc = cur(l)) {
    hash ^= cc;
    hash *= FNV_PRIME;
  }

rule(comment) {
  for (char cc = cur(l); cc > 0 && cc != '\n'; l->pos++, cc = cur(l)) {
  }
  JUMP_TARGET;
}

rule(whitespace) {
  l->pos++;
  JUMP_TARGET;
}

rule(unknown) {
  uint8_t c = cur(l);
  ASSERT(0, "Unexpected byte '%c' (0x%X) in input", c, c)
}

rule(end) {
  out[count++] = INTERN_EOF;
  return count;
}

#undef SINGLE_TOK