01 // a b c NULL (NULL terminated),ends at end of input or '.'
02 U8 *SkipWhitespace(U8 *s) {
03   while(Bt(char_bmp_white_space,*s))
04     s++;
05   return s;
06 }
07 U8 **TokenizeSentence(U8 *str,U8 **en=NULL) {
08   U8 *ostr=str;
09   U8 **ret;
10   I64 tcnt=0;
11 phase1:
12   str=SkipWhitespace(str);
13   switch(*str) {
14     case '[':
15     case ']':
16     case ',':
17       tcnt++;
18       str++;
19       break;
20     default:
21       if(Bt(char_bmp_alpha_numeric,*str)) {
22         tcnt++;
23         while(Bt(char_bmp_alpha_numeric,*str))
24           str++;
25       } else
26         str++; //???
27       break;
28     case '.':
29     case ':':
30     case ';':
31     case 0:
32       goto phase2_st;
33   }
34   goto phase1;
35 phase2_st:
36   ret=CAlloc(8*(tcnt+1));
37   tcnt=0;
38   str=ostr;
39 phase2:
40   str=SkipWhitespace(str);
41   switch(*str) {
42     start:
43       case '[':
44         ret[tcnt]=StrNew("[");
45         break;
46       case ']':
47         ret[tcnt]=StrNew("]");
48         break;
49       case ',':
50         ret[tcnt]=StrNew(",");
51         break;
52     end:
53       tcnt++;
54       str++;
55       break;
56     default:
57       if(Bt(char_bmp_alpha_numeric,*str)) {
58         ostr=str;
59         while(Bt(char_bmp_alpha_numeric,*str))
60           str++;
61         ret[tcnt]=CAlloc(str-ostr+1); //Include NULL terminator
62         MemCpy(ret[tcnt],ostr,str-ostr);
63         tcnt++;
64       } else
65         str++; //???
66       break;
67     case '.':
68     case ':': 
69     case ';':
70      str++; //Ignore '.'
71     case 0:
72      goto fin;
73   }
74   goto phase2;
75 fin:
76   if(en) *en=str;
77   ret[tcnt]=NULL;
78   return ret;
79 }