tokens.h

static char *tokens[] = {"<unk>","<s>","</s>","<0x00>","<0x01>","<0x02>","<0x03>","<0x04>","<0x05>","<0x06>","<0x07>","<0x08>","<0x09>","<0x0A>","<0x0B>","<0x0C>","<0x0D>","<0x0E>","<0x0F>","<0x10>","<0x11>","<0x12>","<0x13>","<0x14>","<0x15>","<0x16>","<0x17>","<0x18>","<0x19>","<0x1A>","<0x1B>","<0x1C>","<0x1D>","<0x1E>","<0x1F>","<0x20>","<0x21>","<0x22>","<0x23>","<0x24>","<0x25>","<0x26>","<0x27>","<0x28>","<0x29>","<0x2A>","<0x2B>","<0x2C>","<0x2D>","<0x2E>","<0x2F>","<0x30>","<0x31>","<0x32>","<0x33>","<0x34>","<0x35>","<0x36>","<0x37>","<0x38>","<0x39>","<0x3A>","<0x3B>","<0x3C>","<0x3D>","<0x3E>","<0x3F>","<0x40>","<0x41>","<0x42>","<0x43>","<0x44>","<0x45>","<0x46>","<0x47>","<0x48>","<0x49>","<0x4A>","<0x4B>","<0x4C>","<0x4D>","<0x4E>","<0x4F>","<0x50>","<0x51>","<0x52>","<0x53>","<0x54>","<0x55>","<0x56>","<0x57>","<0x58>","<0x59>","<0x5A>","<0x5B>","<0x5C>","<0x5D>","<0x5E>","<0x5F>","<0x60>","<0x61>","<0x62>","<0x63>","<0x64>","<0x65>","<0x66>","<0x67>","<0x68>","<0x69>","<0x6A>","<0x6B>","<0x6C>","<0x6D>","<0x6E>","<0x6F>","<0x70>","<0x71>","<0x72>","<0x73>","<0x74>","<0x75>","<0x76>","<0x77>","<0x78>","<0x79>","<0x7A>","<0x7B>","<0x7C>","<0x7D>","<0x7E>","<0x7F>","<0x80>","<0x81>","<0x82>","<0x83>","<0x84>","<0x85>","<0x86>","<0x87>","<0x88>","<0x89>","<0x8A>","<0x8B>","<0x8C>","<0x8D>","<0x8E>","<0x8F>","<0x90>","<0x91>","<0x92>","<0x93>","<0x94>","<0x95>","<0x96>","<0x97>","<0x98>","<0x99>","<0x9A>","<0x9B>","<0x9C>","<0x9D>","<0x9E>","<0x9F>","<0xA0>","<0xA1>","<0xA2>","<0xA3>","<0xA4>","<0xA5>","<0xA6>","<0xA7>","<0xA8>","<0xA9>","<0xAA>","<0xAB>","<0xAC>","<0xAD>","<0xAE>","<0xAF>","<0xB0>","<0xB1>","<0xB2>","<0xB3>","<0xB4>","<0xB5>","<0xB6>","<0xB7>","<0xB8>","<0xB9>","<0xBA>","<0xBB>","<0xBC>","<0xBD>","<0xBE>","<0xBF>","<0xC0>","<0xC1>","<0xC2>","<0xC3>","<0xC4>","<0xC5>","<0xC6>","<0xC7>","<0xC8>","<0xC9>","<0xCA>","<0xCB>","<0xCC>","<0xCD>","<0xCE>","<0xCF>","<0xD0>","<0xD1>","<0xD2>","<0xD3>","<0xD4>","<0xD5>","<0xD6>","<0xD7>","<0xD8>","<0xD9>","<0xDA>","<0xDB>","<0xDC>","<0xDD>","<0xDE>","<0xDF>","<0xE0>","<0xE1>","<0xE2>","<0xE3>","<0xE4>","<0xE5>","<0xE6>","<0xE7>","<0xE8>","<0xE9>","<0xEA>","<0xEB>","<0xEC>","<0xED>","<0xEE>","<0xEF>","<0xF0>","<0xF1>","<0xF2>","<0xF3>","<0xF4>","<0xF5>","<0xF6>","<0xF7>","<0xF8>","<0xF9>","<0xFA>","<0xFB>","<0xFC>","<0xFD>","<0xFE>","<0xFF>"," t","he"," a"," s"," w","nd"," the","ed"," to"," b"," and"," h","in"," f"," wa"," T","it","re","ou"," l"," d"," c"," he"," p","ay"," m","er"," was","om","im","on","il"," The","id","is","at","ar"," sa"," n"," g","ing"," ha"," S","en","an","or","le","ll"," L"," th","ot","ily"," her"," it"," \"","am","ir","et"," Lily"," u"," O"," H"," On"," in","ut"," pl","ri"," Tim","ow"," day"," be","ver","ce","ith","ig"," o"," with"," said"," play"," She","pp","ck","ld"," They","my"," e"," his"," He","oo"," y"," st"," up"," that"," r"," on","ke","ked","st"," mom"," she"," I","ve","nt","itt","very"," you"," happ"," they","end"," B","ime"," big"," fri","se"," of"," friend","ittle"," little","ent"," time","un","ad"," had"," we"," there"," so"," One","her"," for","all","ould"," nam"," want"," M"," happy"," saw"," named","ved"," li"," but"," very"," do"," lo","ch"," Once"," ne"," Timmy","es"," upon","out"," k"," ","e","a","t","o","h","n","i","d","s","r","l","y","m","w","u",".","p","g","c","b","f",",","k","T","v","\"","S","L","'","H","O","I","!","x","B","M","A","W","j","?","z","Y","F","J","D","q","C","N","E","P","R","K","G","-","","",":","","Z","V","U","3","Q",";","1","","0","X","2","5","","","9","4","","","8",")","(","6","7","/","","$","`","+","*","","&","\\","%","","","<",">","|","","[","]","~",""};
static float tokenscore[] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-0,-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,-11,-12,-13,-14,-15,-16,-17,-18,-19,-20,-21,-22,-23,-24,-25,-26,-27,-28,-29,-30,-31,-32,-33,-34,-35,-36,-37,-38,-39,-40,-41,-42,-43,-44,-45,-46,-47,-48,-49,-50,-51,-52,-53,-54,-55,-56,-57,-58,-59,-60,-61,-62,-63,-64,-65,-66,-67,-68,-69,-70,-71,-72,-73,-74,-75,-76,-77,-78,-79,-80,-81,-82,-83,-84,-85,-86,-87,-88,-89,-90,-91,-92,-93,-94,-95,-96,-97,-98,-99,-100,-101,-102,-103,-104,-105,-106,-107,-108,-109,-110,-111,-112,-113,-114,-115,-116,-117,-118,-119,-120,-121,-122,-123,-124,-125,-126,-127,-128,-129,-130,-131,-132,-133,-134,-135,-136,-137,-138,-139,-140,-141,-142,-143,-144,-145,-146,-147,-148,-149,-150,-151,-152,-153,-154,-155,-156,-157,-158,-159,-160,-161,-162,-163,-164,-165,-166,-167,-168,-169,-170,-171,-172,-173,-174,-175,-176,-177,-178,-179,-180,-181,-182,-183,-184,-185,-186,-187,-188,-189,-190,-191,-192,-193,-194,-195,-196,-197,-198,-199,-200,-201,-202,-203,-204,-205,-206,-207,-208,-209,-210,-211,-212,-213,-214,-215,-216,-217,-218,-219,-220,-221,-222,-223,-224,-225,-226,-227,-228,-229,-230,-231,-232,-233,-234,-235,-236,-237,-238,-239,-240,-241,-242,-243,-244,-245,-246,-247,-248,-249,-250,-251,-252};