Changeset 452 for branches/thune/thread_safe/tokenize.c

Timestamp:

08/19/07 18:31:07 (15 months ago)

Author:

krobillard

Message:

Tokenizer handles special char!.
ur_toStrT() now thread-safe for char! values.
Fixed Vim char! syntax.

Files:

• branches/thune/thread_safe/tokenize.c (modified) (5 diffs)

branches/thune/thread_safe/tokenize.c

@@ -31 +31 @@
 
 extern void ur_bindDefault( UThread*, UIndex blkN );
+extern int  ur_escapedCharacter( const char** cp, const char* end );
 
 
@@ -546 +547 @@
 quote:
 
-    if( ((end - it) > 1) && it[1] == '\'' )
-    {
-        cell = ur_appendCell( BLOCK, UT_CHAR );
-        ur_int(cell) = token[1];
-        it += 2;
-        goto set_eol;
-    }
-
     mode = UT_LITWORD;
 
     SCAN_LOOP
-        if( ch == '/' )
+        if( ch == '\'' )
+        {
+            mode = UT_CHAR;
+            break;
+        }
+        else if( ch == '/' )
         {
             mode = UT_LITSELECT;
@@ -563 +561 @@
         else if( ur_bitIsSet( charset_delimiters, ch ) )   // '[' 'word[]
         {
+            if( it == (token + 1) )
+            {
+                ++it;
+                if( (it != end) && (*it == '\'') )
+                    goto make_char;
+                goto quote_err;
+            }
             break;
         }
@@ -572 +577 @@
         if( it == token )
             goto quote_err;
-        if( mode == UT_LITWORD )
+
+        if( mode == UT_CHAR )
+        {
+            ch = *token;
+            if( ch == '^' )
+            {
+                ++token;
+                ch = ur_escapedCharacter( &token, it );
+                if( ch < 0 )
+                {
+                    syntaxError( "Invalid char" );
+                }
+            }
+make_char:
+            ++it;
+            cell = ur_appendCell( BLOCK, UT_CHAR );
+            ur_int(cell) = ch;
+        }
+        else if( mode == UT_LITWORD )
         {
             cell = ur_appendWord( BLOCK, UT_LITWORD, token, it - token );
@@ -632 +655 @@
         {
             mode = WORD_COLON;
-            break;
-        }
-        if( ch == '\'' )
-        {
-            mode = UT_CHAR;
             break;
         }