Changeset 452

Timestamp:

08/19/07 18:31:07 (14 months ago)

Author:

krobillard

Message:

Tokenizer handles special char!.
ur_toStrT() now thread-safe for char! values.
Fixed Vim char! syntax.

Location:

branches/thune/thread_safe

Files:

• charset.c (modified) (1 diff)
• doc/thune.vim (modified) (2 diffs)
• make.c (modified) (3 diffs)
• print.c (modified) (3 diffs)
• tokenize.c (modified) (5 diffs)

branches/thune/thread_safe/charset.c

@@ -37 +37 @@
 
 
-/* Word: 0-9 a-z A-Z ?!.'+-*&|=_~ and all ascii >= 127 */
+/* Word: 0-9 a-z A-Z ?!.+-*&|=_~ and all ascii >= 127 */
 uint8_t charset_word[32] = {
 #ifdef UR_CONFIG_MACROS
-        0x00,0x00,0x00,0x00,0xC2,0x6C,0xFF,0xA3,    // Don't allow < >
+        0x00,0x00,0x00,0x00,0x42,0x6C,0xFF,0xA3,    // Don't allow < >
 #else
-        0x00,0x00,0x00,0x00,0xC2,0x6C,0xFF,0xF3,
+        0x00,0x00,0x00,0x00,0x42,0x6C,0xFF,0xF3,
 #endif
         0xFE,0xFF,0xFF,0x97,0xFF,0xFF,0xFF,0x57,

branches/thune/thread_safe/doc/thune.vim

@@ -18 +18 @@
 syn case ignore
 
-" As per current users documentation
 if version < 600
-  set isk=@,48-57,?,!,.,',+,-,*,&,\|,=,_,~
+  set iskeyword=@,48-57,?,!,.,+,-,*,&,\|,=,_,~
 else
-  setlocal isk=@,48-57,?,!,.,',+,-,*,&,\|,=,_,~
+  setlocal iskeyword=@,48-57,?,!,.,+,-,*,&,\|,=,_,~
 endif
 
@@ -85 +84 @@
 " Tuples
 syn match       thuneTuple      "\(\d\+\.\)\{2,}"
-
 " Characters
-syn region  thuneChar   oneline start="'" end="'" contains=thuneSpecialChar
+syn match   thuneChar   "'\^\=.'"
+"syn match   thuneChar   "'\S\+'"
+
 syn match       thuneSpecialChar contained "\^[^[:space:][]"
 syn match       thuneSpecialChar contained "%\d\+"

branches/thune/thread_safe/make.c

@@ -538 +538 @@
   Returns -1 if failed.
 */
-static int _escapedCharacter( const char** cp, const char* end )
+int ur_escapedCharacter( const char** cp, const char* end )
 {
     int val;
@@ -649 +649 @@
         {
             ++cp;
-            *out++ = _escapedCharacter( &cp, end ); 
+            *out++ = ur_escapedCharacter( &cp, end ); 
         }
         else
@@ -1481 +1481 @@
                     {
                         ++spA;
-                        ur_char(res) = _escapedCharacter( (const char**) &spA,
-                                                          spB ); 
+                        ur_char(res) = ur_escapedCharacter(
+                                            (const char**) &spA, spB ); 
                     }
                     else

branches/thune/thread_safe/print.c

@@ -32 +32 @@
 
 extern int copyUtf16ToAscii( char* dest, const uint16_t* src, int len );
-
-
-static char charStr[] = "'?'";
 
 
@@ -1047 +1044 @@
             switch( ur_int(val) )
             {
-                case '\0':
-                    append( out, "'^0'", 4 );
-                    break;
-
                 case '\a':
                     append( out, "'^(bell)'", 9 );
@@ -1080 +1073 @@
 
                 default:
-                    charStr[1] = (char) ur_int(val);
-                    append(out, charStr, 3);
+                {
+                    char cstr[ 4 ];
+                    cstr[0] = '\'';
+                    if( ur_int(val) < 10 )
+                    {
+                        cstr[1] = '^';
+                        cstr[2] = ur_int(val) + '0';
+                        cstr[3] = '\'';
+                        append( out, cstr, 4 );
+                    }
+                    else
+                    {
+                        cstr[1] = (char) ur_int(val);
+                        cstr[2] = '\'';
+                        append(out, cstr, 3);
+                    }
+                }
                     break;
             }

branches/thune/thread_safe/tokenize.c

@@ -31 +31 @@
 
 extern void ur_bindDefault( UThread*, UIndex blkN );
+extern int  ur_escapedCharacter( const char** cp, const char* end );
 
 
@@ -546 +547 @@
 quote:
 
-    if( ((end - it) > 1) && it[1] == '\'' )
-    {
-        cell = ur_appendCell( BLOCK, UT_CHAR );
-        ur_int(cell) = token[1];
-        it += 2;
-        goto set_eol;
-    }
-
     mode = UT_LITWORD;
 
     SCAN_LOOP
-        if( ch == '/' )
+        if( ch == '\'' )
+        {
+            mode = UT_CHAR;
+            break;
+        }
+        else if( ch == '/' )
         {
             mode = UT_LITSELECT;
@@ -563 +561 @@
         else if( ur_bitIsSet( charset_delimiters, ch ) )   // '[' 'word[]
         {
+            if( it == (token + 1) )
+            {
+                ++it;
+                if( (it != end) && (*it == '\'') )
+                    goto make_char;
+                goto quote_err;
+            }
             break;
         }
@@ -572 +577 @@
         if( it == token )
             goto quote_err;
-        if( mode == UT_LITWORD )
+
+        if( mode == UT_CHAR )
+        {
+            ch = *token;
+            if( ch == '^' )
+            {
+                ++token;
+                ch = ur_escapedCharacter( &token, it );
+                if( ch < 0 )
+                {
+                    syntaxError( "Invalid char" );
+                }
+            }
+make_char:
+            ++it;
+            cell = ur_appendCell( BLOCK, UT_CHAR );
+            ur_int(cell) = ch;
+        }
+        else if( mode == UT_LITWORD )
         {
             cell = ur_appendWord( BLOCK, UT_LITWORD, token, it - token );
@@ -632 +655 @@
         {
             mode = WORD_COLON;
-            break;
-        }
-        if( ch == '\'' )
-        {
-            mode = UT_CHAR;
             break;
         }