Changeset 453

Timestamp:

08/25/07 22:49:09 (13 months ago)

Author:

krobillard

Message:

UTF-8 strings can now be read.
ur_makeString() now takes UCell argument.

Location:

branches/thune/thread_safe

Files:

• config.t (modified) (1 diff)
• doc/UserManual (modified) (3 diffs)
• encoding.c (modified) (5 diffs)
• gl/gx.c (modified) (1 diff)
• gl/joystick.c (modified) (1 diff)
• make.c (modified) (9 diffs)
• print.c (modified) (4 diffs)
• series.c (modified) (1 diff)
• stdio.c (modified) (2 diffs)
• tests/working/utf8.good (added)
• tests/working/utf8.t (added)
• tokenize.c (modified) (2 diffs)
• unix/os.c (modified) (2 diffs)
• urlan.c (modified) (2 diffs)
• urlan.h (modified) (2 diffs)
• win32/os.c (modified) (2 diffs)

branches/thune/thread_safe/config.t

@@ -12 +12 @@
 [ ] emh      "Debugger Hooks"
 [ ] dt-code  "Include 'code datatype"
-[ ] uds      "Library with datatype & gc system only - no eval"
-             disable [bzip2 trig math3d dt-code]
+
+;[ ] uds      "Library with datatype & gc system only - no eval"
+;             disable [bzip2 trig math3d dt-code]
 
 ;eof

branches/thune/thread_safe/doc/UserManual

@@ -19 +19 @@
    * Garbage collected datatype system with prototype based objects.
    * Written in C to work well as an embedded scripting language.
-   * Small (but not tiny) binary & run-time enviroment.
-
+   * Small (but not tiny) binary & run-time environment.
 
 .. Evaluation
    ----------
    Thune uses postfix notation
+
+
+About This Document
+-------------------
+
+This manual is largely incomplete.  It can only serve as a quick function
+reference at this point.
+
 
 
@@ -278 +285 @@
 
 *Connect* also accepts components as arguments.  This is short-hand for
-refering to the first input or output.
+referring to the first input or output.
 The following call makes the same connection as the example above::
 
@@ -466 +473 @@
 find.last   (ser pat -- ser)        Find pattern in series starting from end.
 match_      (ser pat -- end)        Move position to end of matching pattern.
-trim        (str -- str)            Remove whitespace from start and end.
+trim        (str -- str)            Remove white space from start and end.
 ==========  ======================  =================
 

branches/thune/thread_safe/encoding.c

@@ -1 +1 @@
 /*============================================================================
     Urlan Interpreter
-    Copyright (C) 2005-2006  Karl Robillard
+    Copyright (C) 2005-2007  Karl Robillard
 
     This library is free software; you can redistribute it and/or
@@ -31 +31 @@
    Returns number of characters copied.
 */
-int copyUtf16ToAscii( char* dest, const uint16_t* src, int len )
+int copyUcs2ToUtf8( char* dest, const uint16_t* src, int len )
+{
+    // TODO: Prevent overflow of dest.
+    const char* start;
+    const uint16_t* end;
+    uint16_t c;
+
+    start = dest;
+    end = src + len;
+
+    while( src != end )
+    {
+        c = *src++;
+        if( c > 127 )
+        {
+            if( c > 0x07ff )
+            {
+                *dest++ = 0xE0 | (c >> 12);
+                *dest++ = 0x80 | ((c >> 6) & 0x3f);
+                c = 0x80 | (c & 0x3f);
+            }
+            else
+            {
+                *dest++ = 0xC0 | (c >> 6);
+                c = 0x80 | (c & 0x3f);
+            }
+        }
+        *dest++ = (char) c;
+    }
+    return dest - start;
+}
+
+
+#if 0
+int copyUcs2ToAscii( char* dest, const uint16_t* src, int len )
 {
     const uint16_t* end;
@@ -47 +81 @@
     return len;
 }
+#endif
 
 
-#ifndef UR_CONFIG_UDS
 /*
    Returns number of characters copied.
@@ -87 +121 @@
                     case UR_ENC_UTF16:
                         count >>= 1;
-                        strN = ur_makeBinary( count, &bin ); 
-                        bin->used = copyUtf16ToAscii( bin->ptr.c,
-                                                      (uint16_t*) cpA, count );
+                        strN = ur_makeBinary( count * 2, &bin ); 
+                        bin->used = copyUcs2ToUtf8( bin->ptr.c,
+                                                    (uint16_t*) cpA, count );
                         enc = UR_ENC_ASCII;
                         goto set_result;
@@ -156 +190 @@
     }
 }
-#endif
 
 

branches/thune/thread_safe/gl/gx.c

@@ -1345 +1345 @@
 {
     const GLubyte* str;
-    UIndex binN;
 
     UR_CALL_UNUSED_TH
+    UR_CALL_UNUSED_TOS
 
     str = glGetString( GL_EXTENSIONS );
-    binN = ur_makeString( (const char*) str, -1 );
-
-
+    ur_makeString( ur_s_next(UR_TOS), (const char*) str, -1 );
     UR_S_GROW;
-    tos = UR_TOS;
-    ur_initType(tos, UT_STRING);
-    ur_setSeries(tos, binN, 0 );
 }
 

branches/thune/thread_safe/gl/joystick.c

@@ -83 +83 @@
 
             ++val;                          // JV_NAME
-            n = ur_makeString( name, -1 );
-            ur_initString( val, n, 0 );
+            ur_makeString( val, name, -1 );
 
             ++val;                          // JV_AXIS_COUNT

branches/thune/thread_safe/make.c

@@ -1 +1 @@
 /*============================================================================
     Thune Interpreter
-    Copyright (C) 2005-2006  Karl Robillard
+    Copyright (C) 2005-2007  Karl Robillard
 
     This library is free software; you can redistribute it and/or
@@ -335 +335 @@
 
 
-#ifndef UR_CONFIG_UDS
 /*
    Create context from initializer block.
@@ -458 +457 @@
     }
 }
-#endif
 
 
@@ -614 +612 @@
 
 
+/*
+   Convert UTF-8 to UCS-2
+*/
+static void _makeString16( UBinary* str, const uint8_t* cp, int len )
+{
+    uint16_t  ch;
+    uint16_t* out;
+    const uint8_t* end;
+
+    ur_arrayReserve( str, 1, len * sizeof(uint16_t) );
+    out = str->ptr.u16;
+
+    end = cp + len;
+    while( cp != end )
+    {
+        ch = *cp++;
+
+        if( ch <= 0x7f )
+        {
+            *out++ = ch;
+        }
+        else if( ch >= 0xc2 && ch <= 0xdf )
+        {
+            if( cp != end )
+            {
+                *out++ = ((ch & 0x1f) << 6) | (*cp & 0x3f);
+                ++cp;
+            }
+        }
+        else if( ch >= 0xe0 && ch <= 0xef )
+        {
+            if( (end - cp) < 2 )
+                break;
+            *out++ = ((ch    & 0x0f) << 12) |
+                     ((cp[0] & 0x3f) <<  6) |
+                      (cp[1] & 0x3f);
+            cp += 2;
+        }
+        else if( ch >= 0xf0 && ch <= 0xf3 )
+        {
+            if( (end - cp) < 3 )
+                break;
+            *out++ = 0;     // Only handle UCS-2
+            cp += 3;
+        }
+    }
+
+    str->avail /= 2;
+    str->used = out - str->ptr.u16;
+}
+
+
 /**
   If len is less than zero then the length is automatically determined.
   Special characters are translated.
 */
-UIndex ur_makeStringT( UThread* ut, const char* txt, int len )
+UIndex ur_makeStringT( UThread* ut, UCell* res, const char* txt, int len )
 {
     UIndex strN;
@@ -625 +675 @@
     const char* end;
     char* out;
+    int ch;
 
     if( len == 0 )
     {
-        return ur_makeBinary( 0, 0 );
+        strN = ur_makeBinary( 0, 0 );
+        goto init;
     }
 
@@ -653 +705 @@
         else
         {
-            *out++ = *cp++;
+            ch = *cp++;
+            if( ((unsigned int) ch) > 0x7f )
+            {
+                _makeString16( str, (const uint8_t*) txt, len );
+                ur_initString( res, strN, 0 );
+                ur_setEncoding( res, UR_ENC_UTF16 );
+                return strN;
+            }
+            *out++ = ch;
         }
     }
 
     str->used = out - str->ptr.c;
-
     str->ptr.c[ str->used ] = '\0';
+
+init:
+
+    ur_initString( res, strN, 0 );
     return strN;
 }
@@ -1326 +1389 @@
 
 
-#ifndef UR_CONFIG_UDS
 /*
   (datatype! proto -- value)
@@ -1720 +1782 @@
             {
                 if( spA )
-                    binN = ur_makeString( spA, spB - spA );
+                {
+                    binN = ur_makeString( res, spA, spB - spA );
+                }
                 else
+                {
                     binN = ur_makeBinary( 0, 0 );
 init_str:
-                ur_initString( res, binN, 0 );
+                    ur_initString( res, binN, 0 );
+                }
             }
         }
@@ -1871 +1937 @@
     ur_throwErr( UR_ERR_DATATYPE, "Invalid make values" );
 }
-#endif
 
 

branches/thune/thread_safe/print.c

@@ -31 +31 @@
 
 
-extern int copyUtf16ToAscii( char* dest, const uint16_t* src, int len );
+extern int copyUcs2ToUtf8( char* dest, const uint16_t* src, int len );
 
 
@@ -43 +43 @@
 
 
-void ur_strCatUtf16( UString* str, const uint16_t* cp, int len )
-{
-    if( (str->used + len) > str->avail )
-        EXPAND( str, len );
-    str->used += copyUtf16ToAscii( str->ptr.c + str->used, cp, len );
+void ur_strCatUcs2( UString* str, const uint16_t* cp, int len )
+{
+    EXPAND( str, len * 2 );
+    str->used += copyUcs2ToUtf8( str->ptr.c + str->used, cp, len );
 }
 
@@ -78 +77 @@
 
                 case UR_ENC_UTF16:
-                    ur_strCatUtf16( out, str2->ptr.u16 + si, used );
+                    ur_strCatUcs2( out, str2->ptr.u16 + si, used );
                     break;
             }
@@ -271 +270 @@
 #define APP_STR_FUNC    _appendStringUtf16
 #define APP_STR_T       uint16_t
-#define APP_STR_COPY    ur_strCatUtf16
+#define APP_STR_COPY    ur_strCatUcs2
 #include "print_string.c"
 

branches/thune/thread_safe/series.c

@@ -1 +1 @@
 /*============================================================================
     Thune Interpreter
-    Copyright (C) 2005-2006  Karl Robillard
+    Copyright (C) 2005-2007  Karl Robillard
 
     This library is free software; you can redistribute it and/or

branches/thune/thread_safe/stdio.c

@@ -1 +1 @@
 /*============================================================================
     Urlan Interpreter
-    Copyright (C) 2005-2006  Karl Robillard
+    Copyright (C) 2005-2007  Karl Robillard
 
     This library is free software; you can redistribute it and/or
@@ -145 +145 @@
         if( cp )
         {
-            UIndex strN = ur_makeString( cp, -1 );
-
-            ur_initType( tos, UT_STRING );
-            ur_setSeries( tos, strN, 0 );
+            ur_makeString( tos, cp, -1 );
             return;
         }

branches/thune/thread_safe/tokenize.c

@@ -492 +492 @@
 #endif
                     break;
+
+                default:
+                    syntaxError( "Unprintable/Non-ASCII Input" );
                 }
             }
@@ -1050 +1053 @@
 string_end:
 
-    // Make string before appending cell (in case GC is called).
+    // Mark cell as unset in case GC is called by ur_makeString.
+    cell = ur_appendCell( BLOCK, UT_UNSET );
+
     ++token;
-    tn = ur_makeString( token, it - token );
-
-    cell = ur_appendCell( BLOCK, UT_STRING );
-    ur_setSeries( cell, tn, 0 );
+    ur_makeString( cell, token, it - token );
 
     ++it;

branches/thune/thread_safe/unix/os.c

@@ -1 +1 @@
 /*============================================================================
     Urlan Interpreter
-    Copyright (C) 2005-2006  Karl Robillard
+    Copyright (C) 2005-2007  Karl Robillard
 
     This library is free software; you can redistribute it and/or
@@ -258 +258 @@
                 continue;
 
-            cell = ur_appendCell( blk, UT_STRING /*UT_FILE*/ );
-            ur_setSeries( cell, ur_makeString(cp, -1), 0 );
+            // Mark cell as unset in case GC is called by ur_makeString.
+            cell = ur_appendCell( blk, UT_UNSET );
+            ur_makeString( cell, cp, -1 );
         }
 

branches/thune/thread_safe/urlan.c

@@ -432 +432 @@
 //extern ULanguage _parseLanguage;
 extern void ur_rebuildAtomHash( UArray* );
-#ifndef UR_CONFIG_UDS
 extern void ur_makeCallsEval( UThread* );
 extern void ur_makeCallsSeries( UThread* );
@@ -440 +439 @@
 #ifdef UR_CONFIG_NET
 extern UPortDevice _netPortDevice;
-#endif
 #endif
 #ifdef LANG_RUNE

branches/thune/thread_safe/urlan.h

@@ -551 +551 @@
 UIndex  ur_makeBinaryT( UThread*, int size, UBinary** ptr );
 UIndex  ur_makeBinaryFrom( UThread*, const UCell* );
-UIndex  ur_makeStringT( UThread*, const char* txt, int len );
+UIndex  ur_makeStringT( UThread*, UCell*, const char* txt, int len );
 UIndex  ur_makeVectorT( UThread*, int size, UArray** ptr );
 UIndex  ur_makeResourceT( UThread*, int dataType, int size, UResource** );
@@ -617 +617 @@
 #define ur_makeBlock(size)              ur_makeBlockT(ut,size,0)    
 #define ur_makeBinary(size,ptr)         ur_makeBinaryT(ut,size,ptr)
-#define ur_makeString(txt,len)          ur_makeStringT(ut,txt,len)
+#define ur_makeString(cell,txt,len)     ur_makeStringT(ut,cell,txt,len)
 #define ur_makeVector(size,ptr)         ur_makeVectorT(ut,size,ptr)
 #define ur_makeResource(dt,size,rp)     ur_makeResourceT(ut,dt,size,rp)

branches/thune/thread_safe/win32/os.c

@@ -1 +1 @@
 /*============================================================================
     Thune Interpreter
-    Copyright (C) 2005-2006  Karl Robillard
+    Copyright (C) 2005-2007  Karl Robillard
 
     This library is free software; you can redistribute it and/or
@@ -326 +326 @@
                 continue;
 
-            cell = ur_appendCell( blk, UT_STRING /*UT_FILE*/ );
-            ur_setSeries( cell, ur_makeString(fileinfo.name, -1), 0 );
+            // Mark cell as unset in case GC is called by ur_makeString.
+            cell = ur_appendCell( blk, UT_UNSET );
+            ur_makeString( cell, fileinfo.name, -1 );
         }
         while( _findnext( handle, &fileinfo ) != -1 );