Fossil SCM

Add the ability to display content and diffs for UTF16 text files in the web interface.

drh 2014-01-14 12:44 trunk
Commit 5a66b6e7854f985a64324f74ce3a2933a2e4aa39
Parent e0f22dda7b25cd9…
5 files changed +10 -7 +4 +2 +1 -1 +11 -13
~ src/blob.c ~ src/db.c ~ src/diff.c ~ src/info.c ~ src/utf8.c
M src/blob.c
+10 -7
--- src/blob.c
+++ src/blob.c
@@ -223,10 +223,19 @@
223223
 ** Any prior data in the blob is discarded.
224224
 */
225225
 void blob_set(Blob *pBlob, const char *zStr){
226226
 blob_init(pBlob, zStr, -1);
227227
 }
228
+
229
+/*
230
+** Initialize a blob to a nul-terminated string obtained from fossil_malloc().
231
+** The blob will take responsibility for freeing the string.
232
+*/
233
+void blob_set_dynamic(Blob *pBlob, char *zStr){
234
+ blob_init(pBlob, zStr, -1);
235
+ pBlob->xRealloc = blobReallocMalloc;
236
+}
228237
229238
 /*
230239
 ** Initialize a blob to an empty string.
231240
 */
232241
 void blob_zero(Blob *pBlob){
@@ -1095,21 +1104,18 @@
10951104
 ** to be UTF-8 already, so no conversion is done.
10961105
 */
10971106
 void blob_to_utf8_no_bom(Blob *pBlob, int useMbcs){
10981107
 char *zUtf8;
10991108
 int bomSize = 0;
1100
-#if defined(_WIN32) || defined(__CYGWIN__)
11011109
 int bomReverse = 0;
1102
-#endif
11031110
 if( starts_with_utf8_bom(pBlob, &bomSize) ){
11041111
 struct Blob temp;
11051112
 zUtf8 = blob_str(pBlob) + bomSize;
11061113
 blob_zero(&temp);
11071114
 blob_append(&temp, zUtf8, -1);
11081115
 blob_swap(pBlob, &temp);
11091116
 blob_reset(&temp);
1110
-#if defined(_WIN32) || defined(__CYGWIN__)
11111117
 }else if( starts_with_utf16_bom(pBlob, &bomSize, &bomReverse) ){
11121118
 zUtf8 = blob_buffer(pBlob);
11131119
 if( bomReverse ){
11141120
 /* Found BOM, but with reversed bytes */
11151121
 unsigned int i = blob_size(pBlob);
@@ -1122,18 +1128,15 @@
11221128
 }
11231129
 /* Make sure the blob contains two terminating 0-bytes */
11241130
 blob_append(pBlob, "", 1);
11251131
 zUtf8 = blob_str(pBlob) + bomSize;
11261132
 zUtf8 = fossil_unicode_to_utf8(zUtf8);
1127
- blob_zero(pBlob);
1128
- blob_append(pBlob, zUtf8, -1);
1129
- fossil_unicode_free(zUtf8);
1130
-#endif /* _WIN32 || __CYGWIN__ */
1133
+ blob_set_dynamic(pBlob, zUtf8);
11311134
 #if defined(_WIN32)
11321135
 }else if( useMbcs ){
11331136
 zUtf8 = fossil_mbcs_to_utf8(blob_str(pBlob));
11341137
 blob_reset(pBlob);
11351138
 blob_append(pBlob, zUtf8, -1);
11361139
 fossil_mbcs_free(zUtf8);
11371140
 #endif /* _WIN32 */
11381141
 }
11391142
 }
11401143
--- src/blob.c
+++ src/blob.c
@@ -223,10 +223,19 @@
223 ** Any prior data in the blob is discarded.
224 */
225 void blob_set(Blob *pBlob, const char *zStr){
226 blob_init(pBlob, zStr, -1);
227 }
 
 
 
 
 
 
 
 
 
228
229 /*
230 ** Initialize a blob to an empty string.
231 */
232 void blob_zero(Blob *pBlob){
@@ -1095,21 +1104,18 @@
1095 ** to be UTF-8 already, so no conversion is done.
1096 */
1097 void blob_to_utf8_no_bom(Blob *pBlob, int useMbcs){
1098 char *zUtf8;
1099 int bomSize = 0;
1100 #if defined(_WIN32) || defined(__CYGWIN__)
1101 int bomReverse = 0;
1102 #endif
1103 if( starts_with_utf8_bom(pBlob, &bomSize) ){
1104 struct Blob temp;
1105 zUtf8 = blob_str(pBlob) + bomSize;
1106 blob_zero(&temp);
1107 blob_append(&temp, zUtf8, -1);
1108 blob_swap(pBlob, &temp);
1109 blob_reset(&temp);
1110 #if defined(_WIN32) || defined(__CYGWIN__)
1111 }else if( starts_with_utf16_bom(pBlob, &bomSize, &bomReverse) ){
1112 zUtf8 = blob_buffer(pBlob);
1113 if( bomReverse ){
1114 /* Found BOM, but with reversed bytes */
1115 unsigned int i = blob_size(pBlob);
@@ -1122,18 +1128,15 @@
1122 }
1123 /* Make sure the blob contains two terminating 0-bytes */
1124 blob_append(pBlob, "", 1);
1125 zUtf8 = blob_str(pBlob) + bomSize;
1126 zUtf8 = fossil_unicode_to_utf8(zUtf8);
1127 blob_zero(pBlob);
1128 blob_append(pBlob, zUtf8, -1);
1129 fossil_unicode_free(zUtf8);
1130 #endif /* _WIN32 || __CYGWIN__ */
1131 #if defined(_WIN32)
1132 }else if( useMbcs ){
1133 zUtf8 = fossil_mbcs_to_utf8(blob_str(pBlob));
1134 blob_reset(pBlob);
1135 blob_append(pBlob, zUtf8, -1);
1136 fossil_mbcs_free(zUtf8);
1137 #endif /* _WIN32 */
1138 }
1139 }
1140
--- src/blob.c
+++ src/blob.c
@@ -223,10 +223,19 @@
223 ** Any prior data in the blob is discarded.
224 */
225 void blob_set(Blob *pBlob, const char *zStr){
226 blob_init(pBlob, zStr, -1);
227 }
228
229 /*
230 ** Initialize a blob to a nul-terminated string obtained from fossil_malloc().
231 ** The blob will take responsibility for freeing the string.
232 */
233 void blob_set_dynamic(Blob *pBlob, char *zStr){
234 blob_init(pBlob, zStr, -1);
235 pBlob->xRealloc = blobReallocMalloc;
236 }
237
238 /*
239 ** Initialize a blob to an empty string.
240 */
241 void blob_zero(Blob *pBlob){
@@ -1095,21 +1104,18 @@
1104 ** to be UTF-8 already, so no conversion is done.
1105 */
1106 void blob_to_utf8_no_bom(Blob *pBlob, int useMbcs){
1107 char *zUtf8;
1108 int bomSize = 0;
 
1109 int bomReverse = 0;
 
1110 if( starts_with_utf8_bom(pBlob, &bomSize) ){
1111 struct Blob temp;
1112 zUtf8 = blob_str(pBlob) + bomSize;
1113 blob_zero(&temp);
1114 blob_append(&temp, zUtf8, -1);
1115 blob_swap(pBlob, &temp);
1116 blob_reset(&temp);
 
1117 }else if( starts_with_utf16_bom(pBlob, &bomSize, &bomReverse) ){
1118 zUtf8 = blob_buffer(pBlob);
1119 if( bomReverse ){
1120 /* Found BOM, but with reversed bytes */
1121 unsigned int i = blob_size(pBlob);
@@ -1122,18 +1128,15 @@
1128 }
1129 /* Make sure the blob contains two terminating 0-bytes */
1130 blob_append(pBlob, "", 1);
1131 zUtf8 = blob_str(pBlob) + bomSize;
1132 zUtf8 = fossil_unicode_to_utf8(zUtf8);
1133 blob_set_dynamic(pBlob, zUtf8);
 
 
 
1134 #if defined(_WIN32)
1135 }else if( useMbcs ){
1136 zUtf8 = fossil_mbcs_to_utf8(blob_str(pBlob));
1137 blob_reset(pBlob);
1138 blob_append(pBlob, zUtf8, -1);
1139 fossil_mbcs_free(zUtf8);
1140 #endif /* _WIN32 */
1141 }
1142 }
1143
M src/db.c
+4
--- src/db.c
+++ src/db.c
@@ -315,10 +315,14 @@
315315
 return sqlite3_bind_double(pStmt->pStmt, paramIdx(pStmt, zParamName), rValue);
316316
 }
317317
 int db_bind_text(Stmt *pStmt, const char *zParamName, const char *zValue){
318318
 return sqlite3_bind_text(pStmt->pStmt, paramIdx(pStmt, zParamName), zValue,
319319
 -1, SQLITE_STATIC);
320
+}
321
+int db_bind_text16(Stmt *pStmt, const char *zParamName, const char *zValue){
322
+ return sqlite3_bind_text16(pStmt->pStmt, paramIdx(pStmt, zParamName), zValue,
323
+ -1, SQLITE_STATIC);
320324
 }
321325
 int db_bind_null(Stmt *pStmt, const char *zParamName){
322326
 return sqlite3_bind_null(pStmt->pStmt, paramIdx(pStmt, zParamName));
323327
 }
324328
 int db_bind_blob(Stmt *pStmt, const char *zParamName, Blob *pBlob){
325329
--- src/db.c
+++ src/db.c
@@ -315,10 +315,14 @@
315 return sqlite3_bind_double(pStmt->pStmt, paramIdx(pStmt, zParamName), rValue);
316 }
317 int db_bind_text(Stmt *pStmt, const char *zParamName, const char *zValue){
318 return sqlite3_bind_text(pStmt->pStmt, paramIdx(pStmt, zParamName), zValue,
319 -1, SQLITE_STATIC);
 
 
 
 
320 }
321 int db_bind_null(Stmt *pStmt, const char *zParamName){
322 return sqlite3_bind_null(pStmt->pStmt, paramIdx(pStmt, zParamName));
323 }
324 int db_bind_blob(Stmt *pStmt, const char *zParamName, Blob *pBlob){
325
--- src/db.c
+++ src/db.c
@@ -315,10 +315,14 @@
315 return sqlite3_bind_double(pStmt->pStmt, paramIdx(pStmt, zParamName), rValue);
316 }
317 int db_bind_text(Stmt *pStmt, const char *zParamName, const char *zValue){
318 return sqlite3_bind_text(pStmt->pStmt, paramIdx(pStmt, zParamName), zValue,
319 -1, SQLITE_STATIC);
320 }
321 int db_bind_text16(Stmt *pStmt, const char *zParamName, const char *zValue){
322 return sqlite3_bind_text16(pStmt->pStmt, paramIdx(pStmt, zParamName), zValue,
323 -1, SQLITE_STATIC);
324 }
325 int db_bind_null(Stmt *pStmt, const char *zParamName){
326 return sqlite3_bind_null(pStmt->pStmt, paramIdx(pStmt, zParamName));
327 }
328 int db_bind_blob(Stmt *pStmt, const char *zParamName, Blob *pBlob){
329
M src/diff.c
+2
--- src/diff.c
+++ src/diff.c
@@ -1762,10 +1762,12 @@
17621762
 Blob *pTemp = pA_Blob;
17631763
 pA_Blob = pB_Blob;
17641764
 pB_Blob = pTemp;
17651765
 }
17661766
 ignoreEolWs = (diffFlags & DIFF_IGNORE_EOLWS)!=0;
1767
+ blob_to_utf8_no_bom(pA_Blob, 0);
1768
+ blob_to_utf8_no_bom(pB_Blob, 0);
17671769
17681770
 /* Prepare the input files */
17691771
 memset(&c, 0, sizeof(c));
17701772
 c.aFrom = break_into_lines(blob_str(pA_Blob), blob_size(pA_Blob),
17711773
 &c.nFrom, ignoreEolWs);
17721774
--- src/diff.c
+++ src/diff.c
@@ -1762,10 +1762,12 @@
1762 Blob *pTemp = pA_Blob;
1763 pA_Blob = pB_Blob;
1764 pB_Blob = pTemp;
1765 }
1766 ignoreEolWs = (diffFlags & DIFF_IGNORE_EOLWS)!=0;
 
 
1767
1768 /* Prepare the input files */
1769 memset(&c, 0, sizeof(c));
1770 c.aFrom = break_into_lines(blob_str(pA_Blob), blob_size(pA_Blob),
1771 &c.nFrom, ignoreEolWs);
1772
--- src/diff.c
+++ src/diff.c
@@ -1762,10 +1762,12 @@
1762 Blob *pTemp = pA_Blob;
1763 pA_Blob = pB_Blob;
1764 pB_Blob = pTemp;
1765 }
1766 ignoreEolWs = (diffFlags & DIFF_IGNORE_EOLWS)!=0;
1767 blob_to_utf8_no_bom(pA_Blob, 0);
1768 blob_to_utf8_no_bom(pB_Blob, 0);
1769
1770 /* Prepare the input files */
1771 memset(&c, 0, sizeof(c));
1772 c.aFrom = break_into_lines(blob_str(pA_Blob), blob_size(pA_Blob),
1773 &c.nFrom, ignoreEolWs);
1774
M src/info.c
+1 -1
--- src/info.c
+++ src/info.c
@@ -1681,16 +1681,16 @@
16811681
 @ sandbox="allow-same-origin"
16821682
 @ onload="this.height = this.contentDocument.documentElement.scrollHeight;">
16831683
 @ </iframe>
16841684
 }else{
16851685
 style_submenu_element("Hex","Hex", "%s/hexdump?name=%s", g.zTop, zUuid);
1686
+ blob_to_utf8_no_bom(&content, 0);
16861687
 zMime = mimetype_from_content(&content);
16871688
 @ <blockquote>
16881689
 if( zMime==0 ){
16891690
 const char *zLn = P("ln");
16901691
 const char *z;
1691
- blob_to_utf8_no_bom(&content, 0);
16921692
 z = blob_str(&content);
16931693
 if( zLn ){
16941694
 output_text_with_line_numbers(z, zLn);
16951695
 }else{
16961696
 @ <pre>
16971697
--- src/info.c
+++ src/info.c
@@ -1681,16 +1681,16 @@
1681 @ sandbox="allow-same-origin"
1682 @ onload="this.height = this.contentDocument.documentElement.scrollHeight;">
1683 @ </iframe>
1684 }else{
1685 style_submenu_element("Hex","Hex", "%s/hexdump?name=%s", g.zTop, zUuid);
 
1686 zMime = mimetype_from_content(&content);
1687 @ <blockquote>
1688 if( zMime==0 ){
1689 const char *zLn = P("ln");
1690 const char *z;
1691 blob_to_utf8_no_bom(&content, 0);
1692 z = blob_str(&content);
1693 if( zLn ){
1694 output_text_with_line_numbers(z, zLn);
1695 }else{
1696 @ <pre>
1697
--- src/info.c
+++ src/info.c
@@ -1681,16 +1681,16 @@
1681 @ sandbox="allow-same-origin"
1682 @ onload="this.height = this.contentDocument.documentElement.scrollHeight;">
1683 @ </iframe>
1684 }else{
1685 style_submenu_element("Hex","Hex", "%s/hexdump?name=%s", g.zTop, zUuid);
1686 blob_to_utf8_no_bom(&content, 0);
1687 zMime = mimetype_from_content(&content);
1688 @ <blockquote>
1689 if( zMime==0 ){
1690 const char *zLn = P("ln");
1691 const char *z;
 
1692 z = blob_str(&content);
1693 if( zLn ){
1694 output_text_with_line_numbers(z, zLn);
1695 }else{
1696 @ <pre>
1697
M src/utf8.c
+11 -13
--- src/utf8.c
+++ src/utf8.c
@@ -54,18 +54,22 @@
5454
 ** returned pointer when done.
5555
 */
5656
 char *fossil_unicode_to_utf8(const void *zUnicode){
5757
 #if defined(_WIN32) || defined(__CYGWIN__)
5858
 int nByte = WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, 0, 0, 0, 0);
59
- char *zUtf = sqlite3_malloc( nByte );
60
- if( zUtf==0 ){
61
- return 0;
62
- }
59
+ char *zUtf = fossil_malloc( nByte );
6360
 WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, zUtf, nByte, 0, 0);
6461
 return zUtf;
6562
 #else
66
- return fossil_strdup(zUnicode); /* TODO: implement for unix */
63
+ static Stmt q;
64
+ char *zUtf8;
65
+ db_static_prepare(&q, "SELECT :utf8");
66
+ db_bind_text16(&q, ":utf8", zUnicode);
67
+ db_step(&q);
68
+ zUtf8 = fossil_strdup(db_column_text(&q, 0));
69
+ db_reset(&q);
70
+ return zUtf8;
6771
 #endif
6872
 }
6973
7074
 /*
7175
 ** Translate UTF-8 to unicode for use in system calls. Return a pointer to the
@@ -73,31 +77,25 @@
7377
 ** used to store the returned pointer when done.
7478
 */
7579
 void *fossil_utf8_to_unicode(const char *zUtf8){
7680
 #if defined(_WIN32) || defined(__CYGWIN__)
7781
 int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
78
- wchar_t *zUnicode = sqlite3_malloc( nByte * 2 );
79
- if( zUnicode==0 ){
80
- return 0;
81
- }
82
+ wchar_t *zUnicode = fossil_malloc( nByte * 2 );
8283
 MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
8384
 return zUnicode;
8485
 #else
86
+ assert( 0 ); /* Never used in unix */
8587
 return fossil_strdup(zUtf8); /* TODO: implement for unix */
8688
 #endif
8789
 }
8890
8991
 /*
9092
 ** Deallocate any memory that was previously allocated by
9193
 ** fossil_unicode_to_utf8().
9294
 */
9395
 void fossil_unicode_free(void *pOld){
94
-#if defined(_WIN32) || defined(__CYGWIN__)
95
- sqlite3_free(pOld);
96
-#else
9796
 fossil_free(pOld);
98
-#endif
9997
 }
10098
10199
 #if defined(__APPLE__) && !defined(WITHOUT_ICONV)
102100
 # include <iconv.h>
103101
 #endif
104102
--- src/utf8.c
+++ src/utf8.c
@@ -54,18 +54,22 @@
54 ** returned pointer when done.
55 */
56 char *fossil_unicode_to_utf8(const void *zUnicode){
57 #if defined(_WIN32) || defined(__CYGWIN__)
58 int nByte = WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, 0, 0, 0, 0);
59 char *zUtf = sqlite3_malloc( nByte );
60 if( zUtf==0 ){
61 return 0;
62 }
63 WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, zUtf, nByte, 0, 0);
64 return zUtf;
65 #else
66 return fossil_strdup(zUnicode); /* TODO: implement for unix */
 
 
 
 
 
 
 
67 #endif
68 }
69
70 /*
71 ** Translate UTF-8 to unicode for use in system calls. Return a pointer to the
@@ -73,31 +77,25 @@
73 ** used to store the returned pointer when done.
74 */
75 void *fossil_utf8_to_unicode(const char *zUtf8){
76 #if defined(_WIN32) || defined(__CYGWIN__)
77 int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
78 wchar_t *zUnicode = sqlite3_malloc( nByte * 2 );
79 if( zUnicode==0 ){
80 return 0;
81 }
82 MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
83 return zUnicode;
84 #else
 
85 return fossil_strdup(zUtf8); /* TODO: implement for unix */
86 #endif
87 }
88
89 /*
90 ** Deallocate any memory that was previously allocated by
91 ** fossil_unicode_to_utf8().
92 */
93 void fossil_unicode_free(void *pOld){
94 #if defined(_WIN32) || defined(__CYGWIN__)
95 sqlite3_free(pOld);
96 #else
97 fossil_free(pOld);
98 #endif
99 }
100
101 #if defined(__APPLE__) && !defined(WITHOUT_ICONV)
102 # include <iconv.h>
103 #endif
104
--- src/utf8.c
+++ src/utf8.c
@@ -54,18 +54,22 @@
54 ** returned pointer when done.
55 */
56 char *fossil_unicode_to_utf8(const void *zUnicode){
57 #if defined(_WIN32) || defined(__CYGWIN__)
58 int nByte = WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, 0, 0, 0, 0);
59 char *zUtf = fossil_malloc( nByte );
 
 
 
60 WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, zUtf, nByte, 0, 0);
61 return zUtf;
62 #else
63 static Stmt q;
64 char *zUtf8;
65 db_static_prepare(&q, "SELECT :utf8");
66 db_bind_text16(&q, ":utf8", zUnicode);
67 db_step(&q);
68 zUtf8 = fossil_strdup(db_column_text(&q, 0));
69 db_reset(&q);
70 return zUtf8;
71 #endif
72 }
73
74 /*
75 ** Translate UTF-8 to unicode for use in system calls. Return a pointer to the
@@ -73,31 +77,25 @@
77 ** used to store the returned pointer when done.
78 */
79 void *fossil_utf8_to_unicode(const char *zUtf8){
80 #if defined(_WIN32) || defined(__CYGWIN__)
81 int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0);
82 wchar_t *zUnicode = fossil_malloc( nByte * 2 );
 
 
 
83 MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte);
84 return zUnicode;
85 #else
86 assert( 0 ); /* Never used in unix */
87 return fossil_strdup(zUtf8); /* TODO: implement for unix */
88 #endif
89 }
90
91 /*
92 ** Deallocate any memory that was previously allocated by
93 ** fossil_unicode_to_utf8().
94 */
95 void fossil_unicode_free(void *pOld){
 
 
 
96 fossil_free(pOld);
 
97 }
98
99 #if defined(__APPLE__) && !defined(WITHOUT_ICONV)
100 # include <iconv.h>
101 #endif
102

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button