|
1
|
utf8((blob), LOOK_BINARY, 0) & LOOK_BINARY) != LOOK_NONE) |
|
2
|
|
|
3
|
/* |
|
4
|
** Output flags for the looks_like_utf8() and looks_like_utf16() routines used |
|
5
|
** to convey status information about the blob content. |
|
6
|
*/ |
|
7
|
#define LOOK_NONE ((int)0x00000000) /* Nothing special was found. */ |
|
8
|
#define LOOK_NUL ((int)0x00000001) /* One or more NUL chars were found. */ |
|
9
|
#define LOOK_CR ((int)0x00000002) /* One or more CR chars were found. */ |
|
10
|
#define LOOK_LONE_CR ((int)0x00000004) /* An unpaired CR char was found. */ |
|
11
|
#define LOOK_LF ((int)0x00000008) /* One or more LF chars were found. */ |
|
12
|
#define LOOK_LONE_LF ((int)0x00000010) /* An unpaired LF char was found. */ |
|
13
|
#define LOOK_CRLF ((int)0x00000020) /* One or more CR/LF pairs were found. */ |
|
14
|
#define LOOK_LONG ((int)0x00000040) /* An over length line was found. */ |
|
15
|
#define LOOK_ODD ((int)0x00000080) /* An odd number of bytes was found. */ |
|
16
|
#define LOOK_SHORT ((int)0x00000100) /* Unable to perform full check. */ |
|
17
|
#define LOOK_INVALID ((int)0x00000200) /* Invalid sequence was found. */ |
|
18
|
#define LOOK_BINARY (LOOK_NUL | LOOK_LONG | LOOK_SHORT) /* May be binary. */ |
|
19
|
#define LOOK_EOL (LOOK_LONE_CR | LOOK_LONE_LF | LOOK_CRLF) /* Line seps. */ |
|
20
|
#endif /* INTERFACE */ |
|
21
|
|
|
22
|
/* definitions for various UTF-8 sequence lengths, encoded as start value |
|
23
|
* and size of each valid range belonging to some lead byte*/ |
|
24
|
#define US2A 0x80, 0x01 /* for lead byte 0xC0 */ |
|
25
|
#define US2B 0x80, 0x40 /* for lead bytes 0xC2-0xDF */ |
|
26
|
#define US3A 0xA0, 0x20 /* for lead byte 0xE0 */ |
|
27
|
#define US3B 0x80, 0x40 /* for lead bytes 0xE1-0xEF */ |
|
28
|
#define US4A 0x90, 0x30 /* for lead byte 0xF0 */ |
|
29
|
#define US4B 0x80, 0x40 /* for lead bytes 0xF1-0xF3 */ |
|
30
|
#define US4C 0x80, 0x10 /* for lead byte 0xF4 */ |
|
31
|
#define US0A 0x00, 0x00 /* for any other lead byte */ |
|
32
|
|
|
33
|
/* a table used for quick lookup of the definition that goes with a |
|
34
|
* particular lead byte */ |
|
35
|
static const unsigned char lb_tab[] = { |
|
36
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
37
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
38
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
39
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
40
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
41
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
42
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
43
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, |
|
44
|
US2A, US0A, US2B, US2B, US2B, US2B, US2B, US2B, |
|
45
|
US2B, US2B, US2B, US2B, US2B, US2B, US2B, US2B, |
|
46
|
US2B, US2B, US2B, US2B, US2B, US2B, US2B, US2B, |
|
47
|
US2B, US2B, US2B, US2B, US2B, US2B, US2B, US2B, |
|
48
|
US3A, US3B, US3B, US3B, US3B, US3B, US3B, US3B, |
|
49
|
US3B, US3B, US3B, US3B, US3B, US3B, US3B, US3B, |
|
50
|
US4A, US4B, US4B, US4B, US4C, US0A, US0A, US0A, |
|
51
|
US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A |
|
52
|
}; |
|
53
|
|
|
54
|
/* |
|
55
|
** This function attempts to scan each logical line within the blob to |
|
56
|
** determine the type of content it appears to contain. The return value |
|
57
|
** is a combination of one or more of the LOOK_XXX flags (see above): |
|
58
|
** |
|
59
|
** !LOOK_BINARY -- The content appears to consist entirely of text; however, |
|
60
|
** the encoding may not be UTF-8. |
|
61
|
** |
|
62
|
** LOOK_BINARY -- The content appears to be binary because it contains one |
|
63
|
** or more embedded NUL characters or an extremely long line. |
|
64
|
** Since this function does not understand UTF-16, it may |
|
65
|
** falsely consider UTF-16 text to be binary. |
|
66
|
** |
|
67
|
** Additional flags (i.e. those other than the ones included in LOOK_BINARY) |
|
68
|
** may be present in the result as well; however, they should not impact the |
|
69
|
** determination of text versus binary content. |
|
70
|
** |
|
71
|
************************************ WARNING ********************************** |
|
72
|
** |
|
73
|
** This function does not validate that the blob content is properly formed |
|
74
|
** UTF-8. It assumes that all code points are the same size. It does not |
|
75
|
** validate any code points. It makes no attempt to detect if any [invalid] |
|
76
|
** switches between UTF-8 and other encodings occur. |
|
77
|
** |
|
78
|
** The only code points that this function cares about are the NUL character, |
|
79
|
** carriage-return, and line-feed. |
|
80
|
** |
|
81
|
** This function examines the contents of the blob until one of the flags |
|
82
|
** specified in "stopFlags" is set. |
|
83
|
** |
|
84
|
************************************ WARNING ********************************** |
|
85
|
*/ |
|
86
|
i){ |
|
87
|
const char *z = ((((ch) << 8) & 0xff00) | (((ch) >> 8) & 0xff)) |
|
88
|
#define UTF16_SWAP_IF(expr,ch) ((8tempts to scan each logical line w==0 ) return flags; /* Empty file -> text */ |
|
89
|
c = *z; |
|
90
|
if( c==0 ){ |
|
91
|
flags |= LOOK_NUL; /* NUL character in a file -> binary */ |
|
92
|
*** |
|
93
|
*/ |
|
94
|
int looks_like_utfto be binary because it cont=1 || z[1]!='\n' ){ |
|
95
|
flags |= LOOK_LONE_CR; /* Not enough chars or next char not LF */ |
|
96
|
} |
|
97
|
} |
|
98
|
j = (c!='\n'); |
|
99
|
if( !j ) flags |= (LOOK_LF | LOOK_LONE_LF); /* Found LF as first char */ |
|
100
|
while( !(flags&stopFlags) && --n>0 ){ |
|
101
|
int c2 = c; |
|
102
|
c = *++z; ++j; |
|
103
|
if( c==0 ){ |
|
104
|
c = *z; |
|
105
|
if( c==0 ){ |
|
106
|
flags |= LOOK_NUL; /* NUL character }else if( c=cters or an extremely long linn' ){ |
|
107
|
flags |= LOOK_LF; |
|
108
|
if( c2=='\r' ){ |
|
109
|
flags |= (LOOK_CR | LOOK_CRLF); /* Found LF preceded by CR */ |
|
110
|
}else{ |
|
111
|
flags |= LOOK_LO a file -> binary *Assume UTF-8 text, prG; /* Very long line -> binary */ |
|
112
|
} |
|
113
|
j = 0; |
|
114
|
}else if( c=='\r' ){ |
|
115
|
fla=1 || z[1]!='\n' ){ |
|
116
|
y CR */ |
|
117
|
}else{ |
|
118
|
{ |
|
119
|
flags |= LOOK_LONE_} |
|
120
|
} |
|
121
|
} |
|
122
|
if( n ){ |
|
123
|
flags |= LOOK_SHORT; /* The whole blob was not examined */ |
|
124
|
} |
|
125
|
if( j> a file -> binary */ |
|
126
|
if( fVerbose ) fossil_print("NUL at start\n"); |
|
127
|
}else if( c=='\r' ){ |
|
128
|
flags |= LOOK_CR; |
|
129
|
if( fVerbose ) fossil_print("CR at start\n"); |
|
130
|
if( n<=1 || z[1]!='\n' ){ |
|
131
|
flags |= LOOK_LONE_CR; /* Not enough chars or next char not LF */ |
|
132
|
if( fVerbose ) fossil_print("Lone CR at start\n"); |
|
133
|
} |
|
134
|
} |
|
135
|
j = (c!='\n'); |
|
136
|
if( !j ) flags |= (LOOK_LF | LOOK_LONE_LF); /* Found LF as first char */ |
|
137
|
while( !(flags&stopFlags) && --n>0 ){ |
|
138
|
int c2 = c; |
|
139
|
c = *++z; ++j; |
|
140
|
if( c==0 ){ |
|
141
|
if( fVerbose && !(flags&LOOK_NUL) ){ |
|
142
|
fossil_print("NUL on line %d\n", nLine); |
|
143
|
} |
|
144
|
flags |= LOOK_NUL; /* NUL character in a file -> binary */ |
|
145
|
}else if( c=='\n' ){ |
|
146
|
flags |= LOOK_LF; |
|
147
|
if( c2=='\r' ){ |
|
148
|
if( fVerbose && !(flags&LOOK_CRLF) ){ |
|
149
|
fossil_print("CRLF on line %d\n", nLine); |
|
150
|
} |
|
151
|
flags |= (LOOK_CR | LOOK_CRLF); /* Found LF preceded by CR */ |
|
152
|
}else{ |
|
153
|
if( fVerbose && !(flags&LOOK_LONE_LF) ){ |
|
154
|
fossil_print("Lone LF on line %d\n", nLine); |
|
155
|
} |
|
156
|
flags |= LOOK_LONE_LF; |
|
157
|
} |
|
158
|
if( j>LENGTH_MASK ){ |
|
159
|
if( fVerbose && !(flags&LOOK_LONG) ){ |
|
160
|
fossil_print("Line %d is longer than %d bytes\n", nLine, j); |
|
161
|
} |
|
162
|
flags |= LOOK_LONG; /* Very long line -> binary */ |
|
163
|
} |
|
164
|
++nLine; |
|
165
|
j = 0; |
|
166
|
}else if( c=='\r' ){ |
|
167
|
flags |= LOOK_CR; |
|
168
|
if( n<=1 || z[1]!='\n' ){ |
|
169
|
if( fVerbose && !(flags&LOOK_LONE_CR) ){ |
|
170
|
fossil_print("Lone CR on line %d\n", nLine); |
|
171
|
} |
|
172
|
flags |= LOOK_LONE_CR; /* Not enough chars or next char not LF */ |
|
173
|
} |
|
174
|
} |
|
175
|
} |
|
176
|
if( n ){ |
|
177
|
flags |= LOOK_SHORT; /* The whole blob was not examined */ |
|
178
|
} |
|
179
|
if( j>LENGTH_MASK ){ |
|
180
|
flags |= LOOK_LONG; /* Very long line -> binary */ |
|
181
|
} |
|
182
|
return flags; |
|
183
|
} |
|
184
|
|
|
185
|
/* |
|
186
|
** Checks for proper UTF-8. It uses the method described in: |
|
187
|
** http://en.wikipedia.org/wiki/UTF-8#Invalid_byte_sequences |
|
188
|
** except for the "overlong form" of \u0000 which is not considered |
|
189
|
** invalid here: Some languages like Java and Tcl use it. This function |
|
190
|
** also considers valid the derivatives CESU-8 & WTF-8 (as described in |
|
191
|
** the same wikipedia article referenced previously). For UTF-8 characters |
|
192
|
** > 0x7f, the variable 'c' not necessary means the real lead byte. |
|
193
|
** It's number of higher 1-bits indicate the number of continuation |
|
194
|
** bytes that are expected to be followed. E.g. when 'c' has a value |
|
195
|
** in the range 0xc0..0xdf it means that after 'c' a single continuation |
|
196
|
** byte is expected. A value 0xe0..0xef means that after 'c' two more |
|
197
|
** continuation bytes are expected. |
|
198
|
*/ |
|
199
|
|
|
200
|
int invalid_utf8( |
|
201
|
const Blob *pContent |
|
202
|
){ |
|
203
|
const unsigned char *z = (unsigned char *) blob_buffer(pContent); |
|
204
|
unsigned int n = blob_size(pContent); |
|
205
|
unsigned char c; /* lead byte to be handled. */ |
|
206
|
|
|
207
|
if( n==0 ) return 0; /* Empty file -> OK */ |
|
208
|
c = *z; |
|
209
|
while( --n>0 ){ |
|
210
|
if( c>=0x80 ){ |
|
211
|
const unsigned char *def; /* pointer to range table*/ |
|
212
|
|
|
213
|
c <<= 1; /* multiply by 2 and get rid of highest bit */ |
|
214
|
def = &lb_tab[c]; /* search fb's valid range in table */ |
|
215
|
if( (unsigned int)(*++z-def[0])>=def[1] ){ |
|
216
|
return LOOK_INVALID; /* Invalid UTF-8 */ |
|
217
|
} |
|
218
|
c = (c>=0xC0) ? (c|3) : ' '; /* determine next lead byte */ |
|
219
|
} else { |
|
220
|
c = *++z; |
|
221
|
} |
|
222
|
} |
|
223
|
return (c>=0x80) ? LOOK_INVALID : 0; /* Final lead byte must be ASCII. */ |
|
224
|
} |
|
225
|
|
|
226
|
/* |
|
227
|
** Define the type needed to represent a Unicode (UTF-16) character. |
|
228
|
*/ |
|
229
|
#ifndef WCHAR_T |
|
230
|
# ifdef _WIN32 |
|
231
|
# define WCHAR_T wchar_t |
|
232
|
# else |
|
233
|
# define WCHAR_T unsigned short |
|
234
|
# endif |
|
235
|
#endif |
|
236
|
|
|
237
|
/* |
|
238
|
** Maximum length of a line in a text file, in UTF-16 characters. (4096) |
|
239
|
** The number of bytes represented by this value cannot exceed LENGTH_MASK |
|
240
|
** bytes, because that is the line buffer size used by the diff engine. |
|
241
|
*/ |
|
242
|
#define UTF16_LENGTH_MASK_SZ (LENGTH_MASK_SZ-(sizeof(WCHAR_T)-sizeof(char))) |
|
243
|
#define UTF16_LENGTH_MASK ((1<<UTF16_LENGTH_MASK_SZ)-1) |
|
244
|
|
|
245
|
/* |
|
246
|
** This macro is used to swap the byte order of a UTF-16 character in the |
|
247
|
** looks_like_utf16() function. |
|
248
|
*/ |
|
249
|
#define UTF16_SWAP(ch) ((((ch) << 8) & 0xff00) | (((ch) >> 8) & 0xff)) |
|
250
|
#define UTF16_SWAP_IF(expr,ch) ((expr) ? UTF16_SWAP((ch)) : (ch)) |
|
251
|
|
|
252
|
/* |
|
253
|
** This function attempts to scan each logical line within the blob to |
|
254
|
** determine the type of content it appears to contain. The return value |
|
255
|
** is a combination of one or more of the LOOK_XXX flags (see above): |
|
256
|
** |
|
257
|
** !LOOK_BINARY -- The content appears to consist entirely of text; however, |
|
258
|
** the encoding may not be UTF-16. |
|
259
|
** |
|
260
|
** LOOK_BINARY -- The content appears to be binary because it contains one |
|
261
|
** or more embedded NUL characters or an extremely long line. |
|
262
|
** Since this function does not understand UTF-8, it may |
|
263
|
** falsely consider UTF-8 text to be binary. |
|
264
|
** |
|
265
|
** Additional flags (i.e. those other than the ones included in LOOK_BINARY) |
|
266
|
** may be present in the result as well; however, they should not impact the |
|
267
|
** determination of text versus binary content. |
|
268
|
** |
|
269
|
************************************ WARNING ********************************** |
|
270
|
** |
|
271
|
** This function does not validate that the blob content is properly formed |
|
272
|
** UTF-16. It assumes that all code points are the same size. It does not |
|
273
|
** validate any code points. It makes no attempt to detect if any [invalid] |
|
274
|
** switches between the UTF-16be and UTF-16le encodings occur. |
|
275
|
** |
|
276
|
** The only code points that this function cares about are the NUL character, |
|
277
|
** carriage-return, and line-feed. |
|
278
|
** |
|
279
|
** This function examines the contents of the blob until one of the flags |
|
280
|
** specified in "stopFlags" is set. |
|
281
|
** |
|
282
|
************************************ WARNING ********************************** |
|
283
|
*/ |
|
284
|
int looks_like_utf16(const Blob *pContent, int bReverse, int stopFlags){ |
|
285
|
const WCHAR_T *z = (WCHAR_T *)blob_buffer(pContent); |
|
286
|
unsigned int n = blob_size(pContent); |
|
287
|
int j, c, flags = LOOK_NONE; /* Assume UTF-16 text, prove otherwise */ |
|
288
|
|
|
289
|
if( n%sizeof(WCHAR_T) ){ |
|
290
|
flags |= LOOK_ODD; /* Odd number of bytes -> binary (UTF-8?) */ |
|
291
|
} |
|
292
|
if( n<sizeof(WCHAR_T) ) return flags;/* Zero or One byte -> binary (UTF-8?) */ |
|
293
|
c = *z; |
|
294
|
if( bReverse ){ |
|
295
|
c = UTF16_SWAP(c); |
|
296
|
} |
|
297
|
if( c==0 ){ |
|
298
|
flags |= LOOK_NUL; /* NUL character in a file -> binary */ |
|
299
|
}else if( c=='\r' ){ |
|
300
|
flags |= LOOK_CR; |
|
301
|
if( n<(2*sizeof(WCHAR_T)) || UTF16_SWAP_IF(bReverse, z[1])!='\n' ){ |
|
302
|
flags |= LOOK_LONE_CR; /* Not enough chars or next char not LF */ |
|
303
|
} |
|
304
|
} |
|
305
|
j = (c!='\n'); |
|
306
|
if( !j ) flags |= (LOOK_LF | LOOK_LONE_LF); /* Found LF as first char */ |
|
307
|
while( !(flags&stopFlags) && ((n-=sizeof(WCHAR_T))>=sizeof(WCHAR_T)) ){ |
|
308
|
int c2 = c; |
|
309
|
c = *++z; |
|
310
|
if( bReverse ){ |
|
311
|
c = UTF16_SWAP(c); |
|
312
|
} |
|
313
|
++j; |
|
314
|
if( c==0 ){ |
|
315
|
flags |= LOOK_NUL; /* NUL character in a file -> binary */ |
|
316
|
}else if( c=='\n' ){ |
|
317
|
flags |= LOOK_LF; |
|
318
|
if( c2=='\r' ){ |
|
319
|
flags |= (LOOK_CR | LOOK_CRLF); /* Found LF preceded by CR */ |
|
320
|
}else{ |
|
321
|
flags |= LOOK_LONE_LF; |
|
322
|
} |
|
323
|
if( j>UTF16_LENGTH_MASK ){ |
|
324
|
flags |= LOOK_LONG; /* Very long line -> binary */ |
|
325
|
} |
|
326
|
j = 0; |
|
327
|
}else if( c=='\r' ){ |
|
328
|
flags |= LOOK_CR; |
|
329
|
if( n<(2*sizeof(WCHAR_T)) || UTF16_SWAP_IF(bReverse, z[1])!='\n' ){ |
|
330
|
flags |= LOOK_LONE_CR; /* Not enough chars or next char not LF */ |
|
331
|
} |
|
332
|
} |
|
333
|
} |
|
334
|
if( n ){ |
|
335
|
flags |= LOOK_SHORT; /* The whole blob was not examined */ |
|
336
|
} |
|
337
|
if( j>UTF16_LENGTH_MASK ){ |
|
338
|
flags |= LOOK_LONG; /* Very long line -> binary */ |
|
339
|
} |
|
340
|
return flags; |
|
341
|
} |
|
342
|
|
|
343
|
/* |
|
344
|
** This function returns an array of bytes representing the byte-order-mark |
|
345
|
** for UTF-8. |
|
346
|
*/ |
|
347
|
const unsigned char *get_utf8_bom(int *pnByte){ |
|
348
|
static const unsigned char bom[] = { |
|
349
|
0xef, 0xbb, 0xbf, 0x00, 0x00, 0x00 |
|
350
|
}; |
|
351
|
if( pnByte ) *pnByte = 3; |
|
352
|
return bom; |
|
353
|
} |
|
354
|
|
|
355
|
/* |
|
356
|
** This function returns non-zero if the blob starts with a UTF-8 |
|
357
|
** byte-order-mark (BOM). |
|
358
|
*/ |
|
359
|
int starts_with_utf8_bom(const Blob *pContent, int *pnByte){ |
|
360
|
const char *z = blob_buffer(pContent); |
|
361
|
int bomSize = 0; |
|
362
|
const unsigned char *bom = get_utf8_bom(&bomSize); |
|
363
|
|
|
364
|
if( pnByte ) *pnByte = bomSize; |
|
365
|
if( (int)blob_size(pContent)<bomSize ) return 0; |
|
366
|
return memcmp(z, bom, bomSize)==0; |
|
367
|
} |
|
368
|
|
|
369
|
/* |
|
370
|
** This function returns non-zero if the blob starts with a UTF-16 |
|
371
|
** byte-order-mark (BOM), either in the endianness of the machine |
|
372
|
** or in reversed byte order. The UTF-32 BOM is ruled out by checking |
|
373
|
** if the UTF-16 BOM is not immediately followed by (utf16) 0. |
|
374
|
** pnByte is only set when the funct 1 on little-endian and 0 on big-endian platforms. See |
|
375
|
** clause D98 of conformance (section 3.10) of the Unicode standard. |
|
376
|
*/ |
|
377
|
int starts_with_utf16_bom( |
|
378
|
const Blob *pContent, /* IN: Blob content to perform BOM detection on. */ |
|
379
|
int *pnByte, /* OUT: The number of bytes used for the BOM. */ |
|
380
|
int *pbReverse /* OUT: Non-zero for BOM in reverse byte-order. */ |
|
381
|
){ |
|
382
|
const unsigned char *z = (unsigned char *)blob_buffer(pContent); |
|
383
|
int bomSize = sizeof(unsigned short); |
|
384
|
int size = blob_size(pContent); |
|
385
|
unsigned short i0; |
|
386
|
|
|
387
|
if( size<bomSize ) goto noBom; /* No: cannot read BOM. */ |
|
388
|
if( size>=(2*bomSize) && z[2]==0 && z[3]==0 ) goto no"verbose","v",0)!=0; |
|
389
|
int nRepeat = 1; |
|
390
|
|
|
391
|
if( g.argc!=3 ) usage("FILENAME"); |
|
392
|
if( zCount ){ |
|
393
|
nRepeat = atoi(zCount); |
|
394
|
} |
|
395
|
blob_read_from_file(&blob, g.argv[2], ExtFILE); |
|
396
|
while( --nRepeat >= 0 ){ |
|
397
|
fUtf8 = starts_with_utf8_bom(&blob, 0); |
|
398
|
fUtf16 = starts_with_utf16_bom(&blob, 0, &bRevUtf16); |
|
399
|
if( fForceUtf8 ){ |
|
400
|
fUnicode = 0; |
|
401
|
}else{ |
|
402
|
fUnicode = could_be_utf16(&blob, 0) || fForceUtf16; |
|
403
|
} |
|
404
|
if( fUnicode ){ |
|
405
|
lookFlags = looks_like_utf16(&blob, bRevUtf16, 0); |
|
406
|
}else{ |
|
407
|
lookFlags |