Annotation of src/usr.bin/file/ascmagic.c, Revision 1.11
1.11 ! deraadt 1: /* $OpenBSD: ascmagic.c,v 1.10 2009/04/24 18:54:34 chl Exp $ */
1.1 deraadt 2: /*
1.6 ian 3: * Copyright (c) Ian F. Darwin 1986-1995.
4: * Software written by Ian F. Darwin and others;
5: * maintained 1995-present by Christos Zoulas and others.
6: *
7: * Redistribution and use in source and binary forms, with or without
8: * modification, are permitted provided that the following conditions
9: * are met:
10: * 1. Redistributions of source code must retain the above copyright
11: * notice immediately at the beginning of the file, without modification,
12: * this list of conditions, and the following disclaimer.
13: * 2. Redistributions in binary form must reproduce the above copyright
14: * notice, this list of conditions and the following disclaimer in the
15: * documentation and/or other materials provided with the distribution.
16: *
17: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20: * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
21: * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27: * SUCH DAMAGE.
1.1 deraadt 28: */
1.8 tedu 29: /*
30: * ASCII magic -- file types that we know based on keywords
31: * that can appear anywhere in the file.
32: *
33: * Extensively modified by Eric Fischer <enf@pobox.com> in July, 2000,
34: * to handle character codes other than ASCII on a unified basis.
35: *
36: * Joerg Wunsch <joerg@freebsd.org> wrote the original support for 8-bit
37: * international characters, now subsumed into this file.
38: */
1.1 deraadt 39:
1.8 tedu 40: #include "file.h"
41: #include "magic.h"
1.1 deraadt 42: #include <stdio.h>
43: #include <string.h>
1.8 tedu 44: #include <memory.h>
1.1 deraadt 45: #include <ctype.h>
46: #include <stdlib.h>
1.8 tedu 47: #ifdef HAVE_UNISTD_H
1.1 deraadt 48: #include <unistd.h>
1.8 tedu 49: #endif
1.1 deraadt 50: #include "names.h"
51:
1.8 tedu 52: #define MAXLINELEN 300 /* longest sane line length */
53: #define ISSPC(x) ((x) == ' ' || (x) == '\t' || (x) == '\r' || (x) == '\n' \
54: || (x) == 0x85 || (x) == '\f')
55:
56: private int looks_ascii(const unsigned char *, size_t, unichar *, size_t *);
1.10 chl 57: private int looks_utf8_with_BOM(const unsigned char *, size_t, unichar *,
58: size_t *);
59: protected int file_looks_utf8(const unsigned char *, size_t, unichar *, size_t *);
60: private int looks_ucs16(const unsigned char *, size_t, unichar *, size_t *);
1.8 tedu 61: private int looks_latin1(const unsigned char *, size_t, unichar *, size_t *);
62: private int looks_extended(const unsigned char *, size_t, unichar *, size_t *);
63: private void from_ebcdic(const unsigned char *, size_t, unsigned char *);
64: private int ascmatch(const unsigned char *, const unichar *, size_t);
1.10 chl 65: private unsigned char *encode_utf8(unsigned char *, size_t, unichar *, size_t);
1.8 tedu 66:
1.1 deraadt 67:
1.8 tedu 68: protected int
69: file_ascmagic(struct magic_set *ms, const unsigned char *buf, size_t nbytes)
70: {
71: size_t i;
1.10 chl 72: unsigned char *nbuf = NULL, *utf8_buf = NULL, *utf8_end;
1.9 chl 73: unichar *ubuf = NULL;
1.10 chl 74: size_t ulen, mlen;
75: const struct names *p;
1.9 chl 76: int rv = -1;
1.10 chl 77: int mime = ms->flags & MAGIC_MIME;
1.1 deraadt 78:
1.8 tedu 79: const char *code = NULL;
80: const char *code_mime = NULL;
81: const char *type = NULL;
82: const char *subtype = NULL;
83: const char *subtype_mime = NULL;
84:
85: int has_escapes = 0;
86: int has_backspace = 0;
1.9 chl 87: int seen_cr = 0;
1.8 tedu 88:
89: int n_crlf = 0;
90: int n_lf = 0;
91: int n_cr = 0;
92: int n_nel = 0;
93:
1.9 chl 94: size_t last_line_end = (size_t)-1;
1.8 tedu 95: int has_long_lines = 0;
96:
1.1 deraadt 97: /*
1.8 tedu 98: * Undo the NUL-termination kindly provided by process()
99: * but leave at least one byte to look at
1.1 deraadt 100: */
1.8 tedu 101: while (nbytes > 1 && buf[nbytes - 1] == '\0')
102: nbytes--;
103:
1.9 chl 104: if ((nbuf = calloc(1, (nbytes + 1) * sizeof(nbuf[0]))) == NULL)
105: goto done;
106: if ((ubuf = calloc(1, (nbytes + 1) * sizeof(ubuf[0]))) == NULL)
107: goto done;
1.8 tedu 108:
109: /*
110: * Then try to determine whether it's any character code we can
111: * identify. Each of these tests, if it succeeds, will leave
112: * the text converted into one-unichar-per-character Unicode in
113: * ubuf, and the number of characters converted in ulen.
114: */
115: if (looks_ascii(buf, nbytes, ubuf, &ulen)) {
116: code = "ASCII";
117: code_mime = "us-ascii";
118: type = "text";
1.10 chl 119: } else if (looks_utf8_with_BOM(buf, nbytes, ubuf, &ulen) > 0) {
120: code = "UTF-8 Unicode (with BOM)";
121: code_mime = "utf-8";
122: type = "text";
123: } else if (file_looks_utf8(buf, nbytes, ubuf, &ulen) > 1) {
1.8 tedu 124: code = "UTF-8 Unicode";
125: code_mime = "utf-8";
126: type = "text";
1.10 chl 127: } else if ((i = looks_ucs16(buf, nbytes, ubuf, &ulen)) != 0) {
1.8 tedu 128: if (i == 1)
129: code = "Little-endian UTF-16 Unicode";
130: else
131: code = "Big-endian UTF-16 Unicode";
132:
133: type = "character data";
134: code_mime = "utf-16"; /* is this defined? */
135: } else if (looks_latin1(buf, nbytes, ubuf, &ulen)) {
136: code = "ISO-8859";
137: type = "text";
138: code_mime = "iso-8859-1";
139: } else if (looks_extended(buf, nbytes, ubuf, &ulen)) {
140: code = "Non-ISO extended-ASCII";
141: type = "text";
142: code_mime = "unknown";
143: } else {
144: from_ebcdic(buf, nbytes, nbuf);
145:
146: if (looks_ascii(nbuf, nbytes, ubuf, &ulen)) {
147: code = "EBCDIC";
148: type = "character data";
149: code_mime = "ebcdic";
150: } else if (looks_latin1(nbuf, nbytes, ubuf, &ulen)) {
151: code = "International EBCDIC";
152: type = "character data";
153: code_mime = "ebcdic";
154: } else {
1.9 chl 155: rv = 0;
156: goto done; /* doesn't look like text at all */
1.8 tedu 157: }
1.1 deraadt 158: }
159:
1.9 chl 160: if (nbytes <= 1) {
161: rv = 0;
162: goto done;
163: }
164:
1.10 chl 165: /* Convert ubuf to UTF-8 and try text soft magic */
166: /* If original was ASCII or UTF-8, could use nbuf instead of
167: re-converting. */
168: /* malloc size is a conservative overestimate; could be
169: re-converting improved, or at least realloced after
170: re-converting conversion. */
171: mlen = ulen * 6;
172: if ((utf8_buf = malloc(mlen)) == NULL) {
173: file_oomem(ms, mlen);
174: goto done;
1.8 tedu 175: }
1.10 chl 176: if ((utf8_end = encode_utf8(utf8_buf, mlen, ubuf, ulen)) == NULL)
177: goto done;
178: if (file_softmagic(ms, utf8_buf, utf8_end - utf8_buf, TEXTTEST) != 0) {
179: rv = 1;
180: goto done;
1.8 tedu 181: }
182:
183: /* look for tokens from names.h - this is expensive! */
1.9 chl 184: if ((ms->flags & MAGIC_NO_CHECK_TOKENS) != 0)
185: goto subtype_identified;
186:
1.8 tedu 187: i = 0;
188: while (i < ulen) {
189: size_t end;
190:
1.10 chl 191: /* skip past any leading space */
1.8 tedu 192: while (i < ulen && ISSPC(ubuf[i]))
193: i++;
194: if (i >= ulen)
195: break;
196:
1.10 chl 197: /* find the next whitespace */
1.8 tedu 198: for (end = i + 1; end < nbytes; end++)
199: if (ISSPC(ubuf[end]))
200: break;
201:
1.10 chl 202: /* compare the word thus isolated against the token list */
1.8 tedu 203: for (p = names; p < names + NNAMES; p++) {
204: if (ascmatch((const unsigned char *)p->name, ubuf + i,
205: end - i)) {
206: subtype = types[p->type].human;
207: subtype_mime = types[p->type].mime;
208: goto subtype_identified;
209: }
210: }
211:
212: i = end;
213: }
214:
215: subtype_identified:
216:
1.10 chl 217: /* Now try to discover other details about the file. */
1.8 tedu 218: for (i = 0; i < ulen; i++) {
1.9 chl 219: if (ubuf[i] == '\n') {
220: if (seen_cr)
221: n_crlf++;
222: else
223: n_lf++;
224: last_line_end = i;
225: } else if (seen_cr)
226: n_cr++;
227:
228: seen_cr = (ubuf[i] == '\r');
229: if (seen_cr)
230: last_line_end = i;
231:
232: if (ubuf[i] == 0x85) { /* X3.64/ECMA-43 "next line" character */
233: n_nel++;
234: last_line_end = i;
235: }
236:
237: /* If this line is _longer_ than MAXLINELEN, remember it. */
1.8 tedu 238: if (i > last_line_end + MAXLINELEN)
239: has_long_lines = 1;
240:
241: if (ubuf[i] == '\033')
242: has_escapes = 1;
243: if (ubuf[i] == '\b')
244: has_backspace = 1;
1.9 chl 245: }
1.8 tedu 246:
1.9 chl 247: /* Beware, if the data has been truncated, the final CR could have
248: been followed by a LF. If we have HOWMANY bytes, it indicates
249: that the data might have been truncated, probably even before
250: this function was called. */
251: if (seen_cr && nbytes < HOWMANY)
252: n_cr++;
1.8 tedu 253:
1.10 chl 254: if (mime) {
255: if (mime & MAGIC_MIME_TYPE) {
256: if (subtype_mime) {
257: if (file_printf(ms, subtype_mime) == -1)
258: goto done;
259: } else {
260: if (file_printf(ms, "text/plain") == -1)
261: goto done;
262: }
1.8 tedu 263: }
264:
1.10 chl 265: if ((mime == 0 || mime == MAGIC_MIME) && code_mime) {
266: if ((mime & MAGIC_MIME_TYPE) &&
267: file_printf(ms, " charset=") == -1)
1.9 chl 268: goto done;
1.8 tedu 269: if (file_printf(ms, code_mime) == -1)
1.9 chl 270: goto done;
1.8 tedu 271: }
1.10 chl 272:
273: if (mime == MAGIC_MIME_ENCODING)
274: file_printf(ms, "binary");
1.8 tedu 275: } else {
276: if (file_printf(ms, code) == -1)
1.9 chl 277: goto done;
1.8 tedu 278:
279: if (subtype) {
280: if (file_printf(ms, " ") == -1)
1.9 chl 281: goto done;
1.8 tedu 282: if (file_printf(ms, subtype) == -1)
1.9 chl 283: goto done;
1.8 tedu 284: }
285:
286: if (file_printf(ms, " ") == -1)
1.9 chl 287: goto done;
1.8 tedu 288: if (file_printf(ms, type) == -1)
1.9 chl 289: goto done;
1.8 tedu 290:
291: if (has_long_lines)
292: if (file_printf(ms, ", with very long lines") == -1)
1.9 chl 293: goto done;
1.8 tedu 294:
295: /*
296: * Only report line terminators if we find one other than LF,
297: * or if we find none at all.
298: */
299: if ((n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0) ||
300: (n_crlf != 0 || n_cr != 0 || n_nel != 0)) {
301: if (file_printf(ms, ", with") == -1)
1.9 chl 302: goto done;
1.8 tedu 303:
304: if (n_crlf == 0 && n_cr == 0 && n_nel == 0 && n_lf == 0) {
305: if (file_printf(ms, " no") == -1)
1.9 chl 306: goto done;
1.8 tedu 307: } else {
308: if (n_crlf) {
309: if (file_printf(ms, " CRLF") == -1)
1.9 chl 310: goto done;
1.8 tedu 311: if (n_cr || n_lf || n_nel)
312: if (file_printf(ms, ",") == -1)
1.9 chl 313: goto done;
1.8 tedu 314: }
315: if (n_cr) {
316: if (file_printf(ms, " CR") == -1)
1.9 chl 317: goto done;
1.8 tedu 318: if (n_lf || n_nel)
319: if (file_printf(ms, ",") == -1)
1.9 chl 320: goto done;
1.8 tedu 321: }
322: if (n_lf) {
323: if (file_printf(ms, " LF") == -1)
1.9 chl 324: goto done;
1.8 tedu 325: if (n_nel)
326: if (file_printf(ms, ",") == -1)
1.9 chl 327: goto done;
1.8 tedu 328: }
329: if (n_nel)
330: if (file_printf(ms, " NEL") == -1)
1.9 chl 331: goto done;
1.8 tedu 332: }
333:
334: if (file_printf(ms, " line terminators") == -1)
1.9 chl 335: goto done;
1.8 tedu 336: }
337:
338: if (has_escapes)
339: if (file_printf(ms, ", with escape sequences") == -1)
1.9 chl 340: goto done;
1.8 tedu 341: if (has_backspace)
342: if (file_printf(ms, ", with overstriking") == -1)
1.9 chl 343: goto done;
1.8 tedu 344: }
1.9 chl 345: rv = 1;
346: done:
347: if (nbuf)
348: free(nbuf);
349: if (ubuf)
350: free(ubuf);
1.10 chl 351: if (utf8_buf)
352: free(utf8_buf);
1.8 tedu 353:
1.9 chl 354: return rv;
1.8 tedu 355: }
356:
357: private int
358: ascmatch(const unsigned char *s, const unichar *us, size_t ulen)
359: {
360: size_t i;
361:
362: for (i = 0; i < ulen; i++) {
363: if (s[i] != us[i])
364: return 0;
365: }
366:
367: if (s[i])
368: return 0;
369: else
1.1 deraadt 370: return 1;
1.8 tedu 371: }
372:
373: /*
374: * This table reflects a particular philosophy about what constitutes
375: * "text," and there is room for disagreement about it.
376: *
377: * Version 3.31 of the file command considered a file to be ASCII if
378: * each of its characters was approved by either the isascii() or
379: * isalpha() function. On most systems, this would mean that any
380: * file consisting only of characters in the range 0x00 ... 0x7F
381: * would be called ASCII text, but many systems might reasonably
382: * consider some characters outside this range to be alphabetic,
383: * so the file command would call such characters ASCII. It might
384: * have been more accurate to call this "considered textual on the
385: * local system" than "ASCII."
386: *
387: * It considered a file to be "International language text" if each
388: * of its characters was either an ASCII printing character (according
389: * to the real ASCII standard, not the above test), a character in
390: * the range 0x80 ... 0xFF, or one of the following control characters:
391: * backspace, tab, line feed, vertical tab, form feed, carriage return,
392: * escape. No attempt was made to determine the language in which files
393: * of this type were written.
394: *
395: *
396: * The table below considers a file to be ASCII if all of its characters
397: * are either ASCII printing characters (again, according to the X3.4
398: * standard, not isascii()) or any of the following controls: bell,
399: * backspace, tab, line feed, form feed, carriage return, esc, nextline.
400: *
401: * I include bell because some programs (particularly shell scripts)
402: * use it literally, even though it is rare in normal text. I exclude
403: * vertical tab because it never seems to be used in real text. I also
404: * include, with hesitation, the X3.64/ECMA-43 control nextline (0x85),
405: * because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline
406: * character to. It might be more appropriate to include it in the 8859
407: * set instead of the ASCII set, but it's got to be included in *something*
408: * we recognize or EBCDIC files aren't going to be considered textual.
409: * Some old Unix source files use SO/SI (^N/^O) to shift between Greek
410: * and Latin characters, so these should possibly be allowed. But they
411: * make a real mess on VT100-style displays if they're not paired properly,
412: * so we are probably better off not calling them text.
413: *
414: * A file is considered to be ISO-8859 text if its characters are all
415: * either ASCII, according to the above definition, or printing characters
416: * from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF.
417: *
418: * Finally, a file is considered to be international text from some other
419: * character code if its characters are all either ISO-8859 (according to
420: * the above definition) or characters in the range 0x80 ... 0x9F, which
421: * ISO-8859 considers to be control characters but the IBM PC and Macintosh
422: * consider to be printing characters.
423: */
424:
425: #define F 0 /* character never appears in text */
426: #define T 1 /* character appears in plain ASCII text */
427: #define I 2 /* character appears in ISO-8859 text */
428: #define X 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
429:
430: private char text_chars[256] = {
431: /* BEL BS HT LF FF CR */
432: F, F, F, F, F, F, F, T, T, T, T, F, T, T, F, F, /* 0x0X */
433: /* ESC */
434: F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */
435: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */
436: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */
437: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */
438: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */
439: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */
440: T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */
441: /* NEL */
442: X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */
443: X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */
444: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */
445: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */
446: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */
447: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */
448: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */
449: I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I /* 0xfX */
450: };
451:
452: private int
453: looks_ascii(const unsigned char *buf, size_t nbytes, unichar *ubuf,
454: size_t *ulen)
455: {
1.9 chl 456: size_t i;
1.8 tedu 457:
458: *ulen = 0;
459:
460: for (i = 0; i < nbytes; i++) {
461: int t = text_chars[buf[i]];
462:
463: if (t != T)
464: return 0;
465:
466: ubuf[(*ulen)++] = buf[i];
1.1 deraadt 467: }
468:
1.8 tedu 469: return 1;
470: }
471:
472: private int
473: looks_latin1(const unsigned char *buf, size_t nbytes, unichar *ubuf, size_t *ulen)
474: {
1.9 chl 475: size_t i;
1.8 tedu 476:
477: *ulen = 0;
1.3 millert 478:
479: for (i = 0; i < nbytes; i++) {
1.8 tedu 480: int t = text_chars[buf[i]];
481:
482: if (t != T && t != I)
483: return 0;
484:
485: ubuf[(*ulen)++] = buf[i];
1.3 millert 486: }
487:
1.8 tedu 488: return 1;
489: }
490:
491: private int
492: looks_extended(const unsigned char *buf, size_t nbytes, unichar *ubuf,
493: size_t *ulen)
494: {
1.9 chl 495: size_t i;
1.8 tedu 496:
497: *ulen = 0;
498:
499: for (i = 0; i < nbytes; i++) {
500: int t = text_chars[buf[i]];
501:
502: if (t != T && t != I && t != X)
503: return 0;
504:
505: ubuf[(*ulen)++] = buf[i];
506: }
507:
508: return 1;
509: }
510:
1.10 chl 511: /*
512: * Encode Unicode string as UTF-8, returning pointer to character
513: * after end of string, or NULL if an invalid character is found.
514: */
515: private unsigned char *
516: encode_utf8(unsigned char *buf, size_t len, unichar *ubuf, size_t ulen)
517: {
518: size_t i;
519: unsigned char *end = buf + len;
520:
521: for (i = 0; i < ulen; i++) {
522: if (ubuf[i] <= 0x7f) {
523: if (end - buf < 1)
524: return NULL;
525: *buf++ = (unsigned char)ubuf[i];
526: } else if (ubuf[i] <= 0x7ff) {
527: if (end - buf < 2)
528: return NULL;
529: *buf++ = (unsigned char)((ubuf[i] >> 6) + 0xc0);
530: *buf++ = (unsigned char)((ubuf[i] & 0x3f) + 0x80);
531: } else if (ubuf[i] <= 0xffff) {
532: if (end - buf < 3)
533: return NULL;
534: *buf++ = (unsigned char)((ubuf[i] >> 12) + 0xe0);
535: *buf++ = (unsigned char)(((ubuf[i] >> 6) & 0x3f) + 0x80);
536: *buf++ = (unsigned char)((ubuf[i] & 0x3f) + 0x80);
537: } else if (ubuf[i] <= 0x1fffff) {
538: if (end - buf < 4)
539: return NULL;
540: *buf++ = (unsigned char)((ubuf[i] >> 18) + 0xf0);
541: *buf++ = (unsigned char)(((ubuf[i] >> 12) & 0x3f) + 0x80);
542: *buf++ = (unsigned char)(((ubuf[i] >> 6) & 0x3f) + 0x80);
543: *buf++ = (unsigned char)((ubuf[i] & 0x3f) + 0x80);
544: } else if (ubuf[i] <= 0x3ffffff) {
545: if (end - buf < 5)
546: return NULL;
547: *buf++ = (unsigned char)((ubuf[i] >> 24) + 0xf8);
548: *buf++ = (unsigned char)(((ubuf[i] >> 18) & 0x3f) + 0x80);
549: *buf++ = (unsigned char)(((ubuf[i] >> 12) & 0x3f) + 0x80);
550: *buf++ = (unsigned char)(((ubuf[i] >> 6) & 0x3f) + 0x80);
551: *buf++ = (unsigned char)((ubuf[i] & 0x3f) + 0x80);
552: } else if (ubuf[i] <= 0x7fffffff) {
553: if (end - buf < 6)
554: return NULL;
555: *buf++ = (unsigned char)((ubuf[i] >> 30) + 0xfc);
556: *buf++ = (unsigned char)(((ubuf[i] >> 24) & 0x3f) + 0x80);
557: *buf++ = (unsigned char)(((ubuf[i] >> 18) & 0x3f) + 0x80);
558: *buf++ = (unsigned char)(((ubuf[i] >> 12) & 0x3f) + 0x80);
559: *buf++ = (unsigned char)(((ubuf[i] >> 6) & 0x3f) + 0x80);
560: *buf++ = (unsigned char)((ubuf[i] & 0x3f) + 0x80);
561: } else /* Invalid character */
562: return NULL;
563: }
564:
565: return buf;
566: }
567:
568: /*
569: * Decide whether some text looks like UTF-8. Returns:
570: *
571: * -1: invalid UTF-8
572: * 0: uses odd control characters, so doesn't look like text
573: * 1: 7-bit text
574: * 2: definitely UTF-8 text (valid high-bit set bytes)
575: *
576: * If ubuf is non-NULL on entry, text is decoded into ubuf, *ulen;
577: * ubuf must be big enough!
578: */
579: protected int
580: file_looks_utf8(const unsigned char *buf, size_t nbytes, unichar *ubuf, size_t *ulen)
1.8 tedu 581: {
1.9 chl 582: size_t i;
583: int n;
1.8 tedu 584: unichar c;
1.10 chl 585: int gotone = 0, ctrl = 0;
1.8 tedu 586:
1.10 chl 587: if (ubuf)
588: *ulen = 0;
1.8 tedu 589:
590: for (i = 0; i < nbytes; i++) {
591: if ((buf[i] & 0x80) == 0) { /* 0xxxxxxx is plain ASCII */
592: /*
593: * Even if the whole file is valid UTF-8 sequences,
594: * still reject it if it uses weird control characters.
595: */
596:
597: if (text_chars[buf[i]] != T)
1.10 chl 598: ctrl = 1;
1.8 tedu 599:
1.10 chl 600: if (ubuf)
601: ubuf[(*ulen)++] = buf[i];
1.8 tedu 602: } else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
1.10 chl 603: return -1;
1.8 tedu 604: } else { /* 11xxxxxx begins UTF-8 */
605: int following;
606:
607: if ((buf[i] & 0x20) == 0) { /* 110xxxxx */
608: c = buf[i] & 0x1f;
609: following = 1;
610: } else if ((buf[i] & 0x10) == 0) { /* 1110xxxx */
611: c = buf[i] & 0x0f;
612: following = 2;
613: } else if ((buf[i] & 0x08) == 0) { /* 11110xxx */
614: c = buf[i] & 0x07;
615: following = 3;
616: } else if ((buf[i] & 0x04) == 0) { /* 111110xx */
617: c = buf[i] & 0x03;
618: following = 4;
619: } else if ((buf[i] & 0x02) == 0) { /* 1111110x */
620: c = buf[i] & 0x01;
621: following = 5;
622: } else
1.10 chl 623: return -1;
1.8 tedu 624:
625: for (n = 0; n < following; n++) {
626: i++;
627: if (i >= nbytes)
628: goto done;
629:
630: if ((buf[i] & 0x80) == 0 || (buf[i] & 0x40))
1.10 chl 631: return -1;
1.8 tedu 632:
633: c = (c << 6) + (buf[i] & 0x3f);
1.1 deraadt 634: }
1.8 tedu 635:
1.10 chl 636: if (ubuf)
637: ubuf[(*ulen)++] = c;
1.8 tedu 638: gotone = 1;
1.1 deraadt 639: }
640: }
1.8 tedu 641: done:
1.10 chl 642: return ctrl ? 0 : (gotone ? 2 : 1);
643: }
644:
645: /*
646: * Decide whether some text looks like UTF-8 with BOM. If there is no
647: * BOM, return -1; otherwise return the result of looks_utf8 on the
648: * rest of the text.
649: */
650: private int
651: looks_utf8_with_BOM(const unsigned char *buf, size_t nbytes, unichar *ubuf,
652: size_t *ulen)
653: {
654: if (nbytes > 3 && buf[0] == 0xef && buf[1] == 0xbb && buf[2] == 0xbf)
655: return file_looks_utf8(buf + 3, nbytes - 3, ubuf, ulen);
656: else
657: return -1;
1.8 tedu 658: }
659:
660: private int
1.10 chl 661: looks_ucs16(const unsigned char *buf, size_t nbytes, unichar *ubuf,
1.8 tedu 662: size_t *ulen)
663: {
664: int bigend;
1.9 chl 665: size_t i;
1.1 deraadt 666:
1.8 tedu 667: if (nbytes < 2)
668: return 0;
669:
670: if (buf[0] == 0xff && buf[1] == 0xfe)
671: bigend = 0;
672: else if (buf[0] == 0xfe && buf[1] == 0xff)
673: bigend = 1;
674: else
675: return 0;
676:
677: *ulen = 0;
678:
679: for (i = 2; i + 1 < nbytes; i += 2) {
680: /* XXX fix to properly handle chars > 65536 */
681:
682: if (bigend)
683: ubuf[(*ulen)++] = buf[i + 1] + 256 * buf[i];
684: else
685: ubuf[(*ulen)++] = buf[i] + 256 * buf[i + 1];
686:
687: if (ubuf[*ulen - 1] == 0xfffe)
688: return 0;
689: if (ubuf[*ulen - 1] < 128 &&
690: text_chars[(size_t)ubuf[*ulen - 1]] != T)
691: return 0;
1.1 deraadt 692: }
1.8 tedu 693:
694: return 1 + bigend;
1.1 deraadt 695: }
696:
1.8 tedu 697: #undef F
698: #undef T
699: #undef I
700: #undef X
701:
702: /*
703: * This table maps each EBCDIC character to an (8-bit extended) ASCII
704: * character, as specified in the rationale for the dd(1) command in
705: * draft 11.2 (September, 1991) of the POSIX P1003.2 standard.
706: *
707: * Unfortunately it does not seem to correspond exactly to any of the
708: * five variants of EBCDIC documented in IBM's _Enterprise Systems
709: * Architecture/390: Principles of Operation_, SA22-7201-06, Seventh
710: * Edition, July, 1999, pp. I-1 - I-4.
711: *
712: * Fortunately, though, all versions of EBCDIC, including this one, agree
713: * on most of the printing characters that also appear in (7-bit) ASCII.
714: * Of these, only '|', '!', '~', '^', '[', and ']' are in question at all.
715: *
716: * Fortunately too, there is general agreement that codes 0x00 through
717: * 0x3F represent control characters, 0x41 a nonbreaking space, and the
718: * remainder printing characters.
719: *
720: * This is sufficient to allow us to identify EBCDIC text and to distinguish
721: * between old-style and internationalized examples of text.
722: */
723:
724: private unsigned char ebcdic_to_ascii[] = {
725: 0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15,
726: 16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31,
727: 128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7,
728: 144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26,
729: ' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|',
730: '&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~',
731: '-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?',
732: 186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'','=', '"',
733: 195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201,
734: 202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208,
735: 209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215,
736: 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231,
737: '{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237,
738: '}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243,
739: '\\',159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249,
740: '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255
741: };
742:
743: #ifdef notdef
744: /*
745: * The following EBCDIC-to-ASCII table may relate more closely to reality,
746: * or at least to modern reality. It comes from
747: *
748: * http://ftp.s390.ibm.com/products/oe/bpxqp9.html
749: *
750: * and maps the characters of EBCDIC code page 1047 (the code used for
751: * Unix-derived software on IBM's 390 systems) to the corresponding
752: * characters from ISO 8859-1.
753: *
754: * If this table is used instead of the above one, some of the special
755: * cases for the NEL character can be taken out of the code.
756: */
1.1 deraadt 757:
1.8 tedu 758: private unsigned char ebcdic_1047_to_8859[] = {
759: 0x00,0x01,0x02,0x03,0x9C,0x09,0x86,0x7F,0x97,0x8D,0x8E,0x0B,0x0C,0x0D,0x0E,0x0F,
760: 0x10,0x11,0x12,0x13,0x9D,0x0A,0x08,0x87,0x18,0x19,0x92,0x8F,0x1C,0x1D,0x1E,0x1F,
761: 0x80,0x81,0x82,0x83,0x84,0x85,0x17,0x1B,0x88,0x89,0x8A,0x8B,0x8C,0x05,0x06,0x07,
762: 0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9A,0x9B,0x14,0x15,0x9E,0x1A,
763: 0x20,0xA0,0xE2,0xE4,0xE0,0xE1,0xE3,0xE5,0xE7,0xF1,0xA2,0x2E,0x3C,0x28,0x2B,0x7C,
764: 0x26,0xE9,0xEA,0xEB,0xE8,0xED,0xEE,0xEF,0xEC,0xDF,0x21,0x24,0x2A,0x29,0x3B,0x5E,
765: 0x2D,0x2F,0xC2,0xC4,0xC0,0xC1,0xC3,0xC5,0xC7,0xD1,0xA6,0x2C,0x25,0x5F,0x3E,0x3F,
766: 0xF8,0xC9,0xCA,0xCB,0xC8,0xCD,0xCE,0xCF,0xCC,0x60,0x3A,0x23,0x40,0x27,0x3D,0x22,
767: 0xD8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xAB,0xBB,0xF0,0xFD,0xFE,0xB1,
768: 0xB0,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0xAA,0xBA,0xE6,0xB8,0xC6,0xA4,
769: 0xB5,0x7E,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7A,0xA1,0xBF,0xD0,0x5B,0xDE,0xAE,
770: 0xAC,0xA3,0xA5,0xB7,0xA9,0xA7,0xB6,0xBC,0xBD,0xBE,0xDD,0xA8,0xAF,0x5D,0xB4,0xD7,
771: 0x7B,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xAD,0xF4,0xF6,0xF2,0xF3,0xF5,
772: 0x7D,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50,0x51,0x52,0xB9,0xFB,0xFC,0xF9,0xFA,0xFF,
773: 0x5C,0xF7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0xB2,0xD4,0xD6,0xD2,0xD3,0xD5,
774: 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xB3,0xDB,0xDC,0xD9,0xDA,0x9F
775: };
776: #endif
777:
778: /*
779: * Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII.
780: */
781: private void
782: from_ebcdic(const unsigned char *buf, size_t nbytes, unsigned char *out)
783: {
1.9 chl 784: size_t i;
1.8 tedu 785:
786: for (i = 0; i < nbytes; i++) {
787: out[i] = ebcdic_to_ascii[buf[i]];
788: }
789: }
![[BACK]](/icons/back.gif){kind=icon}
Return to ascmagic.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [local] / src / usr.bin / file