528 lines
10 KiB
C
528 lines
10 KiB
C
/* $Vendor-Id: preconv.c,v 1.5 2011/07/24 18:15:14 kristaps Exp $ */
|
|
/*
|
|
* Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
|
|
*
|
|
* Permission to use, copy, modify, and distribute this software for any
|
|
* purpose with or without fee is hereby granted, provided that the above
|
|
* copyright notice and this permission notice appear in all copies.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
|
*/
|
|
#ifdef HAVE_CONFIG_H
|
|
#include "config.h"
|
|
#endif
|
|
|
|
#ifdef HAVE_MMAP
|
|
#include <sys/stat.h>
|
|
#include <sys/mman.h>
|
|
#endif
|
|
|
|
#include <assert.h>
|
|
#include <fcntl.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
|
|
/*
|
|
* The read_whole_file() and resize_buf() functions are copied from
|
|
* read.c, including all dependency code (MAP_FILE, etc.).
|
|
*/
|
|
|
|
#ifndef MAP_FILE
|
|
#define MAP_FILE 0
|
|
#endif
|
|
|
|
enum enc {
|
|
ENC_UTF_8, /* UTF-8 */
|
|
ENC_US_ASCII, /* US-ASCII */
|
|
ENC_LATIN_1, /* Latin-1 */
|
|
ENC__MAX
|
|
};
|
|
|
|
struct buf {
|
|
char *buf; /* binary input buffer */
|
|
size_t sz; /* size of binary buffer */
|
|
size_t offs; /* starting buffer offset */
|
|
};
|
|
|
|
struct encode {
|
|
const char *name;
|
|
int (*conv)(const struct buf *);
|
|
};
|
|
|
|
static int cue_enc(const struct buf *, size_t *, enum enc *);
|
|
static int conv_latin_1(const struct buf *);
|
|
static int conv_us_ascii(const struct buf *);
|
|
static int conv_utf_8(const struct buf *);
|
|
static int read_whole_file(const char *, int,
|
|
struct buf *, int *);
|
|
static void resize_buf(struct buf *, size_t);
|
|
static void usage(void);
|
|
|
|
static const struct encode encs[ENC__MAX] = {
|
|
{ "utf-8", conv_utf_8 }, /* ENC_UTF_8 */
|
|
{ "us-ascii", conv_us_ascii }, /* ENC_US_ASCII */
|
|
{ "latin-1", conv_latin_1 }, /* ENC_LATIN_1 */
|
|
};
|
|
|
|
static const char *progname;
|
|
|
|
static void
|
|
usage(void)
|
|
{
|
|
|
|
fprintf(stderr, "usage: %s "
|
|
"[-D enc] "
|
|
"[-e ENC] "
|
|
"[file]\n", progname);
|
|
}
|
|
|
|
static int
|
|
conv_latin_1(const struct buf *b)
|
|
{
|
|
size_t i;
|
|
unsigned char cu;
|
|
const char *cp;
|
|
|
|
cp = b->buf + (int)b->offs;
|
|
|
|
/*
|
|
* Latin-1 falls into the first 256 code-points of Unicode, so
|
|
* there's no need for any sort of translation. Just make the
|
|
* 8-bit characters use the Unicode escape.
|
|
* Note that binary values 128 < v < 160 are passed through
|
|
* unmodified to mandoc.
|
|
*/
|
|
|
|
for (i = b->offs; i < b->sz; i++) {
|
|
cu = (unsigned char)*cp++;
|
|
cu < 160U ? putchar(cu) : printf("\\[u%.4X]", cu);
|
|
}
|
|
|
|
return(1);
|
|
}
|
|
|
|
static int
|
|
conv_us_ascii(const struct buf *b)
|
|
{
|
|
|
|
/*
|
|
* US-ASCII has no conversion since it falls into the first 128
|
|
* bytes of Unicode.
|
|
*/
|
|
|
|
fwrite(b->buf, 1, b->sz, stdout);
|
|
return(1);
|
|
}
|
|
|
|
static int
|
|
conv_utf_8(const struct buf *b)
|
|
{
|
|
int state, be;
|
|
unsigned int accum;
|
|
size_t i;
|
|
unsigned char cu;
|
|
const char *cp;
|
|
const long one = 1L;
|
|
|
|
cp = b->buf + (int)b->offs;
|
|
state = 0;
|
|
accum = 0U;
|
|
be = 0;
|
|
|
|
/* Quick test for big-endian value. */
|
|
|
|
if ( ! (*((const char *)(&one))))
|
|
be = 1;
|
|
|
|
for (i = b->offs; i < b->sz; i++) {
|
|
cu = (unsigned char)*cp++;
|
|
if (state) {
|
|
if ( ! (cu & 128) || (cu & 64)) {
|
|
/* Bad sequence header. */
|
|
return(0);
|
|
}
|
|
|
|
/* Accept only legitimate bit patterns. */
|
|
|
|
if (cu > 191 || cu < 128) {
|
|
/* Bad in-sequence bits. */
|
|
return(0);
|
|
}
|
|
|
|
accum |= (cu & 63) << --state * 6;
|
|
|
|
/*
|
|
* Accum is held in little-endian order as
|
|
* stipulated by the UTF-8 sequence coding. We
|
|
* need to convert to a native big-endian if our
|
|
* architecture requires it.
|
|
*/
|
|
|
|
if (0 == state && be)
|
|
accum = (accum >> 24) |
|
|
((accum << 8) & 0x00FF0000) |
|
|
((accum >> 8) & 0x0000FF00) |
|
|
(accum << 24);
|
|
|
|
if (0 == state) {
|
|
accum < 128U ? putchar(accum) :
|
|
printf("\\[u%.4X]", accum);
|
|
accum = 0U;
|
|
}
|
|
} else if (cu & (1 << 7)) {
|
|
/*
|
|
* Entering a UTF-8 state: if we encounter a
|
|
* UTF-8 bitmask, calculate the expected UTF-8
|
|
* state from it.
|
|
*/
|
|
for (state = 0; state < 7; state++)
|
|
if ( ! (cu & (1 << (7 - state))))
|
|
break;
|
|
|
|
/* Accept only legitimate bit patterns. */
|
|
|
|
switch (state) {
|
|
case (4):
|
|
if (cu <= 244 && cu >= 240) {
|
|
accum = (cu & 7) << 18;
|
|
break;
|
|
}
|
|
/* Bad 4-sequence start bits. */
|
|
return(0);
|
|
case (3):
|
|
if (cu <= 239 && cu >= 224) {
|
|
accum = (cu & 15) << 12;
|
|
break;
|
|
}
|
|
/* Bad 3-sequence start bits. */
|
|
return(0);
|
|
case (2):
|
|
if (cu <= 223 && cu >= 194) {
|
|
accum = (cu & 31) << 6;
|
|
break;
|
|
}
|
|
/* Bad 2-sequence start bits. */
|
|
return(0);
|
|
default:
|
|
/* Bad sequence bit mask. */
|
|
return(0);
|
|
}
|
|
state--;
|
|
} else
|
|
putchar(cu);
|
|
}
|
|
|
|
if (0 != state) {
|
|
/* Bad trailing bits. */
|
|
return(0);
|
|
}
|
|
|
|
return(1);
|
|
}
|
|
|
|
static void
|
|
resize_buf(struct buf *buf, size_t initial)
|
|
{
|
|
|
|
buf->sz = buf->sz > initial / 2 ?
|
|
2 * buf->sz : initial;
|
|
|
|
buf->buf = realloc(buf->buf, buf->sz);
|
|
if (NULL == buf->buf) {
|
|
perror(NULL);
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
static int
|
|
read_whole_file(const char *f, int fd,
|
|
struct buf *fb, int *with_mmap)
|
|
{
|
|
size_t off;
|
|
ssize_t ssz;
|
|
|
|
#ifdef HAVE_MMAP
|
|
struct stat st;
|
|
if (-1 == fstat(fd, &st)) {
|
|
perror(f);
|
|
return(0);
|
|
}
|
|
|
|
/*
|
|
* If we're a regular file, try just reading in the whole entry
|
|
* via mmap(). This is faster than reading it into blocks, and
|
|
* since each file is only a few bytes to begin with, I'm not
|
|
* concerned that this is going to tank any machines.
|
|
*/
|
|
|
|
if (S_ISREG(st.st_mode) && st.st_size >= (1U << 31)) {
|
|
fprintf(stderr, "%s: input too large\n", f);
|
|
return(0);
|
|
}
|
|
|
|
if (S_ISREG(st.st_mode)) {
|
|
*with_mmap = 1;
|
|
fb->sz = (size_t)st.st_size;
|
|
fb->buf = mmap(NULL, fb->sz, PROT_READ,
|
|
MAP_FILE|MAP_SHARED, fd, 0);
|
|
if (fb->buf != MAP_FAILED)
|
|
return(1);
|
|
}
|
|
#endif
|
|
|
|
/*
|
|
* If this isn't a regular file (like, say, stdin), then we must
|
|
* go the old way and just read things in bit by bit.
|
|
*/
|
|
|
|
*with_mmap = 0;
|
|
off = 0;
|
|
fb->sz = 0;
|
|
fb->buf = NULL;
|
|
for (;;) {
|
|
if (off == fb->sz && fb->sz == (1U << 31)) {
|
|
fprintf(stderr, "%s: input too large\n", f);
|
|
break;
|
|
}
|
|
|
|
if (off == fb->sz)
|
|
resize_buf(fb, 65536);
|
|
|
|
ssz = read(fd, fb->buf + (int)off, fb->sz - off);
|
|
if (ssz == 0) {
|
|
fb->sz = off;
|
|
return(1);
|
|
}
|
|
if (ssz == -1) {
|
|
perror(f);
|
|
break;
|
|
}
|
|
off += (size_t)ssz;
|
|
}
|
|
|
|
free(fb->buf);
|
|
fb->buf = NULL;
|
|
return(0);
|
|
}
|
|
|
|
static int
|
|
cue_enc(const struct buf *b, size_t *offs, enum enc *enc)
|
|
{
|
|
const char *ln, *eoln, *eoph;
|
|
size_t sz, phsz, nsz;
|
|
int i;
|
|
|
|
ln = b->buf + (int)*offs;
|
|
sz = b->sz - *offs;
|
|
|
|
/* Look for the end-of-line. */
|
|
|
|
if (NULL == (eoln = memchr(ln, '\n', sz)))
|
|
return(-1);
|
|
|
|
/* Set next-line marker. */
|
|
|
|
*offs = (size_t)((eoln + 1) - b->buf);
|
|
|
|
/* Check if we have the correct header/trailer. */
|
|
|
|
if ((sz = (size_t)(eoln - ln)) < 10 ||
|
|
memcmp(ln, ".\\\" -*-", 7) ||
|
|
memcmp(eoln - 3, "-*-", 3))
|
|
return(0);
|
|
|
|
/* Move after the header and adjust for the trailer. */
|
|
|
|
ln += 7;
|
|
sz -= 10;
|
|
|
|
while (sz > 0) {
|
|
while (sz > 0 && ' ' == *ln) {
|
|
ln++;
|
|
sz--;
|
|
}
|
|
if (0 == sz)
|
|
break;
|
|
|
|
/* Find the end-of-phrase marker (or eoln). */
|
|
|
|
if (NULL == (eoph = memchr(ln, ';', sz)))
|
|
eoph = eoln - 3;
|
|
else
|
|
eoph++;
|
|
|
|
/* Only account for the "coding" phrase. */
|
|
|
|
if ((phsz = (size_t)(eoph - ln)) < 7 ||
|
|
strncasecmp(ln, "coding:", 7)) {
|
|
sz -= phsz;
|
|
ln += phsz;
|
|
continue;
|
|
}
|
|
|
|
sz -= 7;
|
|
ln += 7;
|
|
|
|
while (sz > 0 && ' ' == *ln) {
|
|
ln++;
|
|
sz--;
|
|
}
|
|
if (0 == sz)
|
|
break;
|
|
|
|
/* Check us against known encodings. */
|
|
|
|
for (i = 0; i < (int)ENC__MAX; i++) {
|
|
nsz = strlen(encs[i].name);
|
|
if (phsz < nsz)
|
|
continue;
|
|
if (strncasecmp(ln, encs[i].name, nsz))
|
|
continue;
|
|
|
|
*enc = (enum enc)i;
|
|
return(1);
|
|
}
|
|
|
|
/* Unknown encoding. */
|
|
|
|
*enc = ENC__MAX;
|
|
return(1);
|
|
}
|
|
|
|
return(0);
|
|
}
|
|
|
|
int
|
|
main(int argc, char *argv[])
|
|
{
|
|
int i, ch, map, fd, rc;
|
|
struct buf b;
|
|
const char *fn;
|
|
enum enc enc, def;
|
|
unsigned char bom[3] = { 0xEF, 0xBB, 0xBF };
|
|
size_t offs;
|
|
extern int optind;
|
|
extern char *optarg;
|
|
|
|
progname = strrchr(argv[0], '/');
|
|
if (progname == NULL)
|
|
progname = argv[0];
|
|
else
|
|
++progname;
|
|
|
|
fn = "<stdin>";
|
|
fd = STDIN_FILENO;
|
|
rc = EXIT_FAILURE;
|
|
enc = def = ENC__MAX;
|
|
map = 0;
|
|
|
|
memset(&b, 0, sizeof(struct buf));
|
|
|
|
while (-1 != (ch = getopt(argc, argv, "D:e:rdvh")))
|
|
switch (ch) {
|
|
case ('D'):
|
|
/* FALLTHROUGH */
|
|
case ('e'):
|
|
for (i = 0; i < (int)ENC__MAX; i++) {
|
|
if (strcasecmp(optarg, encs[i].name))
|
|
continue;
|
|
break;
|
|
}
|
|
if (i < (int)ENC__MAX) {
|
|
if ('D' == ch)
|
|
def = (enum enc)i;
|
|
else
|
|
enc = (enum enc)i;
|
|
break;
|
|
}
|
|
|
|
fprintf(stderr, "%s: Bad encoding\n", optarg);
|
|
return(EXIT_FAILURE);
|
|
case ('r'):
|
|
/* FALLTHROUGH */
|
|
case ('d'):
|
|
/* FALLTHROUGH */
|
|
case ('v'):
|
|
/* Compatibility with GNU preconv. */
|
|
break;
|
|
case ('h'):
|
|
/* Compatibility with GNU preconv. */
|
|
/* FALLTHROUGH */
|
|
default:
|
|
usage();
|
|
return(EXIT_FAILURE);
|
|
}
|
|
|
|
argc -= optind;
|
|
argv += optind;
|
|
|
|
/*
|
|
* Open and read the first argument on the command-line.
|
|
* If we don't have one, we default to stdin.
|
|
*/
|
|
|
|
if (argc > 0) {
|
|
fn = *argv;
|
|
fd = open(fn, O_RDONLY, 0);
|
|
if (-1 == fd) {
|
|
perror(fn);
|
|
return(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
if ( ! read_whole_file(fn, fd, &b, &map))
|
|
goto out;
|
|
|
|
/* Try to read the UTF-8 BOM. */
|
|
|
|
if (ENC__MAX == enc)
|
|
if (b.sz > 3 && 0 == memcmp(b.buf, bom, 3)) {
|
|
b.offs = 3;
|
|
enc = ENC_UTF_8;
|
|
}
|
|
|
|
/* Try reading from the "-*-" cue. */
|
|
|
|
if (ENC__MAX == enc) {
|
|
offs = b.offs;
|
|
ch = cue_enc(&b, &offs, &enc);
|
|
if (0 == ch)
|
|
ch = cue_enc(&b, &offs, &enc);
|
|
}
|
|
|
|
/*
|
|
* No encoding has been detected.
|
|
* Thus, we either fall into our default encoder, if specified,
|
|
* or use Latin-1 if all else fails.
|
|
*/
|
|
|
|
if (ENC__MAX == enc)
|
|
enc = ENC__MAX == def ? ENC_LATIN_1 : def;
|
|
|
|
if ( ! (*encs[(int)enc].conv)(&b)) {
|
|
fprintf(stderr, "%s: Bad encoding\n", fn);
|
|
goto out;
|
|
}
|
|
|
|
rc = EXIT_SUCCESS;
|
|
out:
|
|
#ifdef HAVE_MMAP
|
|
if (map)
|
|
munmap(b.buf, b.sz);
|
|
else
|
|
#endif
|
|
free(b.buf);
|
|
|
|
if (fd > STDIN_FILENO)
|
|
close(fd);
|
|
|
|
return(rc);
|
|
}
|