1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
|
/*
* This file is part of mpv.
*
* Based on code taken from libass (ISC license), which was originally part
* of MPlayer (GPL).
* Copyright (C) 2006 Evgeniy Stepanov <eugeni.stepanov@gmail.com>
*
* mpv is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* mpv is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with mpv. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdlib.h>
#include <errno.h>
#include <strings.h>
#include <assert.h>
#include "config.h"
#include "common/msg.h"
#if HAVE_UCHARDET
#include <uchardet.h>
#endif
#if HAVE_ICONV
#include <iconv.h>
#endif
#include "charset_conv.h"
bool mp_charset_is_utf8(const char *user_cp)
{
return user_cp && (strcasecmp(user_cp, "utf8") == 0 ||
strcasecmp(user_cp, "utf-8") == 0);
}
bool mp_charset_is_utf16(const char *user_cp)
{
bstr s = bstr0(user_cp);
return bstr_case_startswith(s, bstr0("utf16")) ||
bstr_case_startswith(s, bstr0("utf-16"));
}
static const char *const utf_bom[3] = {"\xEF\xBB\xBF", "\xFF\xFE", "\xFE\xFF"};
static const char *const utf_enc[3] = {"utf-8", "utf-16le", "utf-16be"};
static const char *ms_bom_guess(bstr buf)
{
for (int n = 0; n < 3; n++) {
if (bstr_startswith0(buf, utf_bom[n]))
return utf_enc[n];
}
return NULL;
}
#if HAVE_UCHARDET
static const char *mp_uchardet(void *talloc_ctx, struct mp_log *log, bstr buf)
{
uchardet_t det = uchardet_new();
if (!det)
return NULL;
if (uchardet_handle_data(det, buf.start, buf.len) != 0) {
uchardet_delete(det);
return NULL;
}
uchardet_data_end(det);
char *res = talloc_strdup(talloc_ctx, uchardet_get_charset(det));
if (res && !res[0])
res = NULL;
if (res) {
mp_verbose(log, "libuchardet detected charset as %s\n", res);
iconv_t icdsc = iconv_open("UTF-8", res);
if (icdsc == (iconv_t)(-1)) {
mp_warn(log, "Charset '%s' not supported by iconv.\n", res);
res = NULL;
} else {
iconv_close(icdsc);
}
}
uchardet_delete(det);
return res;
}
#endif
// Runs charset auto-detection on the input buffer, and returns the result.
// If auto-detection fails, NULL is returned.
// If user_cp doesn't refer to any known auto-detection (for example because
// it's a real iconv codepage), user_cp is returned without even looking at
// the buf data.
// The return value may (but doesn't have to) be allocated under talloc_ctx.
const char *mp_charset_guess(void *talloc_ctx, struct mp_log *log, bstr buf,
const char *user_cp, int flags)
{
if (user_cp[0] == '+') {
mp_verbose(log, "Forcing charset '%s'.\n", user_cp + 1);
return user_cp + 1;
}
const char *bom_cp = ms_bom_guess(buf);
if (bom_cp) {
mp_verbose(log, "Data has a BOM, assuming %s as charset.\n", bom_cp);
return bom_cp;
}
int r = bstr_validate_utf8(buf);
if (r >= 0 || (r > -8 && (flags & MP_ICONV_ALLOW_CUTOFF))) {
if (strcmp(user_cp, "auto") != 0 && !mp_charset_is_utf8(user_cp))
mp_verbose(log, "Data looks like UTF-8, ignoring user-provided charset.\n");
return "utf-8";
}
const char *res = NULL;
if (strcasecmp(user_cp, "auto") == 0) {
#if HAVE_UCHARDET
res = mp_uchardet(talloc_ctx, log, buf);
#endif
if (!res) {
mp_verbose(log, "Charset auto-detection failed.\n");
res = "UTF-8-BROKEN";
}
} else {
res = user_cp;
}
mp_verbose(log, "Using charset '%s'.\n", res);
return res;
}
// Use iconv to convert buf to UTF-8.
// Returns buf.start==NULL on error. Returns buf if cp is NULL, or if there is
// obviously no conversion required (e.g. if cp is "UTF-8").
// Returns a newly allocated buffer if conversion is done and succeeds. The
// buffer will be terminated with 0 for convenience (the terminating 0 is not
// included in the returned length).
// Free the returned buffer with talloc_free().
// buf: input data
// cp: iconv codepage (or NULL)
// flags: combination of MP_ICONV_* flags
// returns: buf (no conversion), .start==NULL (error), or allocated buffer
bstr mp_iconv_to_utf8(struct mp_log *log, bstr buf, const char *cp, int flags)
{
#if HAVE_ICONV
if (!buf.len)
return buf;
if (!cp || !cp[0] || mp_charset_is_utf8(cp))
return buf;
if (strcasecmp(cp, "ASCII") == 0)
return buf;
if (strcasecmp(cp, "UTF-8-BROKEN") == 0)
return bstr_sanitize_utf8_latin1(NULL, buf);
// Force CP949 over EUC-KR since iconv distinguishes them and
// EUC-KR causes error on CP949 encoded data
if (strcasecmp(cp, "EUC-KR") == 0)
cp = "CP949";
iconv_t icdsc;
if ((icdsc = iconv_open("UTF-8", cp)) == (iconv_t) (-1)) {
if (flags & MP_ICONV_VERBOSE)
mp_err(log, "Error opening iconv with codepage '%s'\n", cp);
goto failure;
}
size_t size = buf.len;
size_t osize = size;
size_t ileft = size;
size_t oleft = size - 1;
char *outbuf = talloc_size(NULL, osize);
char *ip = buf.start;
char *op = outbuf;
while (1) {
int clear = 0;
size_t rc;
if (ileft)
rc = iconv(icdsc, &ip, &ileft, &op, &oleft);
else {
clear = 1; // clear the conversion state and leave
rc = iconv(icdsc, NULL, NULL, &op, &oleft);
}
if (rc == (size_t) (-1)) {
if (errno == E2BIG) {
size_t offset = op - outbuf;
outbuf = talloc_realloc_size(NULL, outbuf, osize + size);
op = outbuf + offset;
osize += size;
oleft += size;
} else {
if (errno == EINVAL && (flags & MP_ICONV_ALLOW_CUTOFF)) {
// This is intended for cases where the input buffer is cut
// at a random byte position. If this happens in the middle
// of the buffer, it should still be an error. We say it's
// fine if the error is within 10 bytes of the end.
if (ileft <= 10)
break;
}
if (flags & MP_ICONV_VERBOSE) {
mp_err(log, "Error recoding text with codepage '%s'\n", cp);
}
talloc_free(outbuf);
iconv_close(icdsc);
goto failure;
}
} else if (clear)
break;
}
iconv_close(icdsc);
outbuf[osize - oleft - 1] = 0;
return (bstr){outbuf, osize - oleft - 1};
failure:
#endif
if (flags & MP_NO_LATIN1_FALLBACK) {
return buf;
} else {
return bstr_sanitize_utf8_latin1(NULL, buf);
}
}
|
