| Re: [AD] Allegro unable to deal with non ascii filenames |
[ Thread Index |
Date Index
| More lists.liballeg.org/allegro-developers Archives
]
On Mon, 2006-05-29 at 17:01 -0700, Chris wrote:
> The patch doesn't seem to do much. If it finds utf8/utf-8 in LC_ALL, it will
> set UTF-8 file encoding, or else it'll use the current. By default, UTF-8 is
> current, so either way is using the same. It should probably use U_ASCII
> instead of U_CURRENT.
Yes, I'm not sure about that. In Grzegorz's case, I assme U_CURRENT will
work better for him - his files use ISO-8859-1, so U_ASCII will truncate
them to 7-bit. This was the original problem. So U_CURRENT seems the
best default - it will always work, unless the libc encoding is
different from Alegro's current encoding.
And as I understand the docs, U_ASCII_CP is only useful if the user
provides their own mapping tables - so we can't use it for auto
detection. What we might do is, include some common 8-bit tables with
Allegro (e.g. ISO-8859-*). Also, maybe we should have an U_ASCII16_CP,
so users could also use 16-bit encodings?
But for Allegro 4.2.1, both of those ideas go too far. A completely
different route for 4.3.0 might be to use libc's wide string support
throughout the library, and be totally encoding agnostic. One problem
with that approach would be font output - that's the one area that would
have to know the encoding libc is using.
> As well, if LC_ALL doesn't exist, you'll get a crash
> (getenv returns NULL for non-existant vars). On my particular machine, LC_*
> vars don't exist. You need to use LANG.
Yes, I couldn't find information if LC_ALL or LANG is the right one,
here both exist and have the same value.
> Here's my (completely untested!) patch which fixes these issues, as well as
> adds the _al_open macro for newer Windows to use _wopen (while also setting
> them to use U_UNICODE filenames).
Very nice. There only is one parenthesis too much here:
+ strstr(locale, "UTF8"))) {
I attached a new patch with only the ) removed, to make it easy for
others to test it (especially Windows, OSX and unix without UTF8).
--
Elias Pschernig
Index: src/win/wsystem.c
===================================================================
--- src/win/wsystem.c (revision 5814)
+++ src/win/wsystem.c (working copy)
@@ -191,6 +191,10 @@
if (init_directx_window() != 0)
goto Error;
+ if (IS_OLD_WINDOWS) {
+ set_filename_encoding(U_UNICODE);
+ }
+
return 0;
Error:
Index: src/win/wfile.c
===================================================================
--- src/win/wfile.c (revision 5814)
+++ src/win/wfile.c (working copy)
@@ -51,11 +51,19 @@
struct _stat s;
char tmp[1024];
- if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE,
- sizeof(tmp)), &s) != 0) {
- *allegro_errno = errno;
- return 0;
+ if (IS_OLD_WINDOWS) {
+ if (_stat(uconvert(filename, U_CURRENT, tmp, U_ASCII, sizeof(tmp)), &s) != 0) {
+ *allegro_errno = errno;
+ return 0;
+ }
}
+ else {
+ if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE,
+ sizeof(tmp)), &s) != 0) {
+ *allegro_errno = errno;
+ return 0;
+ }
+ }
return s.st_size;
}
@@ -70,11 +78,18 @@
struct _stat s;
char tmp[1024];
- if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE,
- sizeof(tmp)), &s) != 0) {
- *allegro_errno = errno;
- return 0;
+ if (IS_OLD_WINDOWS) {
+ if (_stat(uconvert(filename, U_CURRENT, tmp, U_ASCII, sizeof(tmp)), &s) != 0) {
+ *allegro_errno = errno;
+ return 0;
+ }
}
+ else {
+ if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE, sizeof(tmp)), &s) != 0) {
+ *allegro_errno = errno;
+ return 0;
+ }
+ }
return s.st_mtime;
}
@@ -84,7 +99,10 @@
/* structure for use by the directory scanning routines */
struct FF_DATA
{
- struct _wfinddata_t data;
+ union {
+ struct _finddata_t a;
+ struct _wfinddata_t w;
+ } data;
long handle;
int attrib;
};
@@ -98,12 +116,22 @@
{
struct FF_DATA *ff_data = (struct FF_DATA *) info->ff_data;
- info->attrib = ff_data->data.attrib;
- info->time = ff_data->data.time_write;
- info->size = ff_data->data.size;
+ if (IS_OLD_WINDOWS) {
+ info->attrib = ff_data->data.a.attrib;
+ info->time = ff_data->data.a.time_write;
+ info->size = ff_data->data.a.size;
- do_uconvert((const char*)ff_data->data.name, U_UNICODE, info->name, U_CURRENT,
- sizeof(info->name));
+ do_uconvert(ff_data->data.a.name, U_ASCII, info->name, U_CURRENT,
+ sizeof(info->name));
+ }
+ else {
+ info->attrib = ff_data->data.w.attrib;
+ info->time = ff_data->data.w.time_write;
+ info->size = ff_data->data.w.size;
+
+ do_uconvert((const char*)ff_data->data.w.name, U_UNICODE, info->name,
+ U_CURRENT, sizeof(info->name));
+ }
}
@@ -144,23 +172,47 @@
/* start the search */
errno = *allegro_errno = 0;
- ff_data->handle = _wfindfirst((wchar_t*)uconvert(pattern, U_CURRENT, tmp,
- U_UNICODE, sizeof(tmp)), &ff_data->data);
+ if (IS_OLD_WINDOWS) {
+ ff_data->handle = _findfirst(uconvert(pattern, U_CURRENT, tmp,
+ U_ASCII, sizeof(tmp)),
+ &ff_data->data.a);
- if (ff_data->handle < 0) {
- *allegro_errno = errno;
- _AL_FREE(ff_data);
- info->ff_data = NULL;
- return -1;
+ if (ff_data->handle < 0) {
+ *allegro_errno = errno;
+ _AL_FREE(ff_data);
+ info->ff_data = NULL;
+ return -1;
+ }
+
+ if (ff_data->data.a.attrib & ~ff_data->attrib) {
+ if (al_findnext(info) != 0) {
+ al_findclose(info);
+ return -1;
+ }
+ else
+ return 0;
+ }
}
+ else {
+ ff_data->handle = _wfindfirst((wchar_t*)uconvert(pattern, U_CURRENT, tmp,
+ U_UNICODE, sizeof(tmp)),
+ &ff_data->data.w);
- if (ff_data->data.attrib & ~ff_data->attrib) {
- if (al_findnext(info) != 0) {
- al_findclose(info);
+ if (ff_data->handle < 0) {
+ *allegro_errno = errno;
+ _AL_FREE(ff_data);
+ info->ff_data = NULL;
return -1;
}
- else
- return 0;
+
+ if (ff_data->data.w.attrib & ~ff_data->attrib) {
+ if (al_findnext(info) != 0) {
+ al_findclose(info);
+ return -1;
+ }
+ else
+ return 0;
+ }
}
fill_ffblk(info);
@@ -176,14 +228,23 @@
{
struct FF_DATA *ff_data = (struct FF_DATA *) info->ff_data;
- do {
- if (_wfindnext(ff_data->handle, &ff_data->data) != 0) {
- *allegro_errno = errno;
- return -1;
- }
+ if (IS_OLD_WINDOWS) {
+ do {
+ if (_findnext(ff_data->handle, &ff_data->data.a) != 0) {
+ *allegro_errno = errno;
+ return -1;
+ }
+ } while (ff_data->data.a.attrib & ~ff_data->attrib);
+ }
+ else {
+ do {
+ if (_wfindnext(ff_data->handle, &ff_data->data.w) != 0) {
+ *allegro_errno = errno;
+ return -1;
+ }
+ } while (ff_data->data.w.attrib & ~ff_data->attrib);
+ }
- } while (ff_data->data.attrib & ~ff_data->attrib);
-
fill_ffblk(info);
return 0;
}
@@ -233,10 +294,18 @@
{
wchar_t tmp[1024];
- if (_wgetdcwd(drive+1, tmp, sizeof(tmp)/sizeof(tmp[0])))
- do_uconvert((const char*)tmp, U_UNICODE, buf, U_CURRENT, size);
- else
- usetc(buf, 0);
+ if (IS_OLD_WINDOWS) {
+ if (_getdcwd(drive+1, tmp, sizeof(tmp)))
+ do_uconvert(tmp, U_ASCII, buf, U_CURRENT, size);
+ else
+ usetc(buf, 0);
+ }
+ else {
+ if (_wgetdcwd(drive+1, (wchar_t*)tmp, sizeof(tmp)/sizeof(wchar_t*)))
+ do_uconvert(tmp, U_UNICODE, buf, U_CURRENT, size);
+ else
+ usetc(buf, 0);
+ }
}
@@ -245,8 +314,25 @@
*/
uint64_t al_ffblk_get_size(struct al_ffblk *info)
{
+ struct FF_DATA *ff_data;
+
ASSERT(info);
- struct FF_DATA *ff_data = (struct FF_DATA *) info->ff_data;
+ ff_data = (struct FF_DATA *) info->ff_data;
- return ff_data->data.size;
+ if (IS_OLD_WINDOWS) {
+ return ff_data->data.a.size;
+ }
+ else {
+ return ff_data->data.w.size;
+ }
}
+
+int _alwin_open(const char *filename, int mode, int perm)
+{
+ if (IS_OLD_WINDOWS) {
+ return open(filename, mode, perm);
+ }
+ else {
+ return _wopen((wchar_t*)filename, mode, perm);
+ }
+}
Index: src/linux/lsystem.c
===================================================================
--- src/linux/lsystem.c (revision 5814)
+++ src/linux/lsystem.c (working copy)
@@ -164,6 +164,8 @@
_unix_read_os_type();
if (os_type != OSTYPE_LINUX) return -1; /* FWIW */
+ _unix_guess_file_encoding();
+
/* This is the only bit that needs root privileges. First
* we attempt to set our euid to 0, in case this is the
* second time we've been called. */
Index: src/unix/ufile.c
===================================================================
--- src/unix/ufile.c (revision 5814)
+++ src/unix/ufile.c (working copy)
@@ -57,7 +57,9 @@
#endif
#endif
+#define PREFIX_I "al-unix INFO: "
+
/* _al_file_isok:
* Helper function to check if it is safe to access a file on a floppy
* drive.
@@ -500,3 +502,25 @@
ASSERT(ff_data);
return ff_data->size;
}
+
+
+
+void _unix_guess_file_encoding(void)
+{
+ char const *encoding = "unknown";
+ char *locale = getenv("LC_ALL");
+ if (!locale)
+ locale = getenv("LANG");
+
+ if (locale) {
+ if (strstr(locale, "utf8") ||
+ strstr(locale, "UTF-8") ||
+ strstr(locale, "utf-8") ||
+ strstr(locale, "UTF8")) {
+ set_file_encoding(U_UTF8);
+ encoding = "UTF8";
+ }
+ }
+
+ TRACE(PREFIX_I "Assumed libc encoding is %s.\n", encoding);
+}
Index: src/qnx/qsystem.c
===================================================================
--- src/qnx/qsystem.c (revision 5814)
+++ src/qnx/qsystem.c (working copy)
@@ -337,7 +337,9 @@
struct sched_param sparam;
int spolicy;
char tmp[WINDOW_TITLE_SIZE];
-
+
+ _unix_guess_file_encoding();
+
/* install emergency-exit signal handlers */
old_sig_abrt = signal(SIGABRT, qnx_signal_handler);
old_sig_fpe = signal(SIGFPE, qnx_signal_handler);
Index: src/macosx/system.m
===================================================================
--- src/macosx/system.m (revision 5814)
+++ src/macosx/system.m (working copy)
@@ -410,7 +410,9 @@
if (!osx_bootstrap_ok()) {
return -1;
}
-
+
+ _unix_guess_file_encoding();
+
/* Install emergency-exit signal handlers */
old_sig_abrt = signal(SIGABRT, osx_signal_handler);
old_sig_fpe = signal(SIGFPE, osx_signal_handler);
Index: src/file.c
===================================================================
--- src/file.c (revision 5814)
+++ src/file.c (working copy)
@@ -74,6 +74,7 @@
static PACKFILE *pack_fopen_special_file(AL_CONST char *filename, AL_CONST char *mode);
+static int file_encoding = U_ASCII;
#define FA_DAT_FLAGS (FA_RDONLY | FA_ARCH)
@@ -671,6 +672,26 @@
***************************************************/
+/* set_file_encoding:
+ * Sets the encoding to use for filenames. By default, UTF8 is assumed.
+ */
+void set_file_encoding(int encoding)
+{
+ file_encoding = encoding;
+}
+
+
+
+/* get_file_encoding:
+ * Returns the encoding currently assumed for filenames.
+ */
+int get_file_encoding(void)
+{
+ return file_encoding ;
+}
+
+
+
/* file_exists:
* Checks whether a file matching the given name and attributes exists,
* returning non zero if it does. The file attribute may contain any of
@@ -788,7 +809,7 @@
if (!_al_file_isok(filename))
return -1;
- if (unlink(uconvert_toascii(filename, tmp)) != 0) {
+ if (unlink(uconvert_tofilename(filename, tmp)) != 0) {
*allegro_errno = errno;
return -1;
}
@@ -1197,7 +1218,7 @@
/* try any extra environment variable that the parameters say to use */
if (envvar) {
- s = getenv(uconvert_toascii(envvar, tmp));
+ s = getenv(uconvert_tofilename(envvar, tmp));
if (s) {
do_uconvert(s, U_ASCII, path, U_CURRENT, sizeof(path)-ucwidth(OTHER_PATH_SEPARATOR));
@@ -1792,14 +1813,14 @@
#ifndef ALLEGRO_MPW
if (strpbrk(mode, "wW")) /* write mode? */
- fd = open(uconvert_toascii(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, OPEN_PERMS);
+ fd = _al_open(uconvert_tofilename(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, OPEN_PERMS);
else
- fd = open(uconvert_toascii(filename, tmp), O_RDONLY | O_BINARY, OPEN_PERMS);
+ fd = _al_open(uconvert_tofilename(filename, tmp), O_RDONLY | O_BINARY, OPEN_PERMS);
#else
if (strpbrk(mode, "wW")) /* write mode? */
- fd = _al_open(uconvert_toascii(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC);
+ fd = _al_open(uconvert_tofilename(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC);
else
- fd = _al_open(uconvert_toascii(filename, tmp), O_RDONLY | O_BINARY);
+ fd = _al_open(uconvert_tofilename(filename, tmp), O_RDONLY | O_BINARY);
#endif
if (fd < 0) {
Index: src/x/xsystem.c
===================================================================
--- src/x/xsystem.c (revision 5814)
+++ src/x/xsystem.c (working copy)
@@ -167,6 +167,8 @@
_unix_read_os_type();
+ _unix_guess_file_encoding();
+
/* install emergency-exit signal handlers */
old_sig_abrt = signal(SIGABRT, _xwin_signal_handler);
old_sig_fpe = signal(SIGFPE, _xwin_signal_handler);
Index: include/allegro/platform/aintunix.h
===================================================================
--- include/allegro/platform/aintunix.h (revision 5814)
+++ include/allegro/platform/aintunix.h (working copy)
@@ -75,6 +75,9 @@
AL_FUNC(void, _unix_register_midi_driver, (int id, MIDI_DRIVER *driver, int autodetect, int priority));
+ /* File system helpers */
+ AL_FUNC(void, _unix_guess_file_encoding, (void));
+
#ifdef ALLEGRO_WITH_XWINDOWS
AL_FUNCPTR(void, _xwin_keyboard_interrupt, (int pressed, int code));
AL_FUNCPTR(void, _xwin_keyboard_focused, (int focused, int state));
Index: include/allegro/platform/aintwin.h
===================================================================
--- include/allegro/platform/aintwin.h (revision 5814)
+++ include/allegro/platform/aintwin.h (working copy)
@@ -207,7 +207,13 @@
#define _TRACE 1 ? (void) 0 : thread_safe_trace
#endif
+#define IS_OLD_WINDOWS (os_type==OSTYPE_WIN3 || os_type==OSTYPE_WIN95 || \
+ os_type==OSTYPE_WIN98 || os_type==OSTYPE_WINME || \
+ os_type==OSTYPE_UNKNOWN)
+AL_FUNC(int, _alwin_open, (const char*, int, int));
+#define _al_open _alwin_open
+
#ifdef __cplusplus
}
#endif
Index: include/allegro/file.h
===================================================================
--- include/allegro/file.h (revision 5814)
+++ include/allegro/file.h (working copy)
@@ -138,6 +138,11 @@
};
+#define uconvert_tofilename(s, buf) uconvert(s, U_CURRENT, buf, get_file_encoding(), sizeof(buf))
+
+AL_FUNC(void, set_file_encoding, (int encoding));
+AL_FUNC(int, get_file_encoding, (void));
+
AL_FUNC(void, packfile_password, (AL_CONST char *password));
AL_FUNC(PACKFILE *, pack_fopen, (AL_CONST char *filename, AL_CONST char *mode));
AL_FUNC(PACKFILE *, pack_fopen_vtable, (AL_CONST PACKFILE_VTABLE *vtable, void *userdata));
@@ -165,6 +170,11 @@
AL_FUNC(void *, pack_get_userdata, (PACKFILE *f));
+#ifndef _al_open
+ #define _al_open open
+#endif
+
+
#ifdef __cplusplus
}
#endif