Re: [AD] Allegro unable to deal with non ascii filenames

[Chris <chris.kcat@xxxxxxxxxx>]

The patch doesn't seem to do much. If it finds utf8/utf-8 in LC_ALL, it will 
set UTF-8 file encoding, or else it'll use the current. By default, UTF-8 is 
current, so either way is using the same. It should probably use U_ASCII 
instead of U_CURRENT. As well, if LC_ALL doesn't exist, you'll get a crash 
(getenv returns NULL for non-existant vars). On my particular machine, LC_* 
vars don't exist. You need to use LANG.

Here's my (completely untested!) patch which fixes these issues, as well as 
adds the _al_open macro for newer Windows to use _wopen (while also setting 
them to use U_UNICODE filenames).

Index: src/linux/lsystem.c
===================================================================
--- src/linux/lsystem.c	(revision 5814)
+++ src/linux/lsystem.c	(working copy)
@@ -164,6 +164,8 @@
 	_unix_read_os_type();
 	if (os_type != OSTYPE_LINUX) return -1; /* FWIW */
 
+   _unix_guess_file_encoding();
+
 	/* This is the only bit that needs root privileges.  First
 	 * we attempt to set our euid to 0, in case this is the
 	 * second time we've been called. */
Index: src/unix/ufile.c
===================================================================
--- src/unix/ufile.c	(revision 5814)
+++ src/unix/ufile.c	(working copy)
@@ -57,7 +57,9 @@
    #endif
 #endif
 
+#define PREFIX_I "al-unix INFO: "
 
+
 /* _al_file_isok:
  *  Helper function to check if it is safe to access a file on a floppy
  *  drive.
@@ -500,3 +502,25 @@
    ASSERT(ff_data);
    return ff_data->size;
 }
+
+
+
+void _unix_guess_file_encoding(void)
+{
+   char const *encoding = "unknown";
+   char *locale = getenv("LC_ALL");
+   if (!locale)
+      locale = getenv("LANG");
+
+   if (locale) {
+      if (strstr(locale, "utf8") ||
+          strstr(locale, "UTF-8") ||
+          strstr(locale, "utf-8") ||
+          strstr(locale, "UTF8"))) {
+         set_file_encoding(U_UTF8);
+         encoding = "UTF8";
+      }
+   }
+
+   TRACE(PREFIX_I "Assumed libc encoding is %s.\n", encoding);
+}
Index: src/qnx/qsystem.c
===================================================================
--- src/qnx/qsystem.c	(revision 5814)
+++ src/qnx/qsystem.c	(working copy)
@@ -337,7 +337,9 @@
    struct sched_param sparam;
    int spolicy;
    char tmp[WINDOW_TITLE_SIZE];
-   
+
+   _unix_guess_file_encoding();
+
    /* install emergency-exit signal handlers */
    old_sig_abrt = signal(SIGABRT, qnx_signal_handler);
    old_sig_fpe  = signal(SIGFPE,  qnx_signal_handler);
Index: src/macosx/system.m
===================================================================
--- src/macosx/system.m	(revision 5814)
+++ src/macosx/system.m	(working copy)
@@ -410,7 +410,9 @@
    if (!osx_bootstrap_ok()) {
       return -1;
    }
-   
+
+   _unix_guess_file_encoding();
+
    /* Install emergency-exit signal handlers */
    old_sig_abrt = signal(SIGABRT, osx_signal_handler);
    old_sig_fpe  = signal(SIGFPE,  osx_signal_handler);
Index: src/file.c
===================================================================
--- src/file.c	(revision 5814)
+++ src/file.c	(working copy)
@@ -74,6 +74,7 @@
 
 static PACKFILE *pack_fopen_special_file(AL_CONST char *filename, AL_CONST char *mode);
 
+static int file_encoding = U_ASCII;
 
 
 #define FA_DAT_FLAGS  (FA_RDONLY | FA_ARCH)
@@ -671,6 +672,26 @@
  ***************************************************/
 
 
+/* set_file_encoding:
+ *  Sets the encoding to use for filenames. By default, UTF8 is assumed.
+ */
+void set_file_encoding(int encoding)
+{
+    file_encoding = encoding;
+}
+
+
+
+/* get_file_encoding:
+ *  Returns the encoding currently assumed for filenames.
+ */
+int get_file_encoding(void)
+{
+    return file_encoding ;
+}
+
+
+
 /* file_exists:
  *  Checks whether a file matching the given name and attributes exists,
  *  returning non zero if it does. The file attribute may contain any of
@@ -788,7 +809,7 @@
    if (!_al_file_isok(filename))
       return -1;
 
-   if (unlink(uconvert_toascii(filename, tmp)) != 0) {
+   if (unlink(uconvert_tofilename(filename, tmp)) != 0) {
       *allegro_errno = errno;
       return -1;
    }
@@ -1197,7 +1218,7 @@
 
    /* try any extra environment variable that the parameters say to use */
    if (envvar) {
-      s = getenv(uconvert_toascii(envvar, tmp));
+      s = getenv(uconvert_tofilename(envvar, tmp));
 
       if (s) {
 	 do_uconvert(s, U_ASCII, path, U_CURRENT, sizeof(path)-ucwidth(OTHER_PATH_SEPARATOR));
@@ -1792,14 +1813,14 @@
 
 #ifndef ALLEGRO_MPW
    if (strpbrk(mode, "wW"))  /* write mode? */
-      fd = open(uconvert_toascii(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, OPEN_PERMS);
+      fd = _al_open(uconvert_tofilename(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, OPEN_PERMS);
    else
-      fd = open(uconvert_toascii(filename, tmp), O_RDONLY | O_BINARY, OPEN_PERMS);
+      fd = _al_open(uconvert_tofilename(filename, tmp), O_RDONLY | O_BINARY, OPEN_PERMS);
 #else
    if (strpbrk(mode, "wW"))  /* write mode? */
-      fd = _al_open(uconvert_toascii(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC);
+      fd = _al_open(uconvert_tofilename(filename, tmp), O_WRONLY | O_BINARY | O_CREAT | O_TRUNC);
    else
-      fd = _al_open(uconvert_toascii(filename, tmp), O_RDONLY | O_BINARY);
+      fd = _al_open(uconvert_tofilename(filename, tmp), O_RDONLY | O_BINARY);
 #endif
 
    if (fd < 0) {
Index: src/x/xsystem.c
===================================================================
--- src/x/xsystem.c	(revision 5814)
+++ src/x/xsystem.c	(working copy)
@@ -167,6 +167,8 @@
 
    _unix_read_os_type();
 
+   _unix_guess_file_encoding();
+
    /* install emergency-exit signal handlers */
    old_sig_abrt = signal(SIGABRT, _xwin_signal_handler);
    old_sig_fpe  = signal(SIGFPE,  _xwin_signal_handler);
Index: include/allegro/platform/aintunix.h
===================================================================
--- include/allegro/platform/aintunix.h	(revision 5814)
+++ include/allegro/platform/aintunix.h	(working copy)
@@ -75,6 +75,9 @@
    AL_FUNC(void, _unix_register_midi_driver, (int id, MIDI_DRIVER *driver, int autodetect, int priority));
 
 
+   /* File system helpers */
+   AL_FUNC(void, _unix_guess_file_encoding, (void));
+
 #ifdef ALLEGRO_WITH_XWINDOWS
    AL_FUNCPTR(void, _xwin_keyboard_interrupt, (int pressed, int code));
    AL_FUNCPTR(void, _xwin_keyboard_focused, (int focused, int state));
Index: include/allegro/file.h
===================================================================
--- include/allegro/file.h	(revision 5814)
+++ include/allegro/file.h	(working copy)
@@ -138,6 +138,11 @@
 };
 
 
+#define uconvert_tofilename(s, buf)      uconvert(s, U_CURRENT, buf, get_file_encoding(), sizeof(buf))
+
+AL_FUNC(void, set_file_encoding, (int encoding));
+AL_FUNC(int, get_file_encoding, (void));
+
 AL_FUNC(void, packfile_password, (AL_CONST char *password));
 AL_FUNC(PACKFILE *, pack_fopen, (AL_CONST char *filename, AL_CONST char *mode));
 AL_FUNC(PACKFILE *, pack_fopen_vtable, (AL_CONST PACKFILE_VTABLE *vtable, void *userdata));
@@ -165,6 +170,11 @@
 AL_FUNC(void *, pack_get_userdata, (PACKFILE *f));
 
 
+#ifndef _al_open
+   #define _al_open open
+#endif
+
+
 #ifdef __cplusplus
    }
 #endif
Index: include/allegro/platform/aintwin.h
===================================================================
--- include/allegro/platform/aintwin.h	(revision 5814)
+++ include/allegro/platform/aintwin.h	(working copy)
@@ -207,7 +207,13 @@
    #define _TRACE                 1 ? (void) 0 : thread_safe_trace
 #endif
 
+#define IS_OLD_WINDOWS (os_type==OSTYPE_WIN3  || os_type==OSTYPE_WIN95 || \
+                        os_type==OSTYPE_WIN98 || os_type==OSTYPE_WINME || \
+                        os_type==OSTYPE_UNKNOWN)
 
+AL_FUNC(int, _alwin_open, (const char*, int, int));
+#define _al_open _alwin_open
+
 #ifdef __cplusplus
    }
 #endif
Index: src/win/wsystem.c
===================================================================
--- src/win/wsystem.c	(revision 5814)
+++ src/win/wsystem.c	(working copy)
@@ -191,6 +191,10 @@
    if (init_directx_window() != 0)
       goto Error;
 
+   if (IS_OLD_WINDOWS) {
+      set_filename_encoding(U_UNICODE);
+   }
+
    return 0;
 
  Error:
Index: src/win/wfile.c
===================================================================
--- src/win/wfile.c	(revision 5814)
+++ src/win/wfile.c	(working copy)
@@ -51,11 +51,19 @@
    struct _stat s;
    char tmp[1024];
 
-   if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE,
-              sizeof(tmp)), &s) != 0) {
-      *allegro_errno = errno;
-      return 0;
+   if (IS_OLD_WINDOWS) {
+      if (_stat(uconvert(filename, U_CURRENT, tmp, U_ASCII, sizeof(tmp)), &s) != 0) {
+         *allegro_errno = errno;
+         return 0;
+      }
    }
+   else {
+      if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE,
+                 sizeof(tmp)), &s) != 0) {
+         *allegro_errno = errno;
+         return 0;
+      }
+   }
 
    return s.st_size;
 }
@@ -70,11 +78,18 @@
    struct _stat s;
    char tmp[1024];
 
-   if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE,
-              sizeof(tmp)), &s) != 0) {
-      *allegro_errno = errno;
-      return 0;
+   if (IS_OLD_WINDOWS) {
+      if (_stat(uconvert(filename, U_CURRENT, tmp, U_ASCII, sizeof(tmp)), &s) != 0) {
+         *allegro_errno = errno;
+         return 0;
+      }
    }
+   else {
+      if (_wstat((wchar_t*)uconvert(filename, U_CURRENT, tmp, U_UNICODE, sizeof(tmp)), &s) != 0) {
+         *allegro_errno = errno;
+         return 0;
+      }
+   }
 
    return s.st_mtime;
 }
@@ -84,7 +99,10 @@
 /* structure for use by the directory scanning routines */
 struct FF_DATA
 {
-   struct _wfinddata_t data;
+   union {
+      struct _finddata_t a;
+      struct _wfinddata_t w;
+   } data;
    long handle;
    int attrib;
 };
@@ -98,12 +116,22 @@
 {
    struct FF_DATA *ff_data = (struct FF_DATA *) info->ff_data;
 
-   info->attrib = ff_data->data.attrib;
-   info->time = ff_data->data.time_write;
-   info->size = ff_data->data.size;
+   if (IS_OLD_WINDOWS) {
+      info->attrib = ff_data->data.a.attrib;
+      info->time = ff_data->data.a.time_write;
+      info->size = ff_data->data.a.size;
 
-   do_uconvert((const char*)ff_data->data.name, U_UNICODE, info->name, U_CURRENT,
-               sizeof(info->name));
+      do_uconvert(ff_data->data.a.name, U_ASCII, info->name, U_CURRENT,
+                  sizeof(info->name));
+   }
+   else {
+      info->attrib = ff_data->data.w.attrib;
+      info->time = ff_data->data.w.time_write;
+      info->size = ff_data->data.w.size;
+
+      do_uconvert((const char*)ff_data->data.w.name, U_UNICODE, info->name,
+                  U_CURRENT, sizeof(info->name));
+   }
 }
 
 
@@ -144,23 +172,47 @@
    /* start the search */
    errno = *allegro_errno = 0;
 
-   ff_data->handle = _wfindfirst((wchar_t*)uconvert(pattern, U_CURRENT, tmp,
-                                 U_UNICODE, sizeof(tmp)), &ff_data->data);
+   if (IS_OLD_WINDOWS) {
+      ff_data->handle = _findfirst(uconvert(pattern, U_CURRENT, tmp,
+                                            U_ASCII, sizeof(tmp)),
+                                            &ff_data->data.a);
 
-   if (ff_data->handle < 0) {
-      *allegro_errno = errno;
-      _AL_FREE(ff_data);
-      info->ff_data = NULL;
-      return -1;
+      if (ff_data->handle < 0) {
+         *allegro_errno = errno;
+         _AL_FREE(ff_data);
+         info->ff_data = NULL;
+         return -1;
+      }
+
+      if (ff_data->data.a.attrib & ~ff_data->attrib) {
+         if (al_findnext(info) != 0) {
+            al_findclose(info);
+            return -1;
+         }
+         else
+            return 0;
+      }
    }
+   else {
+      ff_data->handle = _wfindfirst((wchar_t*)uconvert(pattern, U_CURRENT, tmp,
+                                                       U_UNICODE, sizeof(tmp)),
+                                                       &ff_data->data.w);
 
-   if (ff_data->data.attrib & ~ff_data->attrib) {
-      if (al_findnext(info) != 0) {
-         al_findclose(info);
+      if (ff_data->handle < 0) {
+         *allegro_errno = errno;
+         _AL_FREE(ff_data);
+         info->ff_data = NULL;
          return -1;
       }
-      else
-         return 0;
+
+      if (ff_data->data.w.attrib & ~ff_data->attrib) {
+         if (al_findnext(info) != 0) {
+            al_findclose(info);
+            return -1;
+         }
+         else
+            return 0;
+      }
    }
 
    fill_ffblk(info);
@@ -176,14 +228,23 @@
 {
    struct FF_DATA *ff_data = (struct FF_DATA *) info->ff_data;
 
-   do {
-      if (_wfindnext(ff_data->handle, &ff_data->data) != 0) {
-         *allegro_errno = errno;
-         return -1;
-      }
+   if (IS_OLD_WINDOWS) {
+      do {
+         if (_findnext(ff_data->handle, &ff_data->data.a) != 0) {
+            *allegro_errno = errno;
+            return -1;
+         }
+      } while (ff_data->data.a.attrib & ~ff_data->attrib);
+   }
+   else {
+      do {
+         if (_wfindnext(ff_data->handle, &ff_data->data.w) != 0) {
+            *allegro_errno = errno;
+            return -1;
+         }
+      } while (ff_data->data.w.attrib & ~ff_data->attrib);
+   }
 
-   } while (ff_data->data.attrib & ~ff_data->attrib);
-
    fill_ffblk(info);
    return 0;
 }
@@ -233,10 +294,18 @@
 {
    wchar_t tmp[1024];
 
-   if (_wgetdcwd(drive+1, tmp, sizeof(tmp)/sizeof(tmp[0])))
-      do_uconvert((const char*)tmp, U_UNICODE, buf, U_CURRENT, size);
-   else
-      usetc(buf, 0);
+   if (IS_OLD_WINDOWS) {
+      if (_getdcwd(drive+1, tmp, sizeof(tmp)))
+         do_uconvert(tmp, U_ASCII, buf, U_CURRENT, size);
+      else
+         usetc(buf, 0);
+   }
+   else {
+      if (_wgetdcwd(drive+1, (wchar_t*)tmp, sizeof(tmp)/sizeof(wchar_t*)))
+         do_uconvert(tmp, U_UNICODE, buf, U_CURRENT, size);
+      else
+         usetc(buf, 0);
+   }
 }
 
 
@@ -245,8 +314,25 @@
  */
 uint64_t al_ffblk_get_size(struct al_ffblk *info)
 {
+   struct FF_DATA *ff_data;
+
    ASSERT(info);
-   struct FF_DATA *ff_data = (struct FF_DATA *) info->ff_data;
+   ff_data = (struct FF_DATA *) info->ff_data;
 
-   return ff_data->data.size;
+   if (IS_OLD_WINDOWS) {
+      return ff_data->data.a.size;
+   }
+   else {
+      return ff_data->data.w.size;
+   }
 }
+
+int _alwin_open(const char *filename, int mode, int perm)
+{
+   if (IS_OLD_WINDOWS) {
+      return open(filename, mode, perm);
+   }
+   else {
+      return _wopen((wchar_t*)filename, mode, perm);
+   }
+}