0.1.1: names reach the system as UTF-8, never through the code page

README.md

@@ -6,7 +6,7 @@ with compile-time platform constants and a few platform modules.
 
 ```toml
 [dependencies]
-os = { namespace = "mcpplibs", version = "0.1.0" }
+os = { namespace = "mcpplibs", version = "0.1.1" }
 ```
 
 ```cpp
@@ -68,6 +68,11 @@ modules.
   is not an odr-use, so it links. CI asserts both.
 - **R6.** Where openkal is not in its shared form, this package's resources do not cross a
   shared-object boundary.
+- **R7.** A name reaches the system as UTF-8 and comes back as UTF-8 on every host, never
+  through the process's code page. `path::string()` and `path(std::string)` use the ANSI code
+  page on Windows, where a name the code page cannot spell throws and one it can is not UTF-8.
+  `os::fs::utf8()`, `generic_utf8()` and `from_utf8()` are the conversions, and the package uses
+  nothing else. The tests write, read and rename names in four scripts.
 
 Interface units import `std` and contain no system header and no macro. Implementation units
 on openkal import only `openkal.*`. The Windows units declare the functions they call

mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "os"
-version     = "0.1.0"
+version     = "0.1.1"
 description = "mcpplibs.os: a C++ operating-system library over openkal --- processes, files, environment, terminals, directories and local IPC, with compile-time platform constants and a few platform modules."
 license     = "Apache-2.0"
 authors     = ["mcpplibs"]

src/dirs.cpp

@@ -3,19 +3,20 @@ module mcpplibs.os.dirs;
 import std;
 import mcpplibs.os.platform;
 import mcpplibs.os.env;
+import mcpplibs.os.fs;
 
 namespace mcpplibs::os::dirs {
 
 namespace {
 std::optional<std::filesystem::path> var(std::string_view name) {
     auto v = env::get(name);
     if (!v || v->empty()) return std::nullopt;
-    return std::filesystem::path(*v);
+    return fs::from_utf8(*v);
 }
 std::optional<std::filesystem::path> under_home(std::string_view rest) {
     auto h = home();
     if (!h) return std::nullopt;
-    return *h / std::filesystem::path(rest);
+    return *h / fs::from_utf8(rest);
 }
 bool windows() { return host() == platform::windows; }
 bool macos() { return host() == platform::macos; }

src/env.cpp

@@ -48,13 +48,13 @@ std::vector<std::string> arguments() {
 std::optional<std::filesystem::path> image() {
     auto s = counted([](char* o, kal_uintptr cap) { return kal_env_image(o, cap); });
     if (!s || s->empty()) return std::nullopt;
-    return std::filesystem::path(*s);
+    return fs::from_utf8(*s);
 }
 
 std::optional<std::filesystem::path> loader() {
     auto s = counted([](char* o, kal_uintptr cap) { return kal_env_loader(o, cap); });
     if (!s || s->empty()) return std::nullopt;
-    return std::filesystem::path(*s);
+    return fs::from_utf8(*s);
 }
 
 std::vector<std::string> self_command() {
@@ -63,8 +63,8 @@ std::vector<std::string> self_command() {
     const auto own = image();
     // Absolute, so that the command starts the program from any directory: an
     // image is named as its loader was given it, which may be relative.
-    if (program) out.push_back(fs::absolute(*program).string());
-    if (own && (!program || *own != *program)) out.push_back(fs::absolute(*own).string());
+    if (program) out.push_back(fs::utf8(fs::absolute(*program)));
+    if (own && (!program || *own != *program)) out.push_back(fs::utf8(fs::absolute(*own)));
     return out;
 }
 

src/fs.cpp

@@ -65,7 +65,7 @@ struct opened_file {
 result<located> where(const std::filesystem::path& p, std::string_view doing) {
     auto l = locate(p);
     if (!l) return std::unexpected(error{ errc::not_found,
-        std::format("{} {}: beneath no directory this program was given", doing, p.generic_string()) });
+        std::format("{} {}: beneath no directory this program was given", doing, generic_utf8(p)) });
     return *l;
 }
 
@@ -100,8 +100,22 @@ result<void> write_all(kal_file f, std::string_view bytes, std::string_view doin
 
 }  // namespace
 
+std::string utf8(const std::filesystem::path& name) {
+    const std::u8string u = name.u8string();
+    return std::string(u.begin(), u.end());
+}
+
+std::string generic_utf8(const std::filesystem::path& name) {
+    const std::u8string u = name.generic_u8string();
+    return std::string(u.begin(), u.end());
+}
+
+std::filesystem::path from_utf8(std::string_view text) {
+    return std::filesystem::path(std::u8string(text.begin(), text.end()));
+}
+
 std::filesystem::path absolute(const std::filesystem::path& name) {
-    const std::string s = name.generic_string();
+    const std::string s = generic_utf8(name);
     const bool windows = host() == platform::windows;
     const bool already = windows
         ? (s.size() >= 3 && std::isalpha(static_cast<unsigned char>(s[0])) && s[1] == ':' && separator(s[2]))
@@ -111,7 +125,7 @@ std::filesystem::path absolute(const std::filesystem::path& name) {
     std::error_code ec;
     const auto cwd = std::filesystem::current_path(ec);
     if (ec) return name;
-    return std::filesystem::path(slashed(cwd.generic_string()) + "/" + s);
+    return from_utf8(slashed(generic_utf8(cwd)) + "/" + s);
 }
 
 namespace {
@@ -139,7 +153,7 @@ bool fold_equal(std::string_view a, std::string_view b, bool fold) {
 
 std::string generic(const std::filesystem::path& name) {
     const bool windows = host() == platform::windows;
-    std::string s = name.generic_string();
+    std::string s = generic_utf8(name);
     if (windows) std::ranges::replace(s, '\\', '/');
     const std::size_t rootLen = root_length(s, windows);
     const std::string root = s.substr(0, rootLen);
@@ -200,7 +214,7 @@ bool names_a_place(std::string_view name) {
 
 std::optional<located> locate(const std::filesystem::path& name) {
     const auto whole = mcpplibs::os::fs::absolute(name);
-    const std::string path = slashed(whole.lexically_normal().generic_string());
+    const std::string path = slashed(generic_utf8(whole.lexically_normal()));
     const bool fold = conventions().case_insensitive_names;
     std::optional<located> best;
     std::size_t best_len = 0;
@@ -232,12 +246,12 @@ result<std::string> read(const std::filesystem::path& file) {
     opened_file f;
     const int rc = kal_fs_open(dir_of(*l), l->relative.data(), l->relative.size(),
                                kal::fs::open::read.bits, &f.f);
-    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("reading {}", file.generic_string())));
+    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("reading {}", generic_utf8(file))));
     std::string out;
     char buf[65536];
     for (;;) {
         const kal_intptr n = kal_stream_read(kal_fs_stream(f.f), buf, sizeof buf);
-        if (n < 0) return std::unexpected(from_kal(static_cast<int>(-n), std::format("reading {}", file.generic_string())));
+        if (n < 0) return std::unexpected(from_kal(static_cast<int>(-n), std::format("reading {}", generic_utf8(file))));
         if (n == 0) break;
         out.append(buf, static_cast<std::size_t>(n));
     }
@@ -250,8 +264,8 @@ result<void> write(const std::filesystem::path& file, std::string_view bytes) {
     opened_file f;
     const auto flags = kal::fs::open::write | kal::fs::open::create | kal::fs::open::truncate;
     const int rc = kal_fs_open(dir_of(*l), l->relative.data(), l->relative.size(), flags.bits, &f.f);
-    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("writing {}", file.generic_string())));
-    return write_all(f.f, bytes, std::format("writing {}", file.generic_string()));
+    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("writing {}", generic_utf8(file))));
+    return write_all(f.f, bytes, std::format("writing {}", generic_utf8(file)));
 }
 
 namespace {
@@ -278,7 +292,7 @@ result<std::string> write_beside(const located& l, std::string_view bytes, std::
 }  // namespace
 
 result<void> write_atomic(const std::filesystem::path& file, std::string_view bytes) {
-    const std::string doing = std::format("writing {}", file.generic_string());
+    const std::string doing = std::format("writing {}", generic_utf8(file));
     auto l = where(file, "writing");
     if (!l) return std::unexpected(l.error());
     auto temp = write_beside(*l, bytes, doing);
@@ -293,7 +307,7 @@ result<void> write_atomic(const std::filesystem::path& file, std::string_view by
 }
 
 result<published> publish(const std::filesystem::path& file, std::string_view bytes) {
-    const std::string doing = std::format("publishing {}", file.generic_string());
+    const std::string doing = std::format("publishing {}", generic_utf8(file));
     auto l = where(file, "publishing");
     if (!l) return std::unexpected(l.error());
     kal_dir parent{};
@@ -335,7 +349,7 @@ result<capabilities> caps(const std::filesystem::path& directory) {
     bool close = false;
     if (l->relative != ".") {
         const int rc = kal_fs_open_dir(dir_of(*l), l->relative.data(), l->relative.size(), &d);
-        if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("asking about {}", directory.generic_string())));
+        if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("asking about {}", generic_utf8(directory))));
         close = true;
     }
     const kal_uintptr p = kal_fs_props(d);
@@ -354,7 +368,7 @@ result<capabilities> caps(const std::filesystem::path& directory) {
 }
 
 result<lock> lock::acquire(const std::filesystem::path& file, kind k, bool wait) {
-    const std::string doing = std::format("locking {}", file.generic_string());
+    const std::string doing = std::format("locking {}", generic_utf8(file));
     auto l = where(file, "locking");
     if (!l) return std::unexpected(l.error());
     kal_file f{};
@@ -388,18 +402,18 @@ void lock::release() noexcept {
 
 result<void> create_link(const std::filesystem::path& target, const std::filesystem::path& link,
                          bool directory) {
-    const std::string doing = std::format("linking {} to {}", link.generic_string(), target.generic_string());
+    const std::string doing = std::format("linking {} to {}", generic_utf8(link), generic_utf8(target));
     auto l = where(link, "linking");
     if (!l) return std::unexpected(l.error());
-    const std::string t = target.generic_string();
+    const std::string t = generic_utf8(target);
     const int rc = kal_fs_link_create(dir_of(*l), l->relative.data(), l->relative.size(), t.data(), t.size(),
                                       directory ? kal::fs::link_directory : 0);
     if (rc != kal_ok) return std::unexpected(from_kal(rc, doing));
     return {};
 }
 
 result<std::string> read_link(const std::filesystem::path& link) {
-    const std::string doing = std::format("reading the link {}", link.generic_string());
+    const std::string doing = std::format("reading the link {}", generic_utf8(link));
     auto l = where(link, "reading the link");
     if (!l) return std::unexpected(l.error());
     std::string out(4096, '\0');
@@ -416,7 +430,7 @@ result<void> set_executable(const std::filesystem::path& file, bool executable)
     auto l = where(file, "marking");
     if (!l) return std::unexpected(l.error());
     const int rc = kal_fs_set_executable_at(dir_of(*l), l->relative.data(), l->relative.size(), executable ? 1 : 0);
-    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("marking {}", file.generic_string())));
+    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("marking {}", generic_utf8(file))));
     return {};
 }
 
@@ -427,8 +441,8 @@ result<kal_node_info> info(const std::filesystem::path& node, kal_u32 wanted, st
     kal_node_info i{};
     i.self_size = sizeof i;
     const int rc = kal_fs_info(dir_of(*l), l->relative.data(), l->relative.size(), 0, wanted, &i);
-    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("{} {}", doing, node.generic_string())));
-    if (i.kind == kal_node_absent) return std::unexpected(error{ errc::not_found, std::format("{} {}", doing, node.generic_string()) });
+    if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("{} {}", doing, generic_utf8(node))));
+    if (i.kind == kal_node_absent) return std::unexpected(error{ errc::not_found, std::format("{} {}", doing, generic_utf8(node)) });
     return i;
 }
 }  // namespace
@@ -443,12 +457,12 @@ result<identity> identity_of(const std::filesystem::path& node) {
     auto i = info(node, kal::fs::field::identity, "asking about");
     if (!i) return std::unexpected(i.error());
     if ((i->present & kal::fs::field::identity) == 0)
-        return std::unexpected(error{ errc::not_supported, std::format("the identity of {}", node.generic_string()) });
+        return std::unexpected(error{ errc::not_supported, std::format("the identity of {}", generic_utf8(node)) });
     return identity{ i->identity[0], i->identity[1] };
 }
 
 result<void> rename(const std::filesystem::path& from, const std::filesystem::path& to, bool replace) {
-    const std::string doing = std::format("renaming {} to {}", from.generic_string(), to.generic_string());
+    const std::string doing = std::format("renaming {} to {}", generic_utf8(from), generic_utf8(to));
     auto a = where(from, "renaming");
     if (!a) return std::unexpected(a.error());
     auto b = where(to, "renaming");

src/fs.cppm

@@ -39,6 +39,14 @@ std::vector<std::string> preopens();
 // the image runs upon.
 std::filesystem::path absolute(const std::filesystem::path& name);
 
+// A name's text in UTF-8 and back, never through the process's code page.
+// UTF-8 is what openkal's names are on every system; `path::string()` and
+// `path(std::string)` use the ANSI code page on Windows, where a name it
+// cannot spell throws and one it can is not UTF-8.
+std::string utf8(const std::filesystem::path& name);
+std::string generic_utf8(const std::filesystem::path& name);  // with '/' separators
+std::filesystem::path from_utf8(std::string_view text);
+
 // NAMES IN THE HOST'S RULES, AND WITHOUT ASKING THE FILE SYSTEM. Each is lexical:
 // it reads the name and never the node, so a link is not followed and a name
 // that does not exist is answered as well as one that does (`identity_of' says

src/ipc.cpp

@@ -19,7 +19,7 @@ struct place { kal_dir dir{}; bool owned = false; std::string doing; };
 // The directory, opened, for a name to be bound in it.
 result<place> directory_of(const std::filesystem::path& d, std::string_view name, std::string_view verb) {
     place p;
-    p.doing = std::format("{} {} in {}", verb, name, d.generic_string());
+    p.doing = std::format("{} {} in {}", verb, name, fs::generic_utf8(d));
     auto l = fs::locate(d);
     if (!l) return std::unexpected(error{ errc::not_found, p.doing + ": beneath no directory this program was given" });
     p.dir = kal_dir{ static_cast<kal_uintptr>(l->directory) };
@@ -131,7 +131,7 @@ result<listener> listener::listen(const std::filesystem::path& directory, std::s
     if (rc != kal_ok) return std::unexpected(from_kal(rc, doing));
     listener out;
     out.listener_ = static_cast<std::uintptr_t>(l.h);
-    out.where_ = directory / std::filesystem::path(std::string(name));
+    out.where_ = directory / fs::from_utf8(name);
     return out;
 }
 

src/process.cpp

@@ -163,27 +163,27 @@ result<process> process::spawn(const spawn_options& o) {
         return std::unexpected(error{ errc::invalid, "a program's input cannot be this program's error output" });
     // The program, by name or by its place.
     std::filesystem::path program = o.program;
-    if (!fs::names_a_place(o.program.generic_string())) {
-        auto found = env::find_executable(program.generic_string());
-        if (!found) return std::unexpected(error{ errc::not_found, std::format("{} is not on PATH", program.generic_string()) });
+    if (!fs::names_a_place(fs::generic_utf8(o.program))) {
+        auto found = env::find_executable(fs::generic_utf8(program));
+        if (!found) return std::unexpected(error{ errc::not_found, std::format("{} is not on PATH", fs::generic_utf8(program)) });
         program = *found;
     }
     std::error_code ec;
     program = fs::absolute(program).lexically_normal();
     const auto where = fs::locate(program);
     if (!where) return std::unexpected(error{ errc::not_found,
-        std::format("starting {}: beneath no directory this program was given", program.generic_string()) });
+        std::format("starting {}: beneath no directory this program was given", fs::generic_utf8(program)) });
 
     const auto cwd_path = o.cwd.empty() ? std::filesystem::current_path(ec) : o.cwd;
     const auto cwd = fs::locate(cwd_path);
     if (!cwd) return std::unexpected(error{ errc::not_found,
         std::format("starting {}: {} is beneath no directory this program was given",
-                    program.generic_string(), cwd_path.generic_string()) });
+                    fs::generic_utf8(program), fs::generic_utf8(cwd_path)) });
     kal_dir work = as_dir(cwd->directory);
     bool close_work = false;
     if (cwd->relative != ".") {
         if (const int rc = kal_fs_open_dir(work, cwd->relative.data(), cwd->relative.size(), &work); rc != kal_ok)
-            return std::unexpected(from_kal(rc, std::format("starting {} in {}", program.generic_string(), cwd_path.generic_string())));
+            return std::unexpected(from_kal(rc, std::format("starting {} in {}", fs::generic_utf8(program), fs::generic_utf8(cwd_path))));
         close_work = true;
     }
     struct closer { kal_dir d; bool on; ~closer() { if (on) kal_fs_close_dir(d); } } work_guard{ work, close_work };
@@ -234,7 +234,7 @@ result<process> process::spawn(const spawn_options& o) {
 
     // The vector, the first element in the system's own spelling: a program
     // that reads its own command line takes a forward slash for a switch.
-    std::string first = o.argv0.value_or(program.string());
+    std::string first = o.argv0.value_or(fs::utf8(program));
     if (!o.argv0 && host() == platform::windows) std::ranges::replace(first, '/', '\\');
     std::vector<std::string> argv_storage{ std::move(first) };
     for (const auto& a : o.args) argv_storage.push_back(a);
@@ -264,7 +264,7 @@ result<process> process::spawn(const spawn_options& o) {
         if (n.file.h) kal_fs_close_file(n.file);
         else if (n.channel && n.child.h) kal_process_channel_close(n.child);
     }
-    if (rc != kal_ok) return fail_streams(from_kal(rc, std::format("starting {}", program.generic_string())));
+    if (rc != kal_ok) return fail_streams(from_kal(rc, std::format("starting {}", fs::generic_utf8(program))));
     if (unit) { s->job = job; s->has_job = true; }
     process p;
     p.state_ = std::move(s);

src/windows/windows.cpp

@@ -4,6 +4,7 @@ module;
 module mcpplibs.os.windows;
 
 import std;
+import mcpplibs.os.fs;
 
 namespace mcpplibs::os::windows {
 
@@ -25,7 +26,7 @@ bool elevated() {
 }
 
 std::optional<bool> on_dev_drive(const std::filesystem::path& directory) {
-    const std::string utf8 = directory.string();
+    const std::string utf8 = fs::utf8(directory);
     const int n = MultiByteToWideChar(65001, 0, utf8.data(), static_cast<int>(utf8.size()), nullptr, 0);
     if (n <= 0) return std::nullopt;
     std::wstring wide(static_cast<std::size_t>(n), L'\0');