diff --git a/README.md b/README.md index 219bc4d..7167b3f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ with compile-time platform constants and a few platform modules. ```toml [dependencies] -os = { namespace = "mcpplibs", version = "0.1.0" } +os = { namespace = "mcpplibs", version = "0.1.1" } ``` ```cpp @@ -68,6 +68,11 @@ modules. is not an odr-use, so it links. CI asserts both. - **R6.** Where openkal is not in its shared form, this package's resources do not cross a shared-object boundary. +- **R7.** A name reaches the system as UTF-8 and comes back as UTF-8 on every host, never + through the process's code page. `path::string()` and `path(std::string)` use the ANSI code + page on Windows, where a name the code page cannot spell throws and one it can is not UTF-8. + `os::fs::utf8()`, `generic_utf8()` and `from_utf8()` are the conversions, and the package uses + nothing else. The tests write, read and rename names in four scripts. Interface units import `std` and contain no system header and no macro. Implementation units on openkal import only `openkal.*`. The Windows units declare the functions they call diff --git a/mcpp.toml b/mcpp.toml index 8725191..4514942 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -1,7 +1,7 @@ [package] namespace = "mcpplibs" name = "os" -version = "0.1.0" +version = "0.1.1" description = "mcpplibs.os: a C++ operating-system library over openkal --- processes, files, environment, terminals, directories and local IPC, with compile-time platform constants and a few platform modules." license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/src/dirs.cpp b/src/dirs.cpp index 330a597..9afc176 100644 --- a/src/dirs.cpp +++ b/src/dirs.cpp @@ -3,6 +3,7 @@ module mcpplibs.os.dirs; import std; import mcpplibs.os.platform; import mcpplibs.os.env; +import mcpplibs.os.fs; namespace mcpplibs::os::dirs { @@ -10,12 +11,12 @@ namespace { std::optional var(std::string_view name) { auto v = env::get(name); if (!v || v->empty()) return std::nullopt; - return std::filesystem::path(*v); + return fs::from_utf8(*v); } std::optional under_home(std::string_view rest) { auto h = home(); if (!h) return std::nullopt; - return *h / std::filesystem::path(rest); + return *h / fs::from_utf8(rest); } bool windows() { return host() == platform::windows; } bool macos() { return host() == platform::macos; } diff --git a/src/env.cpp b/src/env.cpp index 1102aad..d091bbc 100644 --- a/src/env.cpp +++ b/src/env.cpp @@ -48,13 +48,13 @@ std::vector arguments() { std::optional image() { auto s = counted([](char* o, kal_uintptr cap) { return kal_env_image(o, cap); }); if (!s || s->empty()) return std::nullopt; - return std::filesystem::path(*s); + return fs::from_utf8(*s); } std::optional loader() { auto s = counted([](char* o, kal_uintptr cap) { return kal_env_loader(o, cap); }); if (!s || s->empty()) return std::nullopt; - return std::filesystem::path(*s); + return fs::from_utf8(*s); } std::vector self_command() { @@ -63,8 +63,8 @@ std::vector self_command() { const auto own = image(); // Absolute, so that the command starts the program from any directory: an // image is named as its loader was given it, which may be relative. - if (program) out.push_back(fs::absolute(*program).string()); - if (own && (!program || *own != *program)) out.push_back(fs::absolute(*own).string()); + if (program) out.push_back(fs::utf8(fs::absolute(*program))); + if (own && (!program || *own != *program)) out.push_back(fs::utf8(fs::absolute(*own))); return out; } diff --git a/src/fs.cpp b/src/fs.cpp index 3fed3ff..1ad9dd5 100644 --- a/src/fs.cpp +++ b/src/fs.cpp @@ -65,7 +65,7 @@ struct opened_file { result where(const std::filesystem::path& p, std::string_view doing) { auto l = locate(p); if (!l) return std::unexpected(error{ errc::not_found, - std::format("{} {}: beneath no directory this program was given", doing, p.generic_string()) }); + std::format("{} {}: beneath no directory this program was given", doing, generic_utf8(p)) }); return *l; } @@ -100,8 +100,22 @@ result write_all(kal_file f, std::string_view bytes, std::string_view doin } // namespace +std::string utf8(const std::filesystem::path& name) { + const std::u8string u = name.u8string(); + return std::string(u.begin(), u.end()); +} + +std::string generic_utf8(const std::filesystem::path& name) { + const std::u8string u = name.generic_u8string(); + return std::string(u.begin(), u.end()); +} + +std::filesystem::path from_utf8(std::string_view text) { + return std::filesystem::path(std::u8string(text.begin(), text.end())); +} + std::filesystem::path absolute(const std::filesystem::path& name) { - const std::string s = name.generic_string(); + const std::string s = generic_utf8(name); const bool windows = host() == platform::windows; const bool already = windows ? (s.size() >= 3 && std::isalpha(static_cast(s[0])) && s[1] == ':' && separator(s[2])) @@ -111,7 +125,7 @@ std::filesystem::path absolute(const std::filesystem::path& name) { std::error_code ec; const auto cwd = std::filesystem::current_path(ec); if (ec) return name; - return std::filesystem::path(slashed(cwd.generic_string()) + "/" + s); + return from_utf8(slashed(generic_utf8(cwd)) + "/" + s); } namespace { @@ -139,7 +153,7 @@ bool fold_equal(std::string_view a, std::string_view b, bool fold) { std::string generic(const std::filesystem::path& name) { const bool windows = host() == platform::windows; - std::string s = name.generic_string(); + std::string s = generic_utf8(name); if (windows) std::ranges::replace(s, '\\', '/'); const std::size_t rootLen = root_length(s, windows); const std::string root = s.substr(0, rootLen); @@ -200,7 +214,7 @@ bool names_a_place(std::string_view name) { std::optional locate(const std::filesystem::path& name) { const auto whole = mcpplibs::os::fs::absolute(name); - const std::string path = slashed(whole.lexically_normal().generic_string()); + const std::string path = slashed(generic_utf8(whole.lexically_normal())); const bool fold = conventions().case_insensitive_names; std::optional best; std::size_t best_len = 0; @@ -232,12 +246,12 @@ result read(const std::filesystem::path& file) { opened_file f; const int rc = kal_fs_open(dir_of(*l), l->relative.data(), l->relative.size(), kal::fs::open::read.bits, &f.f); - if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("reading {}", file.generic_string()))); + if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("reading {}", generic_utf8(file)))); std::string out; char buf[65536]; for (;;) { const kal_intptr n = kal_stream_read(kal_fs_stream(f.f), buf, sizeof buf); - if (n < 0) return std::unexpected(from_kal(static_cast(-n), std::format("reading {}", file.generic_string()))); + if (n < 0) return std::unexpected(from_kal(static_cast(-n), std::format("reading {}", generic_utf8(file)))); if (n == 0) break; out.append(buf, static_cast(n)); } @@ -250,8 +264,8 @@ result write(const std::filesystem::path& file, std::string_view bytes) { opened_file f; const auto flags = kal::fs::open::write | kal::fs::open::create | kal::fs::open::truncate; const int rc = kal_fs_open(dir_of(*l), l->relative.data(), l->relative.size(), flags.bits, &f.f); - if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("writing {}", file.generic_string()))); - return write_all(f.f, bytes, std::format("writing {}", file.generic_string())); + if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("writing {}", generic_utf8(file)))); + return write_all(f.f, bytes, std::format("writing {}", generic_utf8(file))); } namespace { @@ -278,7 +292,7 @@ result write_beside(const located& l, std::string_view bytes, std:: } // namespace result write_atomic(const std::filesystem::path& file, std::string_view bytes) { - const std::string doing = std::format("writing {}", file.generic_string()); + const std::string doing = std::format("writing {}", generic_utf8(file)); auto l = where(file, "writing"); if (!l) return std::unexpected(l.error()); auto temp = write_beside(*l, bytes, doing); @@ -293,7 +307,7 @@ result write_atomic(const std::filesystem::path& file, std::string_view by } result publish(const std::filesystem::path& file, std::string_view bytes) { - const std::string doing = std::format("publishing {}", file.generic_string()); + const std::string doing = std::format("publishing {}", generic_utf8(file)); auto l = where(file, "publishing"); if (!l) return std::unexpected(l.error()); kal_dir parent{}; @@ -335,7 +349,7 @@ result caps(const std::filesystem::path& directory) { bool close = false; if (l->relative != ".") { const int rc = kal_fs_open_dir(dir_of(*l), l->relative.data(), l->relative.size(), &d); - if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("asking about {}", directory.generic_string()))); + if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("asking about {}", generic_utf8(directory)))); close = true; } const kal_uintptr p = kal_fs_props(d); @@ -354,7 +368,7 @@ result caps(const std::filesystem::path& directory) { } result lock::acquire(const std::filesystem::path& file, kind k, bool wait) { - const std::string doing = std::format("locking {}", file.generic_string()); + const std::string doing = std::format("locking {}", generic_utf8(file)); auto l = where(file, "locking"); if (!l) return std::unexpected(l.error()); kal_file f{}; @@ -388,10 +402,10 @@ void lock::release() noexcept { result create_link(const std::filesystem::path& target, const std::filesystem::path& link, bool directory) { - const std::string doing = std::format("linking {} to {}", link.generic_string(), target.generic_string()); + const std::string doing = std::format("linking {} to {}", generic_utf8(link), generic_utf8(target)); auto l = where(link, "linking"); if (!l) return std::unexpected(l.error()); - const std::string t = target.generic_string(); + const std::string t = generic_utf8(target); const int rc = kal_fs_link_create(dir_of(*l), l->relative.data(), l->relative.size(), t.data(), t.size(), directory ? kal::fs::link_directory : 0); if (rc != kal_ok) return std::unexpected(from_kal(rc, doing)); @@ -399,7 +413,7 @@ result create_link(const std::filesystem::path& target, const std::filesys } result read_link(const std::filesystem::path& link) { - const std::string doing = std::format("reading the link {}", link.generic_string()); + const std::string doing = std::format("reading the link {}", generic_utf8(link)); auto l = where(link, "reading the link"); if (!l) return std::unexpected(l.error()); std::string out(4096, '\0'); @@ -416,7 +430,7 @@ result set_executable(const std::filesystem::path& file, bool executable) auto l = where(file, "marking"); if (!l) return std::unexpected(l.error()); const int rc = kal_fs_set_executable_at(dir_of(*l), l->relative.data(), l->relative.size(), executable ? 1 : 0); - if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("marking {}", file.generic_string()))); + if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("marking {}", generic_utf8(file)))); return {}; } @@ -427,8 +441,8 @@ result info(const std::filesystem::path& node, kal_u32 wanted, st kal_node_info i{}; i.self_size = sizeof i; const int rc = kal_fs_info(dir_of(*l), l->relative.data(), l->relative.size(), 0, wanted, &i); - if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("{} {}", doing, node.generic_string()))); - if (i.kind == kal_node_absent) return std::unexpected(error{ errc::not_found, std::format("{} {}", doing, node.generic_string()) }); + if (rc != kal_ok) return std::unexpected(from_kal(rc, std::format("{} {}", doing, generic_utf8(node)))); + if (i.kind == kal_node_absent) return std::unexpected(error{ errc::not_found, std::format("{} {}", doing, generic_utf8(node)) }); return i; } } // namespace @@ -443,12 +457,12 @@ result identity_of(const std::filesystem::path& node) { auto i = info(node, kal::fs::field::identity, "asking about"); if (!i) return std::unexpected(i.error()); if ((i->present & kal::fs::field::identity) == 0) - return std::unexpected(error{ errc::not_supported, std::format("the identity of {}", node.generic_string()) }); + return std::unexpected(error{ errc::not_supported, std::format("the identity of {}", generic_utf8(node)) }); return identity{ i->identity[0], i->identity[1] }; } result rename(const std::filesystem::path& from, const std::filesystem::path& to, bool replace) { - const std::string doing = std::format("renaming {} to {}", from.generic_string(), to.generic_string()); + const std::string doing = std::format("renaming {} to {}", generic_utf8(from), generic_utf8(to)); auto a = where(from, "renaming"); if (!a) return std::unexpected(a.error()); auto b = where(to, "renaming"); diff --git a/src/fs.cppm b/src/fs.cppm index 83323ea..dae62a6 100644 --- a/src/fs.cppm +++ b/src/fs.cppm @@ -39,6 +39,14 @@ std::vector preopens(); // the image runs upon. std::filesystem::path absolute(const std::filesystem::path& name); +// A name's text in UTF-8 and back, never through the process's code page. +// UTF-8 is what openkal's names are on every system; `path::string()` and +// `path(std::string)` use the ANSI code page on Windows, where a name it +// cannot spell throws and one it can is not UTF-8. +std::string utf8(const std::filesystem::path& name); +std::string generic_utf8(const std::filesystem::path& name); // with '/' separators +std::filesystem::path from_utf8(std::string_view text); + // NAMES IN THE HOST'S RULES, AND WITHOUT ASKING THE FILE SYSTEM. Each is lexical: // it reads the name and never the node, so a link is not followed and a name // that does not exist is answered as well as one that does (`identity_of' says diff --git a/src/ipc.cpp b/src/ipc.cpp index 914c32b..c5e5a02 100644 --- a/src/ipc.cpp +++ b/src/ipc.cpp @@ -19,7 +19,7 @@ struct place { kal_dir dir{}; bool owned = false; std::string doing; }; // The directory, opened, for a name to be bound in it. result directory_of(const std::filesystem::path& d, std::string_view name, std::string_view verb) { place p; - p.doing = std::format("{} {} in {}", verb, name, d.generic_string()); + p.doing = std::format("{} {} in {}", verb, name, fs::generic_utf8(d)); auto l = fs::locate(d); if (!l) return std::unexpected(error{ errc::not_found, p.doing + ": beneath no directory this program was given" }); p.dir = kal_dir{ static_cast(l->directory) }; @@ -131,7 +131,7 @@ result listener::listen(const std::filesystem::path& directory, std::s if (rc != kal_ok) return std::unexpected(from_kal(rc, doing)); listener out; out.listener_ = static_cast(l.h); - out.where_ = directory / std::filesystem::path(std::string(name)); + out.where_ = directory / fs::from_utf8(name); return out; } diff --git a/src/process.cpp b/src/process.cpp index d43eedc..2f00c73 100644 --- a/src/process.cpp +++ b/src/process.cpp @@ -163,27 +163,27 @@ result process::spawn(const spawn_options& o) { return std::unexpected(error{ errc::invalid, "a program's input cannot be this program's error output" }); // The program, by name or by its place. std::filesystem::path program = o.program; - if (!fs::names_a_place(o.program.generic_string())) { - auto found = env::find_executable(program.generic_string()); - if (!found) return std::unexpected(error{ errc::not_found, std::format("{} is not on PATH", program.generic_string()) }); + if (!fs::names_a_place(fs::generic_utf8(o.program))) { + auto found = env::find_executable(fs::generic_utf8(program)); + if (!found) return std::unexpected(error{ errc::not_found, std::format("{} is not on PATH", fs::generic_utf8(program)) }); program = *found; } std::error_code ec; program = fs::absolute(program).lexically_normal(); const auto where = fs::locate(program); if (!where) return std::unexpected(error{ errc::not_found, - std::format("starting {}: beneath no directory this program was given", program.generic_string()) }); + std::format("starting {}: beneath no directory this program was given", fs::generic_utf8(program)) }); const auto cwd_path = o.cwd.empty() ? std::filesystem::current_path(ec) : o.cwd; const auto cwd = fs::locate(cwd_path); if (!cwd) return std::unexpected(error{ errc::not_found, std::format("starting {}: {} is beneath no directory this program was given", - program.generic_string(), cwd_path.generic_string()) }); + fs::generic_utf8(program), fs::generic_utf8(cwd_path)) }); kal_dir work = as_dir(cwd->directory); bool close_work = false; if (cwd->relative != ".") { if (const int rc = kal_fs_open_dir(work, cwd->relative.data(), cwd->relative.size(), &work); rc != kal_ok) - return std::unexpected(from_kal(rc, std::format("starting {} in {}", program.generic_string(), cwd_path.generic_string()))); + return std::unexpected(from_kal(rc, std::format("starting {} in {}", fs::generic_utf8(program), fs::generic_utf8(cwd_path)))); close_work = true; } struct closer { kal_dir d; bool on; ~closer() { if (on) kal_fs_close_dir(d); } } work_guard{ work, close_work }; @@ -234,7 +234,7 @@ result process::spawn(const spawn_options& o) { // The vector, the first element in the system's own spelling: a program // that reads its own command line takes a forward slash for a switch. - std::string first = o.argv0.value_or(program.string()); + std::string first = o.argv0.value_or(fs::utf8(program)); if (!o.argv0 && host() == platform::windows) std::ranges::replace(first, '/', '\\'); std::vector argv_storage{ std::move(first) }; for (const auto& a : o.args) argv_storage.push_back(a); @@ -264,7 +264,7 @@ result process::spawn(const spawn_options& o) { if (n.file.h) kal_fs_close_file(n.file); else if (n.channel && n.child.h) kal_process_channel_close(n.child); } - if (rc != kal_ok) return fail_streams(from_kal(rc, std::format("starting {}", program.generic_string()))); + if (rc != kal_ok) return fail_streams(from_kal(rc, std::format("starting {}", fs::generic_utf8(program)))); if (unit) { s->job = job; s->has_job = true; } process p; p.state_ = std::move(s); diff --git a/src/windows/windows.cpp b/src/windows/windows.cpp index 38c9e2b..3eb5489 100644 --- a/src/windows/windows.cpp +++ b/src/windows/windows.cpp @@ -4,6 +4,7 @@ module; module mcpplibs.os.windows; import std; +import mcpplibs.os.fs; namespace mcpplibs::os::windows { @@ -25,7 +26,7 @@ bool elevated() { } std::optional on_dev_drive(const std::filesystem::path& directory) { - const std::string utf8 = directory.string(); + const std::string utf8 = fs::utf8(directory); const int n = MultiByteToWideChar(65001, 0, utf8.data(), static_cast(utf8.size()), nullptr, 0); if (n <= 0) return std::nullopt; std::wstring wide(static_cast(n), L'\0'); diff --git a/tests/fs_test.cpp b/tests/fs_test.cpp index 7c274ae..4f619fc 100644 --- a/tests/fs_test.cpp +++ b/tests/fs_test.cpp @@ -91,6 +91,25 @@ int main() { check(os::env::split_list(os::env::join_list(dirs)).size() == 2, "and joining is its inverse"); } + { + // Names the Windows code page cannot spell, both ways through the + // system: UTF-8 in and out on every host, never the ANSI code page. + const std::string name = "\xE4\xB8\xAD\xE6\x96\x87 \xE3\x81\xAB\xE3\x81\xBB\xE3\x82\x93 \xD0\xBA\xD0\xB8\xD1\x80 \xE0\xA4\x95"; + check(os::fs::utf8(os::fs::from_utf8(name)) == name, "a UTF-8 name survives the trip through a path"); + const auto sub = dir / os::fs::from_utf8(name); + stdfs::create_directories(sub); + const auto f = sub / os::fs::from_utf8(name + ".txt"); + check(os::fs::write(f, "\xE4\xBD\xA0\xE5\xA5\xBD").has_value(), "a file whose name is not ASCII is written"); + check(os::fs::read(f).value_or("") == "\xE4\xBD\xA0\xE5\xA5\xBD", "and read back"); + check(stdfs::exists(f), "and the system sees it under that name"); + check(os::fs::write_atomic(f, "again").has_value() && os::fs::read(f).value_or("") == "again", "replaced all at once"); + const auto moved = sub / os::fs::from_utf8("\xE6\x96\xB0 " + name); + check(os::fs::rename(f, moved, false).has_value() && stdfs::exists(moved), "and renamed"); + const auto missing = os::fs::read(sub / os::fs::from_utf8("\xE6\xB2\xA1\xE6\x9C\x89")); + check(!missing && missing.error().message().contains("\xE6\xB2\xA1\xE6\x9C\x89"), "an error names the file in UTF-8"); + check(os::fs::generic_utf8(os::fs::absolute(sub)).ends_with(name), "absolute keeps the name"); + } + if (caps && caps->remove_open) { std::error_code ec; stdfs::remove_all(dir, ec);