0.2.0: what mcpp's operating-system layer carried

.github/workflows/ci.yml

@@ -182,6 +182,8 @@ jobs:
       run:
         shell: bash
     steps:
+      - uses: actions/checkout@v4
+        with: { sparse-checkout: mcpp.toml, sparse-checkout-cone-mode: false }
       - uses: actions/download-artifact@v4
         with: { name: os-test-images, path: out }
       - name: Install xlings (Unix)
@@ -202,7 +204,10 @@ jobs:
           mcpp self config --mirror GLOBAL
           mcpp toolchain install llvm 23.1.3
           mcpp toolchain default llvm@23.1.3
-          git clone --quiet --depth 1 --branch openkal-0.17.0 https://github.com/mcpplibs/openkal-image.git "$RUNNER_TEMP/oi"
+          # The okrun of the openkal-tools version this package's manifest names.
+          tools=$(sed -n 's/^openkal-tools = { version = "\([^"]*\)".*/\1/p' mcpp.toml)
+          [ -n "$tools" ] || { echo "::error::mcpp.toml names no openkal-tools version"; exit 1; }
+          git clone --quiet --depth 1 --branch "$tools" https://github.com/mcpplibs/openkal-image.git "$RUNNER_TEMP/oi"
           ( cd "$RUNNER_TEMP/oi/tools" && mcpp build --release )
           okrun=$(find "$RUNNER_TEMP/oi/target" -path '*bin*' \( -name okrun -o -name okrun.exe \) -type f | head -1)
           [ -n "$okrun" ] || { echo "::error::okrun did not build"; exit 1; }

README.md

@@ -6,7 +6,7 @@ with compile-time platform constants and a few platform modules.
 
 ```toml
 [dependencies]
-os = { namespace = "mcpplibs", version = "0.1.1" }
+os = { namespace = "mcpplibs", version = "0.2.0" }
 ```
 
 ```cpp
@@ -41,15 +41,15 @@ platform modules.
 | `mcpplibs.os.platform` | `PLATFORM`, `ARCH`, `is_linux` / `is_macos` / `is_windows` / `is_posix`, `exe_suffix`, `path_list_separator`; `host()`, a constant expression on a native target |
 | `mcpplibs.os.error` | `errc` (openkal's closed set of conditions), `error`, `result<T>` |
 | `mcpplibs.os.env` | variables (names without case on Windows), arguments, the program's own image, lists of directories in the host's separator (`split_list`, `join_list`), PATH search with PATHEXT |
-| `mcpplibs.os.fs` | `write_atomic`, `publish` (no replace), `caps`, `lock`, links, `set_executable`, `stamp_of`, `identity_of`, `locate`, `absolute`, `names_a_place`; names in the host's rules without the file system: `generic`, `native`, `is_absolute`, `same`, `within` |
-| `mcpplibs.os.process` | `spawn`, `run`, pipes, output onto this program's error output, units and `end_unit`, bound lifetime, bounded waits and reads; `self`, `describe`, `alive` |
-| `mcpplibs.os.interrupt` | `requested`, `wait`, `end_as_requested` |
-| `mcpplibs.os.stdio` | the three streams as bytes |
-| `mcpplibs.os.terminal` | `interactive`, `size`, `sequences`, `color`, `raw_mode` |
+| `mcpplibs.os.fs` | `write_atomic`, `publish` (no replace), `append`, `sync`, `caps`, `lock`, links, `set_executable`, `stamp_of`, `identity_of`, `owner_of`, `writable`, `locate`, `absolute`, `names_a_place`; what a volume does, measured (`measure`, `volume_type`), and the operations that follow it (`place`, `exchange`); names in the host's rules without the file system: `generic`, `native`, `extended`, `is_absolute`, `same`, `within` |
+| `mcpplibs.os.process` | `spawn`, `run`, `exec`, a command line joined as it is for the command interpreter (`one_string`), pipes, both outputs on one pipe, output onto this program's error output, units and `end_unit`, bound lifetime, bounded waits and reads; `self`, `describe`, `alive` |
+| `mcpplibs.os.interrupt` | `requested`, `wait`, `end_as_requested`, `end_as_output_closed` |
+| `mcpplibs.os.stdio` | the three streams as bytes; a bounded read of input; the output sent to the error stream for a while (`output_to_error`) |
+| `mcpplibs.os.terminal` | `interactive`, `size`, `sequences`, `color`, `raw_mode` (with or without the interrupting keystroke), `foreground` |
 | `mcpplibs.os.dirs` | `home`, `config`, `data`, `state`, `cache`, `runtime`, `temp` |
 | `mcpplibs.os.ipc` | a listener and a connection at a name in a directory; `single_instance` |
 | `mcpplibs.os.net` | loopback TCP |
-| `mcpplibs.os.machine` | the machine's architecture (through Rosetta and WoW64), memory |
+| `mcpplibs.os.machine` | the machine's architecture (through Rosetta and WoW64), memory and the memory available now, processors (logical, physical, of differing speed) |
 | `mcpplibs.os.random` | bytes from the system's source of unpredictable bytes; `hex` |
 | `mcpplibs.os.library` | shared libraries at run time: `open` (a path or a name the system searches for), `loaded` (one the process has, without loading it), a handle's `symbol` / `function<F>`, `process_symbol` (as the loader resolves one), `suffix` |
 | `mcpplibs.os.linux` / `.macos` / `.windows` / `.posix` | platform modules: declared on every target, defined only on their own |

mcpp.toml

@@ -1,7 +1,7 @@
 [package]
 namespace   = "mcpplibs"
 name        = "os"
-version     = "0.1.2"
+version     = "0.2.0"
 description = "mcpplibs.os: a C++ operating-system library over openkal --- processes, files, environment, terminals, directories and local IPC, with compile-time platform constants and a few platform modules."
 license     = "Apache-2.0"
 authors     = ["mcpplibs"]
@@ -12,16 +12,16 @@ repo        = "https://github.com/mcpplibs/os"
 # own C and C++ runtime. A program that wants the standalone arrangement, or
 # the image arrangement of `<arch>-openkal`, selects it in its own manifest.
 [dependencies]
-openkal = "0.17.0"
+openkal = "0.17.1"
 
 [target.linux.dependencies]
-openkal-linux = "0.18.0"
+openkal-linux = "0.18.2"
 
 [target.macos.dependencies]
-openkal-macos = "0.15.0"
+openkal-macos = "0.15.1"
 
 [target.windows.dependencies]
-openkal-windows = "0.13.0"
+openkal-windows = "0.13.3"
 
 [targets.os]
 kind = "lib"
@@ -62,7 +62,7 @@ sources = ["src/targets/openkal.cppm", "src/openkal/*.cpp"]
 # The tests of this package, built as images, carry the C library and the C++
 # run-time an image holds, and the loader that runs them.
 [target.'cfg(os = "openkal")'.dev-dependencies]
-openkal-llvm-runtime = "0.16.0"
+openkal-llvm-runtime = "0.16.3"
 
 [target.'cfg(os = "openkal")'.build-dependencies]
-openkal-tools = { version = "0.1.0", tools = ["okld", "okrun"] }
+openkal-tools = { version = "0.1.1", tools = ["okld", "okrun"] }

src/fs.cpp

@@ -475,3 +475,184 @@ result<void> rename(const std::filesystem::path& from, const std::filesystem::pa
 }
 
 }  // namespace mcpplibs::os::fs
+
+// --- 0.2.0: appending, durability, long names, measured volumes ----------------
+
+namespace mcpplibs::os::fs {
+
+result<void> append(const std::filesystem::path& file, std::string_view bytes) {
+    const std::string doing = std::format("appending to {}", generic_utf8(file));
+    auto l = where(file, "appending to");
+    if (!l) return std::unexpected(l.error());
+    opened_file f;
+    const auto flags = kal::fs::open::write | kal::fs::open::create | kal::fs::open::append;
+    const int rc = kal_fs_open(dir_of(*l), l->relative.data(), l->relative.size(), flags.bits, &f.f);
+    if (rc != kal_ok) return std::unexpected(from_kal(rc, doing));
+    // One write: a stream that appends places each write whole at the end.
+    const kal_intptr n = kal_stream_write(kal_fs_stream(f.f), bytes.data(), bytes.size());
+    if (n < 0 || static_cast<std::size_t>(n) != bytes.size())
+        return std::unexpected(from_kal(n < 0 ? static_cast<int>(-n) : kal_err_io, doing));
+    return {};
+}
+
+result<void> sync(const std::filesystem::path& node) { return sys::sync(node); }
+
+std::string windows_extended(std::string_view in) {
+    std::string s(in);
+    for (auto& c : s) if (c == '/') c = '\\';
+    // Already extended (`\\?\`) or a device name (`\\.\`): nothing is rewritten.
+    if (s.starts_with("\\\\?\\") || s.starts_with("\\\\.\\")) return s;
+    std::string prefix, rest;
+    if (s.starts_with("\\\\")) {
+        prefix = "\\\\?\\UNC\\";
+        rest = s.substr(2);
+    } else if (s.size() >= 3 && std::isalpha(static_cast<unsigned char>(s[0])) && s[1] == ':' && s[2] == '\\') {
+        prefix = "\\\\?\\";
+        rest = s;
+    } else {
+        return std::string(in);   // relative or rootless
+    }
+    // The prefix turns off `.' and `..', so they are resolved here; `..' never
+    // climbs above the drive or the share.
+    std::vector<std::string> parts;
+    const std::size_t keep = prefix.ends_with("UNC\\") ? 2 : 1;
+    std::size_t start = 0;
+    while (start <= rest.size()) {
+        auto end = rest.find('\\', start);
+        if (end == std::string::npos) end = rest.size();
+        std::string comp = rest.substr(start, end - start);
+        start = end + 1;
+        if (comp.empty() || comp == ".") continue;
+        if (comp == "..") { if (parts.size() > keep) parts.pop_back(); continue; }
+        parts.push_back(std::move(comp));
+    }
+    std::string out = prefix;
+    for (std::size_t i = 0; i < parts.size(); ++i) { if (i) out += '\\'; out += parts[i]; }
+    if (parts.size() == 1 && keep == 1) out += '\\';   // `\\?\C:\`, not `\\?\C:`
+    return out;
+}
+
+std::filesystem::path extended(const std::filesystem::path& name) {
+    if (host() != platform::windows || name.empty()) return name;
+    std::error_code ec;
+    const auto abs = std::filesystem::absolute(name, ec);
+    if (ec) return name;
+    const auto u8 = abs.generic_u8string();
+    const auto spelled = windows_extended(std::string(reinterpret_cast<const char*>(u8.data()), u8.size()));
+    return std::filesystem::path(std::u8string(reinterpret_cast<const char8_t*>(spelled.data()), spelled.size()));
+}
+
+std::string volume_type(const std::filesystem::path& directory) { return sys::volume_type(directory); }
+owner owner_of(const std::filesystem::path& node) { return sys::owner_of(node); }
+bool writable(const std::filesystem::path& node) { return sys::writable(node); }
+
+behaviour measure_now(const std::filesystem::path& directory) {
+    behaviour b;
+    b.type = sys::volume_type(directory);
+    std::error_code ec;
+    std::random_device random;
+    const auto scratch = directory / std::format(".os-measure-{:08x}{:08x}", random(), random());
+    if (!std::filesystem::create_directory(scratch, ec) || ec) return b;
+    const auto touch = [&](const std::filesystem::path& p) { return write(p, "x").has_value(); };
+    if (touch(scratch / "a") && touch(scratch / "c")) {
+        b.measured = true;
+        if (const auto r = rename(scratch / "a", scratch / "b", false); !r) {
+            b.rename_no_replace = r.error().code != errc::not_supported;
+            std::filesystem::rename(scratch / "a", scratch / "b", ec);
+        }
+        if (const int e = sys::exchange(scratch / "b", scratch / "c"); e != 0)
+            b.exchange = !sys::exchange_unsupported(e);
+        ec.clear();
+        std::filesystem::create_hard_link(scratch / "b", scratch / "d", ec);
+        b.hard_links = !ec;
+        ec.clear();
+        std::filesystem::create_symlink("b", scratch / "e", ec);
+        b.symbolic_links = !ec;
+        ec.clear();
+        if (touch(scratch / "Case")) b.case_sensitive = !std::filesystem::exists(scratch / "cASE", ec);
+    }
+    std::filesystem::remove_all(scratch, ec);
+    return b;
+}
+
+behaviour measure(const std::filesystem::path& directory) {
+    static std::mutex guard;
+    static std::map<std::uint64_t, behaviour> known;
+    const auto id = identity_of(directory);
+    const std::uint64_t volume = id ? id->volume : 0;
+    {
+        std::scoped_lock g(guard);
+        if (volume != 0)
+            if (const auto it = known.find(volume); it != known.end()) return it->second;
+    }
+    auto b = measure_now(directory);
+    if (volume != 0 && b.measured) {
+        std::scoped_lock g(guard);
+        known.emplace(volume, b);
+    }
+    return b;
+}
+
+result<void> place(const std::filesystem::path& from, const std::filesystem::path& to) {
+    const std::string doing = std::format("placing {} as {}", generic_utf8(from), generic_utf8(to));
+    auto parent = to.parent_path();
+    if (parent.empty()) parent = ".";
+    const auto b = measure(parent);
+    const auto failed = [&](std::error_code e) {
+        return std::unexpected(error{ e == std::errc::file_exists ? errc::exists : errc::io,
+                                      std::format("{}: {}", doing, e.message()) });
+    };
+    if (b.rename_no_replace) {
+        const auto r = rename(from, to, false);
+        if (r || r.error().code != errc::not_supported) return r;
+    }
+    std::error_code ec;
+    const auto status = std::filesystem::symlink_status(from, ec);
+    if (ec) return failed(ec);
+    if (status.type() == std::filesystem::file_type::symlink) {
+        std::filesystem::copy_symlink(from, to, ec);
+        if (ec) return failed(ec);
+        std::filesystem::remove(from, ec);
+        return {};
+    }
+    if (status.type() != std::filesystem::file_type::directory && b.hard_links) {
+        std::filesystem::create_hard_link(from, to, ec);
+        if (ec) return failed(ec);
+        std::filesystem::remove(from, ec);
+        return {};
+    }
+    const auto there = std::filesystem::symlink_status(to, ec);
+    if (ec && ec != std::errc::no_such_file_or_directory) return failed(ec);
+    if (std::filesystem::exists(there)) return failed(std::make_error_code(std::errc::file_exists));
+    std::filesystem::rename(from, to, ec);
+    if (ec) return failed(ec);
+    return {};
+}
+
+result<void> exchange(const std::filesystem::path& a, const std::filesystem::path& b) {
+    const std::string doing = std::format("exchanging {} and {}", generic_utf8(a), generic_utf8(b));
+    auto parent = b.parent_path();
+    if (parent.empty()) parent = ".";
+    if (measure(parent).exchange) {
+        const int e = sys::exchange(a, b);
+        if (e == 0) return {};
+        if (!sys::exchange_unsupported(e))
+            return std::unexpected(error{ errc::io, std::format("{}: {}", doing, std::generic_category().message(e)), e });
+    }
+    std::random_device random;
+    const auto aside = parent / std::format(".os-exchange-{:08x}{:08x}", random(), random());
+    std::error_code ec;
+    std::filesystem::rename(b, aside, ec);
+    if (ec) return std::unexpected(error{ errc::io, std::format("{}: {}", doing, ec.message()) });
+    std::filesystem::rename(a, b, ec);
+    if (ec) {
+        std::error_code back;
+        std::filesystem::rename(aside, b, back);
+        return std::unexpected(error{ errc::io, std::format("{}: {}", doing, ec.message()) });
+    }
+    std::filesystem::rename(aside, a, ec);
+    if (ec) return std::unexpected(error{ errc::io, std::format("{}: {}", doing, ec.message()) });
+    return {};
+}
+
+}  // namespace mcpplibs::os::fs

src/fs.cppm

@@ -92,6 +92,24 @@ result<void> write_atomic(const std::filesystem::path& file, std::string_view by
 struct published { bool atomic = false; };
 result<published> publish(const std::filesystem::path& file, std::string_view bytes);
 
+// Appended in one write that another program appending to the same file cannot
+// split; the file is created when it is not there.
+result<void> append(const std::filesystem::path& file, std::string_view bytes);
+
+// What was written to the file, or to the directory's entries, is durable: it
+// survives the machine stopping. A volume that keeps nothing durable (a
+// directory on Windows, an image's preopened tree) has nothing to do.
+result<void> sync(const std::filesystem::path& node);
+
+// The name in the spelling that reaches past the system's limit on a name's
+// length: `\\?\' (or `\\?\UNC\') before an absolute name on Windows, with `.'
+// and `..' resolved because that spelling no longer resolves them; the name
+// itself everywhere else, and for a name that is not absolute.
+std::filesystem::path extended(const std::filesystem::path& name);
+// The Windows rule of `extended', of an absolute name in either separator, so
+// that it is a fact on every system.
+std::string windows_extended(std::string_view absolute_name);
+
 // --- what a volume can do ----------------------------------------------------
 
 struct capabilities {
@@ -109,6 +127,52 @@ struct capabilities {
 // Of the volume the directory is on.
 result<capabilities> caps(const std::filesystem::path& directory);
 
+// WHAT THE VOLUME DOES, MEASURED (0.2.0). A volume is whatever a person has
+// --- ext4, APFS, NTFS, and NFSv3, SMB, FAT, a FUSE mount, WSL's drvfs --- and
+// what it claims is not always what it does: NFSv3 answers a rename that may
+// not replace with EINVAL. Measured once per volume per program, with a
+// scratch directory in `directory' that is removed after; `measured' false
+// when the directory cannot be written, and the system's own answers stand.
+struct behaviour {
+    std::string type;                 // "ext4", "nfs", "apfs", "NTFS", ...; "" unknown
+    bool rename_no_replace = true;    // a rename that refuses an existing name, atomically
+    bool exchange          = true;    // two entries swapped atomically
+    bool hard_links        = true;
+    bool symbolic_links    = true;
+    bool case_sensitive    = true;
+    bool measured          = false;
+    // Whether every operation below is the volume's own.
+    bool native() const { return rename_no_replace && exchange && hard_links && symbolic_links; }
+};
+behaviour measure(const std::filesystem::path& directory);
+// The same, measured now and not remembered.
+behaviour measure_now(const std::filesystem::path& directory);
+// The volume's kind, as the system names it; "" where it does not.
+std::string volume_type(const std::filesystem::path& directory);
+
+// Who owns a node: this program's user, the system (the superuser; on Windows
+// SYSTEM or the Administrators), or another user.
+enum class owner { this_user, system, another_user, unknown };
+owner owner_of(const std::filesystem::path& node);
+// Whether this program's user may write the node (POSIX `access(W_OK)';
+// elsewhere what the system's own rules decide when the write is made, so
+// true).
+bool writable(const std::filesystem::path& node);
+
+// An entry made beside `to' (on its volume) published under `to', never over an
+// entry there: by a rename that refuses replacement where the volume has one;
+// else by a hard link (a file) or the same symbolic link again, each refused
+// as atomically when the name exists, the scratch name then removed; else, for
+// a directory, by a rename onto a name that is not there --- the one window, a
+// name made in between, is the caller's under its own exclusion.
+result<void> place(const std::filesystem::path& from, const std::filesystem::path& to);
+
+// `a' and `b' (both present, on one volume) trade places: atomically where the
+// volume exchanges; else `b' is moved aside, `a' renamed onto `b''s name and the
+// old `b' put at `a' --- a reader of `b' meets the name missing for that moment,
+// never a half of either.
+result<void> exchange(const std::filesystem::path& a, const std::filesystem::path& b);
+
 // --- exclusion -----------------------------------------------------------------
 
 // A lock upon a file, held until released or until the holder ends however it
@@ -164,3 +228,14 @@ result<void> rename(const std::filesystem::path& from, const std::filesystem::pa
                     bool replace = true);
 
 }  // namespace mcpplibs::os::fs
+
+// Each system's own (src/<os>/fs.cpp).
+namespace mcpplibs::os::fs::sys {
+result<void> sync(const std::filesystem::path& node);
+// 0, or the system's condition; `exchange_unsupported' says the volume declined.
+int exchange(const std::filesystem::path& a, const std::filesystem::path& b);
+bool exchange_unsupported(int condition);
+std::string volume_type(const std::filesystem::path& directory);
+owner owner_of(const std::filesystem::path& node);
+bool writable(const std::filesystem::path& node);
+}  // namespace mcpplibs::os::fs::sys

src/interrupt.cpp

@@ -51,4 +51,6 @@ void end_as_requested() {
     kal_process_end_as_requested();
 }
 
+void end_as_output_closed() { sys::end_as_output_closed(); }
+
 }  // namespace mcpplibs::os::interrupt