From 677c8e828cb3d3924d2363d4c061855c050f12e3 Mon Sep 17 00:00:00 2001 From: Norman Feske Date: Mon, 23 Jan 2023 16:42:46 +0100 Subject: [PATCH] depot_download,depot_query: support system images This patch enhances the depot_download subsystem with support for downloading and querying system images. The installation ROM support the following two now download types: Internally, the depot-download subsystem employs the depot-query component to determine the missing depot content. This component accepts the following two new queries: If present in the query, depot_query generates reports labeled as "images" and "image_index" respectively. The also tracks the completion of each job depending on the depot- query results, so that the final report contains a result for each installation item requested. Prior this patch, the inactivity of the depot-download manager (indicated by an empty state report) was interpreted as success. But that prevents the proper association of results and requested installation items. Issue #4744 --- repos/gems/include/depot/archive.h | 32 +++- .../raw/depot_download/depot_download.config | 6 + .../depot_download_manager/gen_depot_query.cc | 47 +++-- .../app/depot_download_manager/gen_extract.cc | 3 + .../src/app/depot_download_manager/import.h | 87 +++++++-- .../gems/src/app/depot_download_manager/job.h | 1 + .../src/app/depot_download_manager/main.cc | 58 ++++-- repos/gems/src/app/depot_query/main.cc | 21 +++ repos/gems/src/app/depot_query/main.h | 35 +++- .../src/app/depot_query/query_image_index.cc | 171 ++++++++++++++++++ repos/gems/src/app/depot_query/target.mk | 2 +- 11 files changed, 397 insertions(+), 66 deletions(-) create mode 100644 repos/gems/src/app/depot_query/query_image_index.cc diff --git a/repos/gems/include/depot/archive.h b/repos/gems/include/depot/archive.h index f25fcd0fe9..e3ded7b67e 100644 --- a/repos/gems/include/depot/archive.h +++ b/repos/gems/include/depot/archive.h @@ -30,7 +30,7 @@ struct Depot::Archive typedef String<80> Name; typedef String<40> Version; - enum Type { PKG, RAW, SRC }; + enum Type { PKG, RAW, SRC, IMAGE }; struct Unknown_archive_type : Exception { }; @@ -81,9 +81,10 @@ struct Depot::Archive typedef String<8> Name; Name const name = _path_element(path, 1); - if (name == "src") return SRC; - if (name == "pkg") return PKG; - if (name == "raw") return RAW; + if (name == "src") return SRC; + if (name == "pkg") return PKG; + if (name == "raw") return RAW; + if (name == "image") return IMAGE; throw Unknown_archive_type(); } @@ -96,7 +97,23 @@ struct Depot::Archive return _path_element(path, 1) == "index"; } - static Name name (Path const &path) { return _path_element(path, 2); } + /** + * Return true if 'path' refers to a system-image index file + */ + static bool image_index(Path const &path) + { + return _path_element(path, 1) == "image" && name(path) == "index"; + } + + /** + * Return true if 'path' refers to a system image + */ + static bool image(Path const &path) + { + return _path_element(path, 1) == "image" && name(path) != "index"; + } + + static Name name (Path const &path) { return _path_element (path, 2); } static Version version (Path const &path) { return _path_element(path, 3); } static Version index_version(Path const &path) { return _path_element(path, 2); } @@ -108,10 +125,9 @@ struct Depot::Archive */ static Archive::Path download_file_path(Archive::Path path) { - return Archive::index(path) ? Archive::Path(path, ".xz") - : Archive::Path(path, ".tar.xz"); + return (index(path) || image_index(path)) ? Path(path, ".xz") + : Path(path, ".tar.xz"); } - }; #endif /* _INCLUDE__DEPOT__ARCHIVE_H_ */ diff --git a/repos/gems/recipes/raw/depot_download/depot_download.config b/repos/gems/recipes/raw/depot_download/depot_download.config index 1d23799204..ab99cbe0b7 100644 --- a/repos/gems/recipes/raw/depot_download/depot_download.config +++ b/repos/gems/recipes/raw/depot_download/depot_download.config @@ -28,6 +28,10 @@ report="dynamic -> depot_query -> dependencies"/> + + + + diff --git a/repos/gems/src/app/depot_download_manager/gen_depot_query.cc b/repos/gems/src/app/depot_download_manager/gen_depot_query.cc index efe7e56b16..00469e1882 100644 --- a/repos/gems/src/app/depot_download_manager/gen_depot_query.cc +++ b/repos/gems/src/app/depot_download_manager/gen_depot_query.cc @@ -51,11 +51,14 @@ void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml, return failed; }; - installation.for_each_sub_node("archive", [&] (Xml_node archive) { - - if (job_failed(archive)) - return; + auto for_each_install_sub_node = [&] (auto node_type, auto const &fn) + { + installation.for_each_sub_node(node_type, [&] (Xml_node node) { + if (!job_failed(node)) + fn(node); }); + }; + for_each_install_sub_node("archive", [&] (Xml_node const &archive) { xml.node("dependencies", [&] () { xml.attribute("path", archive.attribute_value("path", Archive::Path())); xml.attribute("source", archive.attribute_value("source", true)); @@ -63,22 +66,40 @@ void Depot_download_manager::gen_depot_query_start_content(Xml_generator &xml, }); }); - installation.for_each_sub_node("index", [&] (Xml_node index) { - - if (job_failed(index)) + for_each_install_sub_node("index", [&] (Xml_node const &index) { + Archive::Path const path = index.attribute_value("path", Archive::Path()); + if (!Archive::index(path)) { + warning("malformed index path '", path, "'"); return; - + } xml.node("index", [&] () { - Archive::Path const path = index.attribute_value("path", Archive::Path()); - if (!Archive::index(path)) { - warning("malformed index path '", path, "'"); - return; - } xml.attribute("user", Archive::user(path)); xml.attribute("version", Archive::_path_element(path, 2)); }); }); + for_each_install_sub_node("image", [&] (Xml_node const &image) { + Archive::Path const path = image.attribute_value("path", Archive::Path()); + if (!Archive::image(path)) { + warning("malformed image path '", path, "'"); + return; + } + xml.node("image", [&] () { + xml.attribute("user", Archive::user(path)); + xml.attribute("name", Archive::name(path)); + }); + }); + + for_each_install_sub_node("image_index", [&] (Xml_node const &image_index) { + Archive::Path const path = image_index.attribute_value("path", Archive::Path()); + if (!Archive::index(path) && Archive::name(path) != "index") { + warning("malformed image-index path '", path, "'"); + return; + } + xml.node("image_index", [&] () { + xml.attribute("user", Archive::user(path)); }); + }); + if (next_user.valid()) xml.node("user", [&] () { xml.attribute("name", next_user); }); }); diff --git a/repos/gems/src/app/depot_download_manager/gen_extract.cc b/repos/gems/src/app/depot_download_manager/gen_extract.cc index f1947730b4..3a5406479c 100644 --- a/repos/gems/src/app/depot_download_manager/gen_extract.cc +++ b/repos/gems/src/app/depot_download_manager/gen_extract.cc @@ -65,6 +65,9 @@ void Depot_download_manager::gen_extract_start_content(Xml_generator &xml, if (Archive::index(path)) xml.attribute("name", Archive::index_version(path)); + + if (Archive::image_index(path)) + xml.attribute("name", "index"); }); }); }); diff --git a/repos/gems/src/app/depot_download_manager/import.h b/repos/gems/src/app/depot_download_manager/import.h index d5c41f85cb..1e16a723e8 100644 --- a/repos/gems/src/app/depot_download_manager/import.h +++ b/repos/gems/src/app/depot_download_manager/import.h @@ -125,8 +125,70 @@ class Depot_download_manager::Import return result; } + static Archive::Path _depdendency_path(Xml_node const &item) + { + return item.attribute_value("path", Archive::Path()); + } + + static Archive::Path _index_path(Xml_node const &item) + { + return Path(item.attribute_value("user", Archive::User()), "/index/", + item.attribute_value("version", Archive::Version())); + } + + static Archive::Path _image_path(Xml_node const &item) + { + return Path(item.attribute_value("user", Archive::User()), "/image/", + item.attribute_value("name", Archive::Name())); + } + + static Archive::Path _image_index_path(Xml_node const &item) + { + return Path(item.attribute_value("user", Archive::User()), "/image/index"); + } + + template + static void _for_each_missing_depot_path(Xml_node const &dependencies, + Xml_node const &index, + Xml_node const &image, + Xml_node const &image_index, + FN const &fn) + { + dependencies.for_each_sub_node("missing", [&] (Xml_node const &item) { + fn(_depdendency_path(item)); }); + + index.for_each_sub_node("missing", [&] (Xml_node const &item) { + fn(_index_path(item)); }); + + image.for_each_sub_node("missing", [&] (Xml_node const &item) { + fn(_image_path(item)); }); + + image_index.for_each_sub_node("missing", [&] (Xml_node const &item) { + fn(_image_index_path(item)); }); + } + public: + template + static void for_each_present_depot_path(Xml_node const &dependencies, + Xml_node const &index, + Xml_node const &image, + Xml_node const &image_index, + FN const &fn) + { + dependencies.for_each_sub_node("present", [&] (Xml_node const &item) { + fn(_depdendency_path(item)); }); + + index.for_each_sub_node("index", [&] (Xml_node const &item) { + fn(_index_path(item)); }); + + image.for_each_sub_node("image", [&] (Xml_node const &item) { + fn(_image_path(item)); }); + + image_index.for_each_sub_node("present", [&] (Xml_node const &item) { + fn(_image_index_path(item)); }); + } + /** * Constructor * @@ -139,26 +201,17 @@ class Depot_download_manager::Import * a future iteration. */ Import(Allocator &alloc, Archive::User const &user, - Xml_node dependencies, Xml_node index) + Xml_node const &dependencies, + Xml_node const &index, + Xml_node const &image, + Xml_node const &image_index) : _alloc(alloc) { - dependencies.for_each_sub_node("missing", [&] (Xml_node item) { - Archive::Path const path = item.attribute_value("path", Archive::Path()); - if (Archive::user(path) == user) - new (alloc) Item(_items, path); - }); - - index.for_each_sub_node("missing", [&] (Xml_node item) { - - Archive::Path const - path(item.attribute_value("user", Archive::User()), - "/index/", - item.attribute_value("version", Archive::Version())); - - if (Archive::user(path) == user) - new (alloc) Item(_items, path); - }); + _for_each_missing_depot_path(dependencies, index, image, image_index, + [&] (Archive::Path const &path) { + if (Archive::user(path) == user) + new (alloc) Item(_items, path); }); } ~Import() diff --git a/repos/gems/src/app/depot_download_manager/job.h b/repos/gems/src/app/depot_download_manager/job.h index 2bda7d64e5..261b2520b7 100644 --- a/repos/gems/src/app/depot_download_manager/job.h +++ b/repos/gems/src/app/depot_download_manager/job.h @@ -30,6 +30,7 @@ struct Depot_download_manager::Job : List_model::Element { bool started = false; bool failed = false; + bool done = false; Archive::Path const path; diff --git a/repos/gems/src/app/depot_download_manager/main.cc b/repos/gems/src/app/depot_download_manager/main.cc index 6c6b1a16ba..ac04270dc2 100644 --- a/repos/gems/src/app/depot_download_manager/main.cc +++ b/repos/gems/src/app/depot_download_manager/main.cc @@ -60,6 +60,8 @@ struct Depot_download_manager::Main : Import::Download_progress Attached_rom_dataspace _installation { _env, "installation" }; Attached_rom_dataspace _dependencies { _env, "dependencies" }; Attached_rom_dataspace _index { _env, "index" }; + Attached_rom_dataspace _image { _env, "image" }; + Attached_rom_dataspace _image_index { _env, "image_index" }; Attached_rom_dataspace _init_state { _env, "init_state" }; Attached_rom_dataspace _fetchurl_progress { _env, "fetchurl_progress" }; @@ -143,15 +145,18 @@ struct Depot_download_manager::Main : Import::Download_progress else { _jobs.for_each([&] (Job const &job) { - if (!job.started) + if (!job.started && !job.done) return; - /* - * If a job has been started and has not failed, it must - * have succeeded at the time when the import is finished. - */ - char const *type = Archive::index(job.path) ? "index" : "archive"; - xml.node(type, [&] () { + auto type = [] (Archive::Path const &path) + { + if (Archive::index(path)) return "index"; + if (Archive::image(path)) return "image"; + if (Archive::image_index(path)) return "image_index"; + return "archive"; + }; + + xml.node(type(job.path), [&] () { xml.attribute("path", job.path); xml.attribute("state", job.failed ? "failed" : "done"); }); @@ -259,6 +264,8 @@ struct Depot_download_manager::Main : Import::Download_progress { _dependencies .sigh(_query_result_handler); _index .sigh(_query_result_handler); + _image .sigh(_query_result_handler); + _image_index .sigh(_query_result_handler); _current_user .sigh(_query_result_handler); _init_state .sigh(_init_state_handler); _verified .sigh(_init_state_handler); @@ -353,6 +360,8 @@ void Depot_download_manager::Main::_handle_query_result() _dependencies.update(); _index.update(); + _image.update(); + _image_index.update(); _current_user.update(); /* validate completeness of depot-user info */ @@ -385,14 +394,21 @@ void Depot_download_manager::Main::_handle_query_result() Xml_node const dependencies = _dependencies.xml(); Xml_node const index = _index.xml(); + Xml_node const image = _image.xml(); + Xml_node const image_index = _image_index.xml(); - if (dependencies.num_sub_nodes() == 0 && index.num_sub_nodes() == 0) - return; + /* mark jobs referring to existing depot content as unneccessary */ + Import::for_each_present_depot_path(dependencies, index, image, image_index, + [&] (Archive::Path const &path) { + _jobs.for_each([&] (Job &job) { + if (job.path == path) + job.done = true; }); }); - bool const missing_dependencies = dependencies.has_sub_node("missing"); - bool const missing_index_files = index.has_sub_node("missing"); - - if (!missing_dependencies && !missing_index_files) { + bool const complete = !dependencies.has_sub_node("missing") + && !index .has_sub_node("missing") + && !image .has_sub_node("missing") + && !image_index .has_sub_node("missing"); + if (complete) { log("installation complete."); _update_state_report(); return; @@ -408,9 +424,16 @@ void Depot_download_manager::Main::_handle_query_result() { Archive::User user { }; - if (missing_index_files) - index.with_optional_sub_node("missing", [&] (Xml_node missing) { - user = missing.attribute_value("user", Archive::User()); }); + auto assign_user_from_missing_xml_sub_node = [&] (Xml_node const node) + { + if (!user.valid()) + node.with_optional_sub_node("missing", [&] (Xml_node missing) { + user = missing.attribute_value("user", Archive::User()); }); + }; + + assign_user_from_missing_xml_sub_node(index); + assign_user_from_missing_xml_sub_node(image); + assign_user_from_missing_xml_sub_node(image_index); if (user.valid()) return user; @@ -435,7 +458,8 @@ void Depot_download_manager::Main::_handle_query_result() } /* start new import */ - _import.construct(_heap, _current_user_name(), dependencies, index); + _import.construct(_heap, _current_user_name(), + dependencies, index, image, image_index); /* mark imported jobs as started */ _import->for_each_download([&] (Archive::Path const &path) { diff --git a/repos/gems/src/app/depot_query/main.cc b/repos/gems/src/app/depot_query/main.cc index 13b94ecccf..d6bc3013cb 100644 --- a/repos/gems/src/app/depot_query/main.cc +++ b/repos/gems/src/app/depot_query/main.cc @@ -64,6 +64,9 @@ Depot_query::Main::_find_rom_in_pkg(File_content const &archives, result = result_from_pkg; }); break; + + case Archive::IMAGE: + break; } }); return result; @@ -208,6 +211,7 @@ void Depot_query::Main::_collect_source_dependencies(Archive::Path const &path, } case Archive::RAW: + case Archive::IMAGE: break; }; } @@ -243,6 +247,9 @@ void Depot_query::Main::_collect_binary_dependencies(Archive::Path const &path, case Archive::RAW: dependencies.record(path); break; + + case Archive::IMAGE: + break; }; } @@ -377,6 +384,20 @@ void Depot_query::Main::_query_index(Archive::User const &user, } +void Depot_query::Main::_query_image(Archive::User const &user, + Archive::Name const &name, + Xml_generator &xml) +{ + Directory::Path const image_path("depot/", user, "/image/", name); + char const *node_type = _root.directory_exists(image_path) + ? "image" : "missing"; + xml.node(node_type, [&] () { + xml.attribute("user", user); + xml.attribute("name", name); + }); +} + + void Component::construct(Genode::Env &env) { static Depot_query::Main main(env); diff --git a/repos/gems/src/app/depot_query/main.h b/repos/gems/src/app/depot_query/main.h index c033192725..f60449a741 100644 --- a/repos/gems/src/app/depot_query/main.h +++ b/repos/gems/src/app/depot_query/main.h @@ -267,6 +267,8 @@ struct Depot_query::Main Constructible_reporter _dependencies_reporter { }; Constructible_reporter _user_reporter { }; Constructible_reporter _index_reporter { }; + Constructible_reporter _image_reporter { }; + Constructible_reporter _image_index_reporter { }; template static void _construct_if(bool condition, Constructible &obj, ARGS &&... args) @@ -341,6 +343,8 @@ struct Depot_query::Main void _gen_index_node_rec(Xml_generator &, Xml_node const &, unsigned) const; void _gen_index_for_arch(Xml_generator &, Xml_node const &) const; void _query_index(Archive::User const &, Archive::Version const &, bool, Xml_generator &); + void _query_image(Archive::User const &, Archive::Name const &, Xml_generator &); + void _query_image_index(Xml_node const &, Xml_generator &); void _handle_config() { @@ -371,9 +375,6 @@ struct Depot_query::Main Xml_node const query = (query_from_rom ? _query_rom->xml() : config); - _construct_if(query.has_sub_node("scan"), - _scan_reporter, _env, "scan", "scan"); - /* * Use 64 KiB as initial report size to avoid the repetitive querying * when successively expanding the reporter. @@ -382,14 +383,18 @@ struct Depot_query::Main _blueprint_reporter, _env, "blueprint", "blueprint", Expanding_reporter::Initial_buffer_size { 64*1024 }); - _construct_if(query.has_sub_node("dependencies"), - _dependencies_reporter, _env, "dependencies", "dependencies"); + auto construct_reporter_if_needed = [&] (auto &reporter, auto query_type) + { + _construct_if(query.has_sub_node(query_type), + reporter, _env, query_type, query_type); + }; - _construct_if(query.has_sub_node("user"), - _user_reporter, _env, "user", "user"); - - _construct_if(query.has_sub_node("index"), - _index_reporter, _env, "index", "index"); + construct_reporter_if_needed(_scan_reporter, "scan"); + construct_reporter_if_needed(_dependencies_reporter, "dependencies"); + construct_reporter_if_needed(_user_reporter, "user"); + construct_reporter_if_needed(_index_reporter, "index"); + construct_reporter_if_needed(_image_reporter, "image"); + construct_reporter_if_needed(_image_index_reporter, "image_index"); _root.apply_config(config.sub_node("vfs")); @@ -453,6 +458,16 @@ struct Depot_query::Main node.attribute_value("version", Archive::Version()), node.attribute_value("content", false), xml); }); }); + + _gen_versioned_report(_image_reporter, version, [&] (Xml_generator &xml) { + query.for_each_sub_node("image", [&] (Xml_node node) { + _query_image(node.attribute_value("user", Archive::User()), + node.attribute_value("name", Archive::Name()), + xml); }); }); + + _gen_versioned_report(_image_index_reporter, version, [&] (Xml_generator &xml) { + query.for_each_sub_node("image_index", [&] (Xml_node node) { + _query_image_index(node, xml); }); }); } Main(Env &env) : _env(env) diff --git a/repos/gems/src/app/depot_query/query_image_index.cc b/repos/gems/src/app/depot_query/query_image_index.cc new file mode 100644 index 0000000000..22cbca80d8 --- /dev/null +++ b/repos/gems/src/app/depot_query/query_image_index.cc @@ -0,0 +1,171 @@ +/* + * \brief Querying system-image information from a depot + * \author Norman Feske + * \date 2023-01-26 + */ + +/* + * Copyright (C) 2023 Genode Labs GmbH + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +/* Genode includes */ +#include + +/* local includes */ +#include + + +void Depot_query::Main::_query_image_index(Xml_node const &index_query, + Xml_generator &xml) +{ + using User = Archive::User; + using Version = String<16>; + using Os = String<16>; + using Board = String<32>; + + User const user = index_query.attribute_value("user", User()); + Os const os = index_query.attribute_value("os", Os()); + Board const board = index_query.attribute_value("board", Board()); + + struct Version_reverse : Version + { + using Version::Version; + + bool operator > (Version_reverse const &other) const + { + return strcmp(string(), other.string()) < 0; + } + }; + + struct Image_info; + + using Image_dict = Dictionary; + + struct Image_info : Image_dict::Element + { + Constructible from_index { }; + + enum Presence { PRESENT, ABSENT } const presence; + + Image_info(Image_dict &dict, Version_reverse const &version, Presence presence) + : Image_dict::Element(dict, version), presence(presence) { } + + void generate(Xml_generator &xml) const + { + xml.node("image", [&] { + xml.attribute("version", name); + + if (presence == PRESENT) + xml.attribute("present", "yes"); + + if (!from_index.constructed()) + return; + + from_index->xml().for_each_sub_node("info", [&] (Xml_node const &info) { + using Text = String<160>; + Text const text = info.attribute_value("text", Text()); + if (text.valid()) + xml.node("info", [&] { + xml.attribute("text", text); }); + }); + }); + } + }; + + Image_dict images { }; + + Directory::Path const prefix(os, "-", board, "-"); + + /* return version part of the image-file name */ + auto version_from_name = [&prefix] (auto name) + { + size_t const prefix_chars = prefix.length() - 1; + + if (strcmp(prefix.string(), name.string(), prefix_chars)) + return Version_reverse(); /* prefix mismatch */ + + return Version_reverse(name.string() + prefix_chars); + }; + + Directory::Path const image_path("depot/", user, "/image"); + if (_root.directory_exists(image_path)) { + + Directory(_root, image_path).for_each_entry([&] (Directory::Entry const &entry) { + + Directory::Entry::Name const name = entry.name(); + Version_reverse const version = version_from_name(name); + + if (entry.dir() && version.length() > 1) + new (_heap) Image_info { images, version, Image_info::PRESENT }; + }); + } + + /* + * Supplement information found in the index file, if present + */ + Directory::Path const index_path("depot/", user, "/image/index"); + + bool index_exists = _root.file_exists(index_path); + + if (index_exists) { + try { + File_content const + file(_heap, _root, index_path, File_content::Limit{16*1024}); + + file.xml([&] (Xml_node node) { + + node.for_each_sub_node("image", [&] (Xml_node const &image) { + + bool const os_and_board_match = + (image.attribute_value("os", Os()) == os) && + (image.attribute_value("board", Board()) == board); + + if (!os_and_board_match) + return; + + Version_reverse const version { + image.attribute_value("version", Version()).string() }; + + if (!images.exists(version)) + new (_heap) Image_info(images, version, Image_info::ABSENT); + + images.with_element(version, + [&] (Image_info &info) { + info.from_index.construct(_heap, image); }, + [&] () { } + ); + }); + }); + } + catch (Directory::Nonexistent_file) { + index_exists = false; + } + } + + /* + * Give feedback to depot_download_manager about the absence of the index + * file. + */ + xml.node(index_exists ? "present" : "missing", [&] () { + xml.attribute("user", user); }); + + /* + * Report aggregated image information with the newest version first. + */ + xml.node("user", [&] () { + + xml.attribute("name", user); + xml.attribute("os", os); + xml.attribute("board", board); + + images.for_each([&] (Image_info const &info) { + info.generate(xml); }); + }); + + auto destroy_image_info = [&] (Image_info &info) { destroy(_heap, &info); }; + + while (images.with_any_element(destroy_image_info)); +} diff --git a/repos/gems/src/app/depot_query/target.mk b/repos/gems/src/app/depot_query/target.mk index 9052515dd7..4cd7a53ab7 100644 --- a/repos/gems/src/app/depot_query/target.mk +++ b/repos/gems/src/app/depot_query/target.mk @@ -1,4 +1,4 @@ TARGET := depot_query -SRC_CC := main.cc +SRC_CC := main.cc query_image_index.cc LIBS += base vfs INC_DIR += $(REP_DIR)/src/app/fs_query $(PRG_DIR)