diff --git a/repos/gems/recipes/src/depot_remove/content.mk b/repos/gems/recipes/src/depot_remove/content.mk new file mode 100644 index 0000000000..1d4cffd34c --- /dev/null +++ b/repos/gems/recipes/src/depot_remove/content.mk @@ -0,0 +1,10 @@ +SRC_DIR := src/app/depot_remove + +include $(GENODE_DIR)/repos/base/recipes/src/content.inc + +MIRROR_FROM_REP_DIR := include/depot + +content: $(MIRROR_FROM_REP_DIR) + +$(MIRROR_FROM_REP_DIR): + $(mirror_from_rep_dir) diff --git a/repos/gems/recipes/src/depot_remove/hash b/repos/gems/recipes/src/depot_remove/hash new file mode 100644 index 0000000000..4bf12e2674 --- /dev/null +++ b/repos/gems/recipes/src/depot_remove/hash @@ -0,0 +1 @@ +2023-05-09T1814 a46d237336206b23977c7e447cec72c1c49e0799 diff --git a/repos/gems/recipes/src/depot_remove/used_apis b/repos/gems/recipes/src/depot_remove/used_apis new file mode 100644 index 0000000000..ade9ce3115 --- /dev/null +++ b/repos/gems/recipes/src/depot_remove/used_apis @@ -0,0 +1,4 @@ +base +os +vfs +report_session diff --git a/repos/gems/run/depot_remove.run b/repos/gems/run/depot_remove.run new file mode 100644 index 0000000000..f9a76626a8 --- /dev/null +++ b/repos/gems/run/depot_remove.run @@ -0,0 +1,284 @@ +assert_spec linux +assert_spec x86_64 + +create_boot_directory + +import_from_depot [depot_user]/src/[base_src] \ + [depot_user]/src/report_rom \ + [depot_user]/src/vfs \ + [depot_user]/src/lx_fs \ + [depot_user]/src/vfs_import \ + [depot_user]/src/init + +create_tar_from_depot_binaries [run_dir]/genode/depot.tar \ + [depot_user]/pkg/chroot \ + [depot_user]/pkg/system_shell \ + [depot_user]/pkg/fonts_fs \ + [depot_user]/pkg/wm \ + [depot_user]/pkg/nano3d \ + [depot_user]/pkg/window_layouter \ + [depot_user]/pkg/motif_decorator \ + [depot_user]/pkg/themed_decorator \ + [depot_user]/pkg/sticks_blue_backdrop + +if { [get_cmd_switch --autopilot] } { + import_from_depot [depot_user]/src/depot_remove +} else { + build { app/depot_remove } +} + +install_config { + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +} + +exec mkdir [run_dir]/genode/depot +exec tar xvf [run_dir]/genode/depot.tar -C [run_dir]/genode/depot +exec chmod -R +r [run_dir]/genode/depot + +if { [get_cmd_switch --autopilot] } { + build_boot_image {} +} else { + build_boot_image [build_artifacts] +} + +proc install_test_config { args } { + set fd [open [run_dir]/genode/depot_remove_config w] + puts $fd "[join $args {}]" + close $fd +} + +proc depot_state { } { + + set archives_to_keep {} + + foreach dir [glob -type d [run_dir]/genode/depot/[depot_user]/pkg/*] { + set pkg_path [depot_user]/pkg/[file tail $dir]/[_current_depot_archive_version pkg [file tail $dir]] + + set fd [open [run_dir]/genode/depot/$pkg_path/archives r] + set pkg_deps [split [string trim [read $fd]]] + close $fd + + foreach dependency $pkg_deps { + set idx [lsearch -exact $archives_to_keep $dependency] + if { $idx == -1 } { + lappend archives_to_keep $dependency + } + } + lappend archives_to_keep $pkg_path + } + + set context [dict create] + dict set context pkg "" + dict set context archives_to_delete {} + dict set context archives_to_keep $archives_to_keep + + return $context +} + +proc depot_state_for_pkg { archive } { + set archive_path [depot_user]/pkg/$archive/[_current_depot_archive_version pkg $archive] + + # Dependencies of $archive are archives to delete + set fd [open [run_dir]/genode/depot/$archive_path/archives r] + set archives_to_delete [split [string trim [read $fd]]] + close $fd + + set archives_to_keep {} + + foreach dir [glob -type d [run_dir]/genode/depot/[depot_user]/pkg/*] { + + if { [file tail $dir] != $archive } { + + set pkg_path [depot_user]/pkg/[file tail $dir]/[_current_depot_archive_version pkg [file tail $dir]] + + set fd [open [run_dir]/genode/depot/$pkg_path/archives r] + set pkg_deps [split [string trim [read $fd]]] + close $fd + + foreach dependency $archives_to_delete { + set idx [lsearch -exact $pkg_deps $dependency] + if { $idx != -1 } { + lappend archives_to_keep $dependency + } + } + } + } + + # Remove archive to keep from archive to delete + foreach dependency $archives_to_keep { + set idx [lsearch -exact $archives_to_delete $dependency] + set archives_to_delete [lreplace $archives_to_delete $idx $idx] + } + + set context [dict create] + dict set context pkg "$archive_path" + dict set context archives_to_delete $archives_to_delete + dict set context archives_to_keep $archives_to_keep + + return $context +} + +proc check_depot_state { context } { + + foreach archive [dict get $context archives_to_delete] { + + regexp [_depot_archive_versioned_path_pattern] $archive dummy archive_user archive_type archive_name + + if { $archive_type == "src" } { + set archive $archive_user/bin/x86_64/$archive_name/[_current_depot_archive_version src $archive_name] + } + + if { [file isdirectory [run_dir]/genode/depot/$archive] } { + puts "ERROR: $archive is present but should has been deleted." + return 1 + } + } + + foreach archive [dict get $context archives_to_keep] { + + regexp [_depot_archive_versioned_path_pattern] $archive dummy archive_user archive_type archive_name + + if { $archive_type == "src" } { + set archive $archive_user/bin/x86_64/$archive_name/[_current_depot_archive_version src $archive_name] + } + + if { ![file isdirectory [run_dir]/genode/depot/$archive] } { + puts "ERROR: $archive should still be there but it has been deleted." + return 1 + } + } + + if { [dict get $context pkg] != "" && [file isdirectory [run_dir]/genode/depot/[dict get $context pkg]] } { + puts "ERROR: [dict get $context pkg] is present but shoud have been deleted." + return 1 + } + + return 0 +} + +## TEST 1 --- Delete nano3d --------------------------------------------------- + +set context [depot_state_for_pkg "nano3d"] + +install_test_config { + + + + +} + +run_genode_until ".*" 10 + +if { [check_depot_state $context] } { + puts " TEST 1 --- Delete nano3d -- ERROR" + exit 1 +} + +puts " TEST 1 --- Delete nano3d -- SUCCESS" + +## TEST 2 --- Delete non existing archive ------------------------------------- + +set context [depot_state] + +install_test_config { + + + + +} + +run_genode_until ".*" 10 + +if { [check_depot_state $context] } { + puts " TEST 2 --- Delete non existing archive -- ERROR" + exit 1 +} + +puts " TEST 2 --- Delete non existing archive -- SUCCESS" + +## TEST 3 --- Delete a PKG archive with deps to keep -------------------------- + +set context [depot_state_for_pkg "fonts_fs"] + +install_test_config { + + + + +} + +run_genode_until ".*" 10 + +if { [check_depot_state $context] } { + puts " TEST 3 --- Delete a PKG archive with deps to keep --- ERROR" + exit 1 +} + +puts " TEST 3 --- Delete a PKG archive with deps to keep --- SUCCESS" + +## TEST 4 --- Remove all, keep themed_decorator PKG -------------------------- + +set context [depot_state_for_pkg "themed_decorator"] + +install_test_config { + + + + + + +} + +run_genode_until ".*" 10 + +if { ![check_depot_state $context] } { + puts " TEST 4 --- Remove all, keep themed_decorator PKG --- ERROR" + exit 1 +} + +puts " TEST 4 --- Remove all, keep themed_decorator PKG --- SUCCESS" + diff --git a/repos/gems/src/app/depot_remove/README b/repos/gems/src/app/depot_remove/README new file mode 100644 index 0000000000..7c135edc9f --- /dev/null +++ b/repos/gems/src/app/depot_remove/README @@ -0,0 +1,73 @@ +This directory contains the depot_remove component. It can delete PKGs and +its dependencies from the depot. It operates by reading its configuration and +processes delete operations based on the provided rules. This component +listens for configuration changes and will reactivate if it has been updated. + +Configuration +~~~~~~~~~~~~~ + +A typical configuration looks as follows. + +! +! +! +! +! +! +! + +The '' node comes with two possible attributes. + +:arch: + + This a mandatory attribute used to identify the correct directory for binary + archives. + +:report (default "no"): + + This is an optional attribute. If set to "yes", a "removed_archives" report + is created, listing each deleted archive. + +The '' node instructs the component to remove a PKG archive from the +depot. The '' node instructs the component to remove all PGK archives. +It can be combined with a '' node to instruct the component to keep a +given PKG archive while removing all the others. + +The '' and '' nodes accept the following attributes. + + +:pkg: + + This is a mandatory attribute that is used to identify the targeted PKG + archive. + +:user: + + This is an optional attribute to identify the depot user for the given pkg. + +:version: + + This is an optional attribute to identify the version for the given pkg. + +Reporting +~~~~~~~~~ + +When activated, the component reports the following content. + +! +! +! +! +! +! +! +! +! +! + +Example +~~~~~~~ + +Please refer to the _gems/run/depot_remove.run_ script for a practical example +of using the _depot_remove_ component. + diff --git a/repos/gems/src/app/depot_remove/main.cc b/repos/gems/src/app/depot_remove/main.cc new file mode 100644 index 0000000000..d0362d7df8 --- /dev/null +++ b/repos/gems/src/app/depot_remove/main.cc @@ -0,0 +1,291 @@ +/* + * \brief Tool for deleting packages from a depot and resolving unused dependencies + * \author Alice Domage + * \date 2023-06-14 + */ + +/* + * Copyright (C) 2023 Genode Labs GmbH + * Copyright (C) 2023 gapfruit AG + * + * This file is part of the Genode OS framework, which is distributed + * under the terms of the GNU Affero General Public License version 3. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace Depot_remove { + + using namespace Genode; + + struct Main; + class Archive_remover; + +} + + +class Depot_remove::Archive_remover +{ + public: + + using Archive_path = Depot::Archive::Path; + using Registered_path = Genode::Registered_no_delete; + using Path = Directory::Path; + + + private: + + Allocator &_alloc; + String<32> const _arch; + Registry _deleted_archives { }; + Registry _pkg_to_delete { }; + Registry _archive_to_delete { }; + + void _remove_directory(Directory &depot, Path path) const + { + Directory dir { depot, path }; + Registry dirent_files { }; + + dir.for_each_entry([&] (auto const &entry) { + if (entry.name() == ".." || entry.name() == ".") + return; + else if (entry.type() == Vfs::Directory_service::Dirent_type::DIRECTORY) + _remove_directory(depot, Directory::join(path, entry.name())); + else + /* + * Deleting file within the for_each_entry() confuses lx_fs dirent + * offset computation and some files, such as 'README', is consitently + * omitted, thus the unlink operation fails. Thus create a list + * to delete file out of the lambda. + */ + new (_alloc) Registered_path(dirent_files, Directory::join(path, entry.name())); }); + + dirent_files.for_each([&](Registered_path &sub_path) { + depot.unlink(sub_path); + destroy(_alloc, &sub_path); }); + + depot.unlink(path); + } + + template + void _for_each_subdir(Directory &depot, Path const &parent_dir, FUNC fn) + { + Directory pkg { depot, parent_dir }; + + pkg.for_each_entry([&fn, &parent_dir](auto const &entry) { + if (entry.name() == ".." || entry.name() == ".") + return; + Path subdir_path { Directory::join(parent_dir, entry.name()) }; + fn(subdir_path); }); + } + + template + void _for_each_pkg(Directory &depot, FUNC fn) + { + depot.for_each_entry([&](auto const &entry) { + Path pkg_path { entry.name(), "/pkg" }; + if (depot.directory_exists(pkg_path)) { + _for_each_subdir(depot, pkg_path, [&] (Path const &pkg_path) { + _for_each_subdir(depot, pkg_path, [&] (Path const &pkg_version_path) { + fn(pkg_version_path); }); }); } }); + } + + void _autoremove_pkg_and_dependencies(Directory &depot) + { + + /* collect all archive dependencies to delete */ + _pkg_to_delete.for_each([&](Archive_path &elem) { + Path pkg_version_path { elem }; + Path archive_file_path { Directory::join(pkg_version_path, "archives") }; + File_content archives { _alloc, depot, archive_file_path, { 8192 } }; + archives.for_each_line([&](auto const &dependency_path) { + if (Depot::Archive::type(dependency_path) == Depot::Archive::Type::PKG) + return; + new (_alloc) Registered_path(_archive_to_delete, dependency_path); }); + _remove_directory(depot, pkg_version_path); + /* try to delete the parent if it is empty, if not empty the operation fails */ + _remove_directory(depot, Genode::Directory::join(pkg_version_path, "..")); + new (_alloc) Registered_path(_deleted_archives, pkg_version_path); }); + + /* keep archive dependencies that are still referenced by another PKG */ + _for_each_pkg(depot, [&](Path const &pkg_version_path) { + Path archive_file_path { Directory::join(pkg_version_path, "archives") }; + File_content archives { _alloc, depot, archive_file_path, { 8192 } }; + archives.for_each_line([&] (auto const &dependency_path) { + if (Depot::Archive::type(dependency_path) == Depot::Archive::Type::PKG) + return; + _archive_to_delete.for_each([&](Registered_path &path){ + if (dependency_path == path) + destroy(_alloc, &path); }); }); }); + + /* delete archive dependencies */ + _archive_to_delete.for_each([&](Archive_path &path) { + Path archive {}; + if (Depot::Archive::type(path) == Depot::Archive::Type::SRC) { + archive = Directory::join(Depot::Archive::user(path), "bin"); + archive = Directory::join(archive, _arch); + archive = Directory::join(archive, Depot::Archive::name(path)); + archive = Directory::join(archive, Depot::Archive::version(path)); + } else { + archive = path; + } + /* if directory does not exist, it might has been deleted before, return silently */ + if (!depot.directory_exists(archive)) + return; + _remove_directory(depot, archive); + new (_alloc) Registered_path(_deleted_archives, archive); + /* try to delete the parent if it is empty, if not empty the operation fails */ + _remove_directory(depot, Directory::join(archive, "..")); }); + } + + static bool _config_node_match_pkg(Genode::Xml_node const &node, Path pkg) + { + if (!node.has_attribute("user")) + return false; + + if (Depot::Archive::user(pkg) != node.attribute_value("user", Archive_path {})) + return false; + + if (!node.has_attribute("pkg")) + return true; + + if (Depot::Archive::name(pkg) != node.attribute_value("pkg", Archive_path {})) + return false; + + if (!node.has_attribute("version")) + return true; + + if (Depot::Archive::version(pkg) != node.attribute_value("version", Archive_path {})) + return false; + + return true; + }; + + void _configure_remove_pkgs(Directory &depot, Xml_node const &config) + { + _for_each_pkg(depot, [&] (Path const &pkg_path) { + config.for_each_sub_node("remove", [&](Xml_node const &node) { + if (_config_node_match_pkg(node, pkg_path)) + new (_alloc) Registered_path(_pkg_to_delete, pkg_path); }); }); + } + + void _configure_remove_all_pkgs(Directory &depot, Xml_node const &config) + { + _for_each_pkg(depot, [&] (Path const &pkg_path) { + bool keep = false; + config.for_each_sub_node("remove-all", [&](Xml_node const &remove_all_node) { + remove_all_node.for_each_sub_node("keep", [&](Xml_node const &node) { + if (_config_node_match_pkg(node, pkg_path)) + keep = true; }); }); + if (!keep) + new (_alloc) Registered_path(_pkg_to_delete, pkg_path); }); + } + + + public: + + void generate_report(Expanding_reporter &reporter) const + { + reporter.generate([&](Reporter::Xml_generator &xml) { + _deleted_archives.for_each([&] (auto &path) { + xml.node("removed", [&]() { + xml.attribute("path", path); }); }); }); + } + + Archive_remover(Allocator &alloc, + Directory &depot, + Xml_node const &config) + : + _alloc { alloc }, + _arch { config.attribute_value("arch", String<32>()) } + { + if (config.has_sub_node("remove") && config.has_sub_node("remove-all")) { + warning(" and are mutually exclusive"); + return; + } + + if (config.has_sub_node("remove")) _configure_remove_pkgs(depot, config); + if (config.has_sub_node("remove-all")) _configure_remove_all_pkgs(depot, config); + + _autoremove_pkg_and_dependencies(depot); + } + + ~Archive_remover() { + _pkg_to_delete.for_each([this] (auto &elem) { + destroy(_alloc, &elem); }); + _archive_to_delete.for_each([this] (auto &elem) { + destroy(_alloc, &elem); }); + _deleted_archives.for_each([this] (auto &elem) { + destroy(_alloc, &elem); }); + } +}; + + +struct Depot_remove::Main +{ + Env &_env; + Heap _heap { _env.ram(), _env.rm() }; + Attached_rom_dataspace _config_rom { _env, "config" }; + Signal_handler
_config_handler { _env.ep(), *this, &Main::_handle_config }; + Constructible _reporter { }; + + Main(Env &env) + : + _env { env }, + _config_handler { env.ep(), *this, &Main::_handle_config } + { + _config_rom.sigh(_config_handler); + _handle_config(); + } + + void _handle_config() + { + _config_rom.update(); + Xml_node const &config { _config_rom.xml() }; + + if (!config.has_attribute("arch")) { + warning("missing arch attribute"); + return; + } + + if (!config.has_sub_node("vfs")) { + warning("configuration misses a configuration node"); + return; + } + + Directory::Path depot_path { "depot" }; + Root_directory root_directory { _env, _heap, config.sub_node("vfs") }; + Directory depot { root_directory, depot_path }; + + try { + Archive_remover archive_cleaner { _heap, depot, config }; + + _reporter.conditional(config.attribute_value("report", false), + _env, "removed_archives", "archive_list"); + + if (_reporter.constructed()) + archive_cleaner.generate_report(*_reporter); + } catch (...) { + /* catch any exceptions to prevent the component to abort */ + error("Depot autoclean job finished with error(s)."); + } + } +}; + + +void Component::construct(Genode::Env &env) +{ + static Depot_remove::Main main(env); +} + diff --git a/repos/gems/src/app/depot_remove/target.mk b/repos/gems/src/app/depot_remove/target.mk new file mode 100644 index 0000000000..aa43301f48 --- /dev/null +++ b/repos/gems/src/app/depot_remove/target.mk @@ -0,0 +1,6 @@ + +TARGET := depot_remove + +SRC_CC := main.cc + +LIBS := base vfs diff --git a/tool/autopilot.list b/tool/autopilot.list index a6e5069a4b..150f18ad63 100644 --- a/tool/autopilot.list +++ b/tool/autopilot.list @@ -9,6 +9,7 @@ demo depot_autopilot depot_download depot_query +depot_remove event_filter extract fb_bench