From 076b2622f3f829bf7bc04b8145a6e8bdcbc703d6 Mon Sep 17 00:00:00 2001 From: Marc Nickert Date: Fri, 31 Jan 2025 15:27:03 +0100 Subject: [PATCH 1/7] Clean up most code_sync artifacts in extract_dir This commit adds clean up for most code_sync artifacts in the extract_dir folder, on child node shutdown. Some empty folders may be left behind. The cleanup reduces disk usage from every child start from round 500mb to a few kb in my case with EXLA on more libraries synced. --- lib/flame/code_sync.ex | 2 ++ lib/flame/terminator/cleaner.ex | 37 ++++++++++++++++++++++++++++++ lib/flame/terminator/supervisor.ex | 8 +++++++ 3 files changed, 47 insertions(+) create mode 100644 lib/flame/terminator/cleaner.ex diff --git a/lib/flame/code_sync.ex b/lib/flame/code_sync.ex index a360798..4572ee1 100644 --- a/lib/flame/code_sync.ex +++ b/lib/flame/code_sync.ex @@ -232,6 +232,8 @@ defmodule FLAME.CodeSync do # add code paths :ok = add_code_paths_from_tar(pkg, extract_dir) + FLAME.Terminator.Cleaner.watch_path(FLAME.Terminator.Cleaner, extract_dir) + File.rm(target_tmp_path) # purge any deleted modules diff --git a/lib/flame/terminator/cleaner.ex b/lib/flame/terminator/cleaner.ex new file mode 100644 index 0000000..28567a7 --- /dev/null +++ b/lib/flame/terminator/cleaner.ex @@ -0,0 +1,37 @@ +defmodule FLAME.Terminator.Cleaner do + @moduledoc false + alias Hex.Shell + use GenServer + + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name)) + end + + def watch_path(server, path) do + GenServer.call(server, {:watch, path}) + end + + def list_paths(server) do + GenServer.call(server, :list) + end + + def init(_opts) do + Process.flag(:trap_exit, true) + {:ok, %{paths: []}} + end + + def handle_call({:watch, path}, _from, state) do + {:reply, :ok, %{state | paths: [path | state.paths]}} + end + + def handle_call(:list, _from, state) do + {:reply, state.paths, state} + end + + def terminate(_reason, state) do + for path <- state.paths do + File.rm_rf(path) + end + :ok + end +end diff --git a/lib/flame/terminator/supervisor.ex b/lib/flame/terminator/supervisor.ex index 76a0b9a..b057cc9 100644 --- a/lib/flame/terminator/supervisor.ex +++ b/lib/flame/terminator/supervisor.ex @@ -23,6 +23,8 @@ defmodule FLAME.Terminator.Supervisor do child_placement_sup = child_placement_sup_name(name) terminator_opts = Keyword.merge(opts, child_placement_sup: child_placement_sup) + cleaner_opts = Keyword.put([], :name, FLAME.Terminator.Cleaner) + children = [ {DynamicSupervisor, name: child_placement_sup, strategy: :one_for_one}, @@ -31,6 +33,12 @@ defmodule FLAME.Terminator.Supervisor do start: {FLAME.Terminator, :start_link, [terminator_opts]}, type: :worker, shutdown: shutdown_timeout + }, + %{ + id: FLAME.Terminator.Cleaner, + start: {FLAME.Terminator.Cleaner, :start_link, [cleaner_opts]}, + type: :worker, + shutdown: shutdown_timeout } ] From b647c8fac90da0bb832b7ac75c88a329755f53df Mon Sep 17 00:00:00 2001 From: Marc Nickert Date: Fri, 7 Feb 2025 15:31:48 +0100 Subject: [PATCH 2/7] Change to use terminator to cleanup Paths --- lib/flame/code_sync.ex | 3 ++- lib/flame/terminator.ex | 19 +++++++++++++++ lib/flame/terminator/cleaner.ex | 37 ------------------------------ lib/flame/terminator/supervisor.ex | 7 ------ 4 files changed, 21 insertions(+), 45 deletions(-) delete mode 100644 lib/flame/terminator/cleaner.ex diff --git a/lib/flame/code_sync.ex b/lib/flame/code_sync.ex index 4572ee1..f6d00d1 100644 --- a/lib/flame/code_sync.ex +++ b/lib/flame/code_sync.ex @@ -232,7 +232,8 @@ defmodule FLAME.CodeSync do # add code paths :ok = add_code_paths_from_tar(pkg, extract_dir) - FLAME.Terminator.Cleaner.watch_path(FLAME.Terminator.Cleaner, extract_dir) + # add path to clean up + FLAME.Terminator.watch_path(FLAME.Terminator, extract_dir) File.rm(target_tmp_path) diff --git a/lib/flame/terminator.ex b/lib/flame/terminator.ex index c0fbc29..32b5fc6 100644 --- a/lib/flame/terminator.ex +++ b/lib/flame/terminator.ex @@ -32,6 +32,7 @@ defmodule FLAME.Terminator do single_use: false, calls: %{}, watchers: %{}, + paths: [], log: false, status: nil, failsafe_timer: nil, @@ -73,6 +74,10 @@ defmodule FLAME.Terminator do GenServer.call(terminator, {:watch, pids}) end + def watch_path(terminator, path) do + GenServer.call(terminator, {:watch_path, path}) + end + def deadline_me(terminator, timeout) do GenServer.call(terminator, {:deadline, timeout}) end @@ -263,6 +268,10 @@ defmodule FLAME.Terminator do {:reply, :ok, cancel_idle_shutdown(state)} end + def handle_call({:watch_path, path}, _from, %Terminator{watchers: paths} = state) do + {:reply, :ok, %{state | paths: [path | paths]}} + end + def handle_call(:system_shutdown, _from, %Terminator{} = state) do {:reply, :ok, system_stop(state, "system shutdown instructed from parent #{inspect(state.parent.pid)}")} @@ -288,6 +297,12 @@ defmodule FLAME.Terminator do {:reply, :ok, schedule_idle_shutdown(new_state)} end + defp clean_up_paths(paths) do + for path <- paths do + File.rm_rf(path) + end + end + @impl true def terminate(_reason, %Terminator{} = state) do state = @@ -295,6 +310,9 @@ defmodule FLAME.Terminator do |> cancel_idle_shutdown() |> system_stop("terminating") + # clean up any paths that were watched before waiting to not be killed + clean_up_paths(state.paths) + # supervisor will force kill us if we take longer than configured shutdown_timeout Enum.each(state.calls, fn # skip callers that placed a child since they are on the remote node @@ -306,6 +324,7 @@ defmodule FLAME.Terminator do {:DOWN, ^ref, :process, _pid, _reason} -> :ok end end) + end defp update_caller(%Terminator{} = state, ref, func) diff --git a/lib/flame/terminator/cleaner.ex b/lib/flame/terminator/cleaner.ex deleted file mode 100644 index 28567a7..0000000 --- a/lib/flame/terminator/cleaner.ex +++ /dev/null @@ -1,37 +0,0 @@ -defmodule FLAME.Terminator.Cleaner do - @moduledoc false - alias Hex.Shell - use GenServer - - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: Keyword.fetch!(opts, :name)) - end - - def watch_path(server, path) do - GenServer.call(server, {:watch, path}) - end - - def list_paths(server) do - GenServer.call(server, :list) - end - - def init(_opts) do - Process.flag(:trap_exit, true) - {:ok, %{paths: []}} - end - - def handle_call({:watch, path}, _from, state) do - {:reply, :ok, %{state | paths: [path | state.paths]}} - end - - def handle_call(:list, _from, state) do - {:reply, state.paths, state} - end - - def terminate(_reason, state) do - for path <- state.paths do - File.rm_rf(path) - end - :ok - end -end diff --git a/lib/flame/terminator/supervisor.ex b/lib/flame/terminator/supervisor.ex index b057cc9..43bc143 100644 --- a/lib/flame/terminator/supervisor.ex +++ b/lib/flame/terminator/supervisor.ex @@ -23,7 +23,6 @@ defmodule FLAME.Terminator.Supervisor do child_placement_sup = child_placement_sup_name(name) terminator_opts = Keyword.merge(opts, child_placement_sup: child_placement_sup) - cleaner_opts = Keyword.put([], :name, FLAME.Terminator.Cleaner) children = [ @@ -33,12 +32,6 @@ defmodule FLAME.Terminator.Supervisor do start: {FLAME.Terminator, :start_link, [terminator_opts]}, type: :worker, shutdown: shutdown_timeout - }, - %{ - id: FLAME.Terminator.Cleaner, - start: {FLAME.Terminator.Cleaner, :start_link, [cleaner_opts]}, - type: :worker, - shutdown: shutdown_timeout } ] From 8f96db13cf1760736005decd6438e4373f9c1b8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 7 Feb 2025 16:44:13 +0100 Subject: [PATCH 3/7] Apply suggestions from code review --- lib/flame/terminator.ex | 1 - lib/flame/terminator/supervisor.ex | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/flame/terminator.ex b/lib/flame/terminator.ex index 32b5fc6..3871123 100644 --- a/lib/flame/terminator.ex +++ b/lib/flame/terminator.ex @@ -324,7 +324,6 @@ defmodule FLAME.Terminator do {:DOWN, ^ref, :process, _pid, _reason} -> :ok end end) - end defp update_caller(%Terminator{} = state, ref, func) diff --git a/lib/flame/terminator/supervisor.ex b/lib/flame/terminator/supervisor.ex index 43bc143..76a0b9a 100644 --- a/lib/flame/terminator/supervisor.ex +++ b/lib/flame/terminator/supervisor.ex @@ -23,7 +23,6 @@ defmodule FLAME.Terminator.Supervisor do child_placement_sup = child_placement_sup_name(name) terminator_opts = Keyword.merge(opts, child_placement_sup: child_placement_sup) - children = [ {DynamicSupervisor, name: child_placement_sup, strategy: :one_for_one}, From 6a2aed0c70cb2a43ad8660d7a6aa6f91ad2766f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 7 Feb 2025 16:49:54 +0100 Subject: [PATCH 4/7] Update code_sync.ex --- lib/flame/code_sync.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/flame/code_sync.ex b/lib/flame/code_sync.ex index f6d00d1..605c079 100644 --- a/lib/flame/code_sync.ex +++ b/lib/flame/code_sync.ex @@ -216,7 +216,7 @@ defmodule FLAME.CodeSync do defp trim_leading_slash([?/ | path]), do: path defp trim_leading_slash([_ | _] = path), do: path - def extract_packaged_stream(%PackagedStream{} = pkg) do + def extract_packaged_stream(%PackagedStream{} = pkg, terminator) do if pkg.stream do verbose = if pkg.verbose, do: [:verbose], else: [] compressed = if pkg.compress, do: [:compressed], else: [] @@ -233,7 +233,7 @@ defmodule FLAME.CodeSync do :ok = add_code_paths_from_tar(pkg, extract_dir) # add path to clean up - FLAME.Terminator.watch_path(FLAME.Terminator, extract_dir) + FLAME.Terminator.watch_path(terminator, extract_dir) File.rm(target_tmp_path) From 4e11bbb78bfb5935310906cc64c3194d4ba3282d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 7 Feb 2025 16:51:06 +0100 Subject: [PATCH 5/7] Update runner.ex --- lib/flame/runner.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/flame/runner.ex b/lib/flame/runner.ex index 5dd2649..2dcb894 100644 --- a/lib/flame/runner.ex +++ b/lib/flame/runner.ex @@ -254,7 +254,7 @@ defmodule FLAME.Runner do {new_state, %CodeSync.PackagedStream{} = parent_pkg} -> remote_call!(state.runner, state.backend_state, state.runner.boot_timeout, false, fn -> - :ok = CodeSync.extract_packaged_stream(parent_pkg) + :ok = CodeSync.extract_packaged_stream(parent_pkg, state.runner.terminator) end) CodeSync.rm_packaged_stream(parent_pkg) From 0cbf185949d11fee7c4e539da567806b058017ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 7 Feb 2025 16:54:01 +0100 Subject: [PATCH 6/7] Update runner.ex --- lib/flame/runner.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/flame/runner.ex b/lib/flame/runner.ex index 2dcb894..f4a9e08 100644 --- a/lib/flame/runner.ex +++ b/lib/flame/runner.ex @@ -253,8 +253,10 @@ defmodule FLAME.Runner do new_state {new_state, %CodeSync.PackagedStream{} = parent_pkg} -> + terminator = state.runner.terminator + remote_call!(state.runner, state.backend_state, state.runner.boot_timeout, false, fn -> - :ok = CodeSync.extract_packaged_stream(parent_pkg, state.runner.terminator) + :ok = CodeSync.extract_packaged_stream(parent_pkg, terminator) end) CodeSync.rm_packaged_stream(parent_pkg) @@ -306,8 +308,8 @@ defmodule FLAME.Runner do # ensure app is fully started if parent connects before up if otp_app, do: {:ok, _} = Application.ensure_all_started(otp_app) - if base_sync_stream, do: CodeSync.extract_packaged_stream(base_sync_stream) - if beams_stream, do: CodeSync.extract_packaged_stream(beams_stream) + if base_sync_stream, do: CodeSync.extract_packaged_stream(base_sync_stream, term) + if beams_stream, do: CodeSync.extract_packaged_stream(beams_stream, term) :ok = Terminator.schedule_idle_shutdown(term, idle_after, idle_check, single_use) From e181a39f4e81b2b66b3cfa0ad29137a7e366321a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Valim?= Date: Fri, 7 Feb 2025 16:58:52 +0100 Subject: [PATCH 7/7] Update lib/flame/runner.ex --- lib/flame/runner.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/flame/runner.ex b/lib/flame/runner.ex index f4a9e08..6e70a34 100644 --- a/lib/flame/runner.ex +++ b/lib/flame/runner.ex @@ -308,8 +308,13 @@ defmodule FLAME.Runner do # ensure app is fully started if parent connects before up if otp_app, do: {:ok, _} = Application.ensure_all_started(otp_app) - if base_sync_stream, do: CodeSync.extract_packaged_stream(base_sync_stream, term) - if beams_stream, do: CodeSync.extract_packaged_stream(beams_stream, term) + if base_sync_stream do + CodeSync.extract_packaged_stream(base_sync_stream, term) + end + + if beams_stream do + CodeSync.extract_packaged_stream(beams_stream, term) + end :ok = Terminator.schedule_idle_shutdown(term, idle_after, idle_check, single_use)