From fb848caab48f95cb31b29c6c7b491ed8817d0470 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Thu, 15 Jul 2021 22:46:41 +0100 Subject: [PATCH 1/4] Add target def.n for replot single file removal --- src/plotman/resources/target_definitions.yaml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml index 6ef6e001..d6efb020 100644 --- a/src/plotman/resources/target_definitions.yaml +++ b/src/plotman/resources/target_definitions.yaml @@ -58,6 +58,38 @@ target_definitions: "${command}" ${options} "${source}" "${url_root}/${relative_path}/" transfer_process_name: "{command}" transfer_process_argument_prefix: "rsync://{user}@{host}:{rsync_port}/{site}" + rsyncd_replot: + env: + # A value of null indicates a mandatory option + command: rsync + options: --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file + rsync_port: 873 + ssh_port: 22 + user: null + host: null + site_root: null + site: null + path_suffix: "" + # Provide the file glob to determine the next file to be deleted i.e. "${site_root}/*/plot-k32-2021-0[3-6]-*" + replot_glob: null + disk_space_script: | + #!/bin/bash + set -evx + site_root_stripped=$(echo "${site_root}" | sed 's;/\+$;;') + # printf with %.0f used to handle mawk such as in Ubuntu Docker images + # otherwise it saturates and you get saturated sizes like 2147483647 + ssh -p "${ssh_port}" "${user}@${host}" "df -aBK | grep \" $(echo "${site_root_stripped}" | sed 's;/\+$;;')/\" | awk '{ gsub(/K\$/,\"\",\$4); printf \"%s:%.0f\n\", \$6, \$4*1024 }'" + transfer_script: | + #!/bin/bash + set -evx + echo Launching transfer activity + full_destination=$(realpath --canonicalize-missing "${destination}/${path_suffix}") + relative_path=$(realpath --canonicalize-missing --relative-to="${site_root}" "${full_destination}") + url_root="rsync://${user}@${host}:${rsync_port}/${site}" + "${command}" ${options} "${source}" "${url_root}/${relative_path}/" + ssh -p "${ssh_port}" "${user}@${host}" ""ls -1U ${replot_glob}.plot | head -n 1 | xargs rm -fv"" + transfer_process_name: "{command}" + transfer_process_argument_prefix: "rsync://{user}@{host}:{rsync_port}/{site}" # external_script: # env: # some_common_value_with_a_default: /a/path From 64964616830755bbce2cc2a61aad48d7a2a5a8ad Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Thu, 15 Jul 2021 22:49:25 +0100 Subject: [PATCH 2/4] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a6a575d..a3c16e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Allows explicit specification of the plotter executable path if this is preferred over setting the `PATH` environment variable to find the program. Presently does not support executables other than the expected names (`chia`, and `chia_plot`). ([#823](https://github.com/ericaltendorf/plotman/pull/823)) +- New rsyncd target definition which removes a single file after archiving based on a file glob (for replotting) ## [0.5] - 2021-07-07 ### Fixed From cbe5d6c026a3c3960e90cb9a1d023711fe1668b1 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Thu, 15 Jul 2021 23:27:28 +0100 Subject: [PATCH 3/4] Correct double quotes (too helpful editor) --- src/plotman/resources/target_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml index d6efb020..a2e6005e 100644 --- a/src/plotman/resources/target_definitions.yaml +++ b/src/plotman/resources/target_definitions.yaml @@ -87,7 +87,7 @@ target_definitions: relative_path=$(realpath --canonicalize-missing --relative-to="${site_root}" "${full_destination}") url_root="rsync://${user}@${host}:${rsync_port}/${site}" "${command}" ${options} "${source}" "${url_root}/${relative_path}/" - ssh -p "${ssh_port}" "${user}@${host}" ""ls -1U ${replot_glob}.plot | head -n 1 | xargs rm -fv"" + ssh -p "${ssh_port}" "${user}@${host}" "ls -1U ${replot_glob}.plot | head -n 1 | xargs rm -fv" transfer_process_name: "{command}" transfer_process_argument_prefix: "rsync://{user}@{host}:{rsync_port}/{site}" # external_script: From e7c80325e809a61e3a16a67c3f36e050bd388ca9 Mon Sep 17 00:00:00 2001 From: Graeme Seaton Date: Fri, 16 Jul 2021 00:31:29 +0100 Subject: [PATCH 4/4] Use site_root and change example --- src/plotman/resources/target_definitions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml index a2e6005e..d097913e 100644 --- a/src/plotman/resources/target_definitions.yaml +++ b/src/plotman/resources/target_definitions.yaml @@ -70,7 +70,7 @@ target_definitions: site_root: null site: null path_suffix: "" - # Provide the file glob to determine the next file to be deleted i.e. "${site_root}/*/plot-k32-2021-0[3-6]-*" + # Provide the file glob to determine the next file to be deleted e.g. "*/plot-k32-2021-0[3-6]-*" replot_glob: null disk_space_script: | #!/bin/bash @@ -87,7 +87,7 @@ target_definitions: relative_path=$(realpath --canonicalize-missing --relative-to="${site_root}" "${full_destination}") url_root="rsync://${user}@${host}:${rsync_port}/${site}" "${command}" ${options} "${source}" "${url_root}/${relative_path}/" - ssh -p "${ssh_port}" "${user}@${host}" "ls -1U ${replot_glob}.plot | head -n 1 | xargs rm -fv" + ssh -p "${ssh_port}" "${user}@${host}" "ls -1U ${site_root}/${replot_glob}.plot | head -n 1 | xargs rm -fv" transfer_process_name: "{command}" transfer_process_argument_prefix: "rsync://{user}@{host}:{rsync_port}/{site}" # external_script: