diff --git a/.github/workflows/ci-develop.yml b/.github/workflows/ci-develop.yml new file mode 100644 index 0000000..056e45d --- /dev/null +++ b/.github/workflows/ci-develop.yml @@ -0,0 +1,77 @@ +name: CI Develop + +on: + push: + branches: [develop] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-test: + name: Build and Test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: src/docker/build/docker-image/Dockerfile + push: false + load: true + tags: seedsync:test + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Test container starts + run: | + docker run -d --name test-container -p 8800:8800 seedsync:test + sleep 5 + curl -f http://localhost:8800/ || exit 1 + docker logs test-container + docker stop test-container + + publish-develop: + name: Publish Develop Image + needs: [build-and-test] + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: src/docker/build/docker-image/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:develop + cache-from: type=gha + cache-to: type=gha,mode=max + provenance: false + sbom: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b2a14..f6f6710 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [0.10.4] - 2026-01-27 + +### Fixed +- **Remote paths with tilde (~)** - Fixed remote scanner to properly handle paths containing `~`. The tilde is now converted to `$HOME` for shell expansion, allowing users whose SSH and LFTP paths differ (e.g., LFTP locked to home directory). (#14) +- **LftpJobStatusParser crash on empty output** - Fixed parser crashing with "Missing queue header line 1" when lftp `jobs -v` returns empty or unexpected output. Now gracefully returns empty status instead of crashing. (#15) +- **ANSI escape codes in LFTP output** - Fixed parser failing on "First line is not a matching header" when LFTP output contains ANSI escape sequences like bracketed paste mode (`^[[?2004l`). These terminal control codes are now stripped before parsing. (#15) + +--- + ## [0.10.3] - 2026-01-27 ### Fixed diff --git a/src/angular/package.json b/src/angular/package.json index cf80003..010240f 100644 --- a/src/angular/package.json +++ b/src/angular/package.json @@ -1,6 +1,6 @@ { "name": "seedsync", - "version": "0.10.3", + "version": "0.10.4", "license": "Apache 2.0", "scripts": { "ng": "ng", diff --git a/src/python/controller/scan/remote_scanner.py b/src/python/controller/scan/remote_scanner.py index f89a3f2..5863a0a 100644 --- a/src/python/controller/scan/remote_scanner.py +++ b/src/python/controller/scan/remote_scanner.py @@ -13,6 +13,23 @@ from system import SystemFile +def _escape_remote_path_single(path: str) -> str: + """ + Escape a remote path using single quotes (no variable expansion). + """ + return "'{}'".format(path) + + +def _escape_remote_path_double(path: str) -> str: + """ + Escape a remote path using double quotes (allows $HOME expansion). + Converts ~ to $HOME for shell expansion. + """ + if path.startswith("~"): + path = "$HOME" + path[1:] + return '"{}"'.format(path) + + class RemoteScanner(IScanner): """ Scanner implementation to scan the remote filesystem @@ -51,10 +68,18 @@ def scan(self) -> List[SystemFile]: self._install_scanfs() try: - out = self.__ssh.shell("'{}' '{}'".format( - self.__remote_path_to_scan_script, - self.__remote_path_to_scan) - ) + # Use consistent quoting: double quotes if scan path has tilde (for $HOME expansion), + # single quotes otherwise (protects literal characters) + if self.__remote_path_to_scan.startswith("~"): + out = self.__ssh.shell("{} {}".format( + _escape_remote_path_double(self.__remote_path_to_scan_script), + _escape_remote_path_double(self.__remote_path_to_scan)) + ) + else: + out = self.__ssh.shell("{} {}".format( + _escape_remote_path_single(self.__remote_path_to_scan_script), + _escape_remote_path_single(self.__remote_path_to_scan)) + ) except SshcpError as e: self.logger.warning("Caught an SshcpError: {}".format(str(e))) recoverable = True @@ -88,7 +113,8 @@ def _install_scanfs(self): local_md5sum = hashlib.md5(f.read()).hexdigest() self.logger.debug("Local scanfs md5sum = {}".format(local_md5sum)) try: - out = self.__ssh.shell("md5sum {} | awk '{{print $1}}' || echo".format(self.__remote_path_to_scan_script)) + out = self.__ssh.shell("md5sum '{}' | awk '{{print $1}}' || echo".format( + self.__remote_path_to_scan_script)) out = out.decode() if out == local_md5sum: self.logger.info("Skipping remote scanfs installation: already installed") diff --git a/src/python/lftp/job_status_parser.py b/src/python/lftp/job_status_parser.py index 2eeb3cd..b7d6d5c 100644 --- a/src/python/lftp/job_status_parser.py +++ b/src/python/lftp/job_status_parser.py @@ -73,8 +73,22 @@ def _eta_to_seconds(eta: str) -> int: eta_s = int((result.group("eta_s") or '0s')[:-1]) return eta_d*24*3600 + eta_h*3600 + eta_m*60 + eta_s + @staticmethod + def _strip_ansi_codes(text: str) -> str: + """ + Strip ANSI escape sequences from text. + These can appear in lftp output when terminal features like + bracketed paste mode are enabled (e.g., ^[[?2004l, ^[[?2004h). + """ + # Match ANSI escape sequences: ESC [ ... + # This covers CSI sequences including bracketed paste mode + ansi_pattern = re.compile(r'\x1b\[[0-9;?]*[a-zA-Z]') + return ansi_pattern.sub('', text) + def parse(self, output: str) -> List[LftpJobStatus]: statuses = list() + # Strip ANSI escape codes that may be present in terminal output + output = self._strip_ansi_codes(output) lines = [s.strip() for s in output.splitlines()] lines = list(filter(None, lines)) # remove blank lines # remove all lines before the first 'jobs -v' @@ -479,22 +493,29 @@ def __parse_queue(lines: List[str]) -> List[LftpJobStatus]: queue_done_m = re.compile(LftpJobStatusParser.__QUEUE_DONE_REGEX) if len(lines) == 1: if not queue_done_m.match(lines[0]): - raise ValueError("Unrecognized line '{}'".format(lines[0])) + # Single unrecognized line - might be empty output, skip gracefully + return queue lines.pop(0) if lines: # Look for the header lines if len(lines) < 2: - raise ValueError("Missing queue header") + # Not enough lines for a valid queue header - return empty queue + return queue header1_pattern = r"^\[\d+\] queue \(sftp://.*@.*\)(?:\s+--\s+(?:\d+\.\d+|\d+)\s({})\/s)?$"\ .format(LftpJobStatusParser.__SIZE_UNITS_REGEX) header2_pattern = "^sftp://.*@.*$" line = lines.pop(0) if not re.match(header1_pattern, line): - raise ValueError("Missing queue header line 1: {}".format(line)) + # First line doesn't match queue header - no active queue, return empty + # Put the line back for __parse_jobs to handle + lines.insert(0, line) + return queue line = lines.pop(0) if not re.match(header2_pattern, line): - raise ValueError("Missing queue header line 2: {}".format(line)) + # Second line doesn't match - malformed but not fatal, return empty queue + lines.insert(0, line) + return queue if not lines: raise ValueError("Missing queue status") diff --git a/src/python/tests/unittests/test_controller/test_scan/test_remote_scanner.py b/src/python/tests/unittests/test_controller/test_scan/test_remote_scanner.py index 73cdbcf..1edf900 100644 --- a/src/python/tests/unittests/test_controller/test_scan/test_remote_scanner.py +++ b/src/python/tests/unittests/test_controller/test_scan/test_remote_scanner.py @@ -162,7 +162,7 @@ def ssh_shell(*args): scanner.scan() self.assertEqual(2, self.mock_ssh.shell.call_count) self.mock_ssh.shell.assert_has_calls([ - call("md5sum /remote/path/to/scan/script | awk '{print $1}' || echo"), + call("md5sum '/remote/path/to/scan/script' | awk '{print $1}' || echo"), call(ANY) ]) @@ -288,6 +288,38 @@ def ssh_shell(*args): "'/remote/path/to/scan/script' '/remote/path/to/scan'" ) + def test_handles_tilde_path_for_shell_expansion(self): + """Test that paths starting with ~ are converted to $HOME for shell expansion""" + scanner = RemoteScanner( + remote_address="my remote address", + remote_username="my remote user", + remote_password="my password", + remote_port=1234, + remote_path_to_scan="~/data/torrents", + local_path_to_scan_script=TestRemoteScanner.temp_scan_script, + remote_path_to_scan_script="/remote/path/to/scan/script" + ) + + self.ssh_run_command_count = 0 + + def ssh_shell(*args): + self.ssh_run_command_count += 1 + if self.ssh_run_command_count == 1: + # md5sum check + return b'' + else: + # later tries + return pickle.dumps([]) + self.mock_ssh.shell.side_effect = ssh_shell + + scanner.scan() + self.assertEqual(2, self.mock_ssh.shell.call_count) + # When scan path has tilde, both paths use double quotes for consistent quoting + # Tilde is converted to $HOME for shell expansion + self.mock_ssh.shell.assert_called_with( + "\"/remote/path/to/scan/script\" \"$HOME/data/torrents\"" + ) + def test_raises_nonrecoverable_error_on_first_failed_ssh(self): scanner = RemoteScanner( remote_address="my remote address", diff --git a/src/python/tests/unittests/test_lftp/test_job_status_parser.py b/src/python/tests/unittests/test_lftp/test_job_status_parser.py index 236ab17..518545f 100644 --- a/src/python/tests/unittests/test_lftp/test_job_status_parser.py +++ b/src/python/tests/unittests/test_lftp/test_job_status_parser.py @@ -64,7 +64,7 @@ def test_empty_output_3(self): def test_empty_output_4(self): output = """ - [0] queue (sftp://someone:@localhost) + [0] queue (sftp://someone:@localhost) sftp://someone:@localhost/home/someone [0] Done (queue (sftp://someone:@localhost)) """ @@ -72,6 +72,53 @@ def test_empty_output_4(self): statuses = parser.parse(output) self.assertEqual(0, len(statuses)) + def test_empty_output_just_jobs_command(self): + """Test handling when lftp returns just the jobs command echo with no data""" + output = """ + jobs -v + """ + parser = LftpJobStatusParser() + statuses = parser.parse(output) + self.assertEqual(0, len(statuses)) + + def test_empty_output_jobs_command_with_blank_lines(self): + """Test handling when lftp returns jobs command with extra blank lines""" + output = """ + + jobs -v + + """ + parser = LftpJobStatusParser() + statuses = parser.parse(output) + self.assertEqual(0, len(statuses)) + + def test_ansi_escape_codes_stripped(self): + """Test that ANSI escape codes (like bracketed paste mode) are stripped from output""" + # \x1b[?2004l and \x1b[?2004h are bracketed paste mode disable/enable + output = """ + jobs -v + \x1b[?2004l + \x1b[?2004h + """ + parser = LftpJobStatusParser() + statuses = parser.parse(output) + self.assertEqual(0, len(statuses)) + + def test_ansi_escape_codes_in_actual_output(self): + """Test parsing works when ANSI codes are mixed with real job data""" + output = """ + \x1b[?2004ljobs -v + [0] queue (sftp://someone:@localhost)\x1b[?2004h + sftp://someone:@localhost/home/someone + Queue is stopped. + Commands queued: + 1. mirror -c /tmp/test_lftp/remote/a /tmp/test_lftp/local/ + """ + parser = LftpJobStatusParser() + statuses = parser.parse(output) + self.assertEqual(1, len(statuses)) + self.assertEqual("a", statuses[0].name) + def test_queued_items(self): """Queued items, no jobs running""" output = """