From 2d706885597365edaee8c89340a85f98166cbbb0 Mon Sep 17 00:00:00 2001 From: superr Date: Wed, 31 Mar 2021 11:26:26 -0400 Subject: [PATCH 1/7] New command line argument "--skip_hash" Just add it to the beginning of the command (./extract.py --skip_hash) and the hash will be skipped. Fixed bug where if the old file that matches the new file does not exist we get a crash. Now we attempt to extract the new file without the old file. Added automatic file renaming. You can put your .img files in the "old" directory and they will be renamed automatically, and back at the end of the extraction. Additionally, the "output" directory will get .img appended to their file names after the extraction. Removed the need for the "LD_LIBRARY_PATH=./lib64/" part of the command by setting it within the script. Updated README.md to reflect all changes. --- README.md | 6 +++--- extract.py | 23 +++++++++++++++++++---- update_payload/applier.py | 24 ++++++++++++++---------- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d57aa81..6f037c1 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ ### Full OTA -- LD_LIBRARY_PATH=./lib64/ ./extract.py --output_dir output/ payload.bin +- ./extract.py [--skip_hash] --output_dir output/ payload.bin - This will start to extract the images within the payload.bin file to the output folder you are in. ### Incremental OTA -- Copy original images (from full OTA or dumped from devices) to old folder (with part name without file extension, ex: boot, system) -- LD_LIBRARY_PATH=./lib64/ ./extract.py --output_dir output/ --old_dir old/ payload.bin +- Copy original images (from full OTA or dumped from devices) to old folder +- ./extract.py [--skip_hash] --output_dir output/ --old_dir old/ payload.bin diff --git a/extract.py b/extract.py index b7a7b55..9fb159e 100755 --- a/extract.py +++ b/extract.py @@ -2,11 +2,13 @@ import argparse import errno +import glob import os import update_payload from update_payload import applier +os.environ['LD_LIBRARY_PATH'] = './lib64/' def list_content(payload_file_name): with open(payload_file_name, 'rb') as payload_file: @@ -18,13 +20,16 @@ def list_content(payload_file_name): part.new_partition_info.size)) -def extract(payload_file_name, output_dir="output", old_dir="old", partition_names=None): +def extract(payload_file_name, output_dir="output", old_dir="old", partition_names=None, skip_hash=None): try: os.makedirs(output_dir) except OSError as e: if e.errno != errno.EEXIST: raise + for i in glob.glob(old_dir + '/*.img'): + os.rename(i, i[:-4]) + with open(payload_file_name, 'rb') as payload_file: payload = update_payload.Payload(payload_file) payload.Init() @@ -40,12 +45,20 @@ def extract(payload_file_name, output_dir="output", old_dir="old", partition_nam helper._ApplyToPartition( part.operations, part.partition_name, 'install_operations', output_file, - part.new_partition_info, old_file, part.old_partition_info) + part.new_partition_info, old_file, + part.old_partition_info, skip_hash) else: helper._ApplyToPartition( part.operations, part.partition_name, 'install_operations', output_file, - part.new_partition_info) + part.new_partition_info, + skip_hash=skip_hash) + + for i in glob.glob(old_dir + '/*'): + os.rename(i, i + '.img') + + for i in glob.glob(output_dir + '/*'): + os.rename(i, i + '.img') if __name__ == '__main__': parser = argparse.ArgumentParser() @@ -59,9 +72,11 @@ def extract(payload_file_name, output_dir="output", old_dir="old", partition_nam help="Name of the partitions to extract") parser.add_argument("--list_partitions", action="store_true", help="List the partitions included in the payload.bin") + parser.add_argument("--skip_hash", action="store_true", + help="Skip the hash check for individual img files") args = parser.parse_args() if args.list_partitions: list_content(args.payload) else: - extract(args.payload, args.output_dir, args.old_dir, args.partitions) + extract(args.payload, args.output_dir, args.old_dir, args.partitions, args.skip_hash) diff --git a/update_payload/applier.py b/update_payload/applier.py index c05095a..ad974c5 100644 --- a/update_payload/applier.py +++ b/update_payload/applier.py @@ -517,7 +517,8 @@ def _ApplyOperations(self, operations, base_name, old_part_file, def _ApplyToPartition(self, operations, part_name, base_name, new_part_file_name, new_part_info, - old_part_file_name=None, old_part_info=None): + old_part_file_name=None, old_part_info=None, + skip_hash=None): """Applies an update to a partition. Args: @@ -528,16 +529,18 @@ def _ApplyToPartition(self, operations, part_name, base_name, new_part_info: size and expected hash of dest partition old_part_file_name: file name of source partition (optional) old_part_info: size and expected hash of source partition (optional) + skip_hash: command line arg to skip hash checks Raises: PayloadError if anything goes wrong with the update. """ # Do we have a source partition? if old_part_file_name: - # Verify the source partition. - with open(old_part_file_name, 'rb') as old_part_file: - _VerifySha256(old_part_file, old_part_info.hash, - 'old ' + part_name, length=old_part_info.size) + # Verify the source partition if skip_hash arg was not given. + if not skip_hash: + with open(old_part_file_name, 'rb') as old_part_file: + _VerifySha256(old_part_file, old_part_info.hash, + 'old ' + part_name, length=old_part_info.size) new_part_file_mode = 'r+b' open(new_part_file_name, 'w').close() @@ -548,7 +551,7 @@ def _ApplyToPartition(self, operations, part_name, base_name, # Apply operations. with open(new_part_file_name, new_part_file_mode) as new_part_file: old_part_file = (open(old_part_file_name, 'r+b') - if old_part_file_name else None) + if os.path.exists(old_part_file_name) else None) try: self._ApplyOperations(operations, base_name, old_part_file, new_part_file, new_part_info.size) @@ -563,10 +566,11 @@ def _ApplyToPartition(self, operations, part_name, base_name, new_part_file.seek(new_part_info.size) new_part_file.truncate() - # Verify the resulting partition. - with open(new_part_file_name, 'rb') as new_part_file: - _VerifySha256(new_part_file, new_part_info.hash, - 'new ' + part_name, length=new_part_info.size) + # Verify the resulting partition if skip_hash arg was not given. + if not skip_hash: + with open(new_part_file_name, 'rb') as new_part_file: + _VerifySha256(new_part_file, new_part_info.hash, + 'new ' + part_name, length=new_part_info.size) def Run(self, new_parts, old_parts=None): """Applier entry point, invoking all update operations. From 2b75d6858d6e16567e0d2a5e1a1736f6d622f9c6 Mon Sep 17 00:00:00 2001 From: Captain Throwback Date: Sat, 2 Jul 2022 14:26:46 -0400 Subject: [PATCH 2/7] applier.py: remove os.path.exists This was changed with the skip_hash check and appears to break the ability for the tool to run in full payload mode. --- update_payload/applier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update_payload/applier.py b/update_payload/applier.py index ad974c5..9b813af 100644 --- a/update_payload/applier.py +++ b/update_payload/applier.py @@ -551,7 +551,7 @@ def _ApplyToPartition(self, operations, part_name, base_name, # Apply operations. with open(new_part_file_name, new_part_file_mode) as new_part_file: old_part_file = (open(old_part_file_name, 'r+b') - if os.path.exists(old_part_file_name) else None) + if old_part_file_name else None) try: self._ApplyOperations(operations, base_name, old_part_file, new_part_file, new_part_info.size) From d00b1b8fb0fd49004048c464269db7956e392917 Mon Sep 17 00:00:00 2001 From: Captain Throwback Date: Sat, 2 Jul 2022 14:30:10 -0400 Subject: [PATCH 3/7] Update README.md Specifying output folder name is optional Add six dependency & requirements.txt --- README.md | 18 ++++++++++++------ requirements.txt | 2 ++ 2 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index 6f037c1..0dc84c6 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,20 @@ -## System requirement +## System requirements - Python3, pip -- google protobuf for python `pip3 install protobuf` +- google protobuf for python, six +``` +pip3 install -r requirements.txt +``` ### Full OTA - -- ./extract.py [--skip_hash] --output_dir output/ payload.bin -- This will start to extract the images within the payload.bin file to the output folder you are in. +``` +./extract.py [--skip_hash] [--output_dir] payload.bin +``` +- This will start to extract the images within the payload.bin file to a folder named "output" by default (a different folder name can be specified). ### Incremental OTA - Copy original images (from full OTA or dumped from devices) to old folder -- ./extract.py [--skip_hash] --output_dir output/ --old_dir old/ payload.bin +``` +./extract.py [--skip_hash] [--output_dir] --old_dir payload.bin +``` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8ae93ea --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +protobuf>=3.19.3 +six>=1.16.0 From 41750faa4c20576828b6d5fdd4c0bc531e85dd5f Mon Sep 17 00:00:00 2001 From: Captain Throwback Date: Sat, 2 Jul 2022 23:07:26 -0400 Subject: [PATCH 4/7] README: clean up script syntax --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0dc84c6..9b81c71 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ pip3 install -r requirements.txt ### Full OTA ``` -./extract.py [--skip_hash] [--output_dir] payload.bin +./extract.py [--skip_hash] [--output dir ] payload.bin ``` - This will start to extract the images within the payload.bin file to a folder named "output" by default (a different folder name can be specified). ### Incremental OTA -- Copy original images (from full OTA or dumped from devices) to old folder +- Copy original images (from full OTA or dumped from devices) to old_dir folder ``` -./extract.py [--skip_hash] [--output_dir] --old_dir payload.bin +./extract.py [--skip_hash] [--output dir ] --old_dir payload.bin ``` From 65e967dde53158074f490a97b3b7b790dbb660b8 Mon Sep 17 00:00:00 2001 From: superr Date: Mon, 14 Aug 2023 13:11:22 -0400 Subject: [PATCH 5/7] Add command line argument "--ignore_block_size" On some newer devices, script will error if destination image size is larger than the source. This is possible on virtual AB devices and should be allowed in some cases. It should be noted that this may result in unusable images in some cases. Use with caution. --- README.md | 2 +- extract.py | 8 +++++--- requirements.txt | 2 +- update_payload/applier.py | 5 +++-- 4 files changed, 10 insertions(+), 7 deletions(-) mode change 100755 => 100644 extract.py diff --git a/README.md b/README.md index 9b81c71..4cb4bcb 100644 --- a/README.md +++ b/README.md @@ -16,5 +16,5 @@ pip3 install -r requirements.txt - Copy original images (from full OTA or dumped from devices) to old_dir folder ``` -./extract.py [--skip_hash] [--output dir ] --old_dir payload.bin +./extract.py [--skip_hash] [--ignore_block_size] [--output dir ] [--old_dir ] payload.bin ``` diff --git a/extract.py b/extract.py old mode 100755 new mode 100644 index 9fb159e..885a685 --- a/extract.py +++ b/extract.py @@ -20,7 +20,7 @@ def list_content(payload_file_name): part.new_partition_info.size)) -def extract(payload_file_name, output_dir="output", old_dir="old", partition_names=None, skip_hash=None): +def extract(payload_file_name, output_dir="output", old_dir="old", partition_names=None, skip_hash=None, ignore_block_size=None): try: os.makedirs(output_dir) except OSError as e: @@ -34,7 +34,7 @@ def extract(payload_file_name, output_dir="output", old_dir="old", partition_nam payload = update_payload.Payload(payload_file) payload.Init() - helper = applier.PayloadApplier(payload) + helper = applier.PayloadApplier(payload, ignore_block_size) for part in payload.manifest.partitions: if partition_names and part.partition_name not in partition_names: continue @@ -74,9 +74,11 @@ def extract(payload_file_name, output_dir="output", old_dir="old", partition_nam help="List the partitions included in the payload.bin") parser.add_argument("--skip_hash", action="store_true", help="Skip the hash check for individual img files") + parser.add_argument("--ignore_block_size", action="store_true", + help="Ignore block size") args = parser.parse_args() if args.list_partitions: list_content(args.payload) else: - extract(args.payload, args.output_dir, args.old_dir, args.partitions, args.skip_hash) + extract(args.payload, args.output_dir, args.old_dir, args.partitions, args.skip_hash, args.ignore_block_size) diff --git a/requirements.txt b/requirements.txt index 8ae93ea..c950008 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -protobuf>=3.19.3 +protobuf<=3.20.0 six>=1.16.0 diff --git a/update_payload/applier.py b/update_payload/applier.py index 9b813af..13aa84b 100644 --- a/update_payload/applier.py +++ b/update_payload/applier.py @@ -206,7 +206,7 @@ class PayloadApplier(object): applying an update payload. """ - def __init__(self, payload, bsdiff_in_place=True, bspatch_path="./bspatch", + def __init__(self, payload, ignore_block_size, bsdiff_in_place=True, bspatch_path="./bspatch", puffpatch_path="./puffin", truncate_to_expected_size=True): """Initialize the applier. @@ -221,6 +221,7 @@ def __init__(self, payload, bsdiff_in_place=True, bspatch_path="./bspatch", """ assert payload.is_init, 'uninitialized update payload' self.payload = payload + self.ignore_block_size = ignore_block_size self.block_size = payload.manifest.block_size self.minor_version = payload.manifest.minor_version self.bsdiff_in_place = bsdiff_in_place @@ -271,7 +272,7 @@ def _ApplyReplaceOperation(self, op, op_name, out_data, part_file, part_size): part_size)) # Make sure that we have enough data to write. - if data_end >= data_length + block_size: + if not self.ignore_block_size and (data_end >= data_length + block_size): raise PayloadError( '%s: more dst blocks than data (even with padding)') From 738ce114ac7aae118c2233a21982117b1178e117 Mon Sep 17 00:00:00 2001 From: Captain Throwback Date: Fri, 1 Mar 2024 17:45:39 -0500 Subject: [PATCH 6/7] Add support for arm[64] Separate arm & arm64 binaries will still be needed, which need to be installed to system/bin on mobile devices. The necessary library dependencies will also already have to be present on the device (most are standard on current Android versions) --- extract.py | 10 +++++++++- update_payload/applier.py | 15 +++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) mode change 100644 => 100755 extract.py diff --git a/extract.py b/extract.py old mode 100644 new mode 100755 index 885a685..e156f32 --- a/extract.py +++ b/extract.py @@ -4,11 +4,19 @@ import errno import glob import os +import platform import update_payload from update_payload import applier -os.environ['LD_LIBRARY_PATH'] = './lib64/' +if platform.machine == 'x86_64': + os.environ['LD_LIBRARY_PATH'] = './lib64/' +if platform.machine == 'x86': + os.environ['LD_LIBRARY_PATH'] = './lib/' +elif platform.machine == 'aarch64': + os.environ['LD_LIBRARY_PATH'] = '/system/lib64:/system/lib' +elif platform.machine == 'arm': + os.environ['LD_LIBRARY_PATH'] = '/system/lib' def list_content(payload_file_name): with open(payload_file_name, 'rb') as payload_file: diff --git a/update_payload/applier.py b/update_payload/applier.py index 13aa84b..14cb0aa 100644 --- a/update_payload/applier.py +++ b/update_payload/applier.py @@ -45,6 +45,7 @@ except ImportError: pass import os +import platform import subprocess import sys import tempfile @@ -206,8 +207,8 @@ class PayloadApplier(object): applying an update payload. """ - def __init__(self, payload, ignore_block_size, bsdiff_in_place=True, bspatch_path="./bspatch", - puffpatch_path="./puffin", truncate_to_expected_size=True): + def __init__(self, payload, ignore_block_size, arch='x86_64', bspatch_path='bspatch', puffpatch_path='puffin', + bsdiff_in_place=True, truncate_to_expected_size=True): """Initialize the applier. Args: @@ -224,9 +225,15 @@ def __init__(self, payload, ignore_block_size, bsdiff_in_place=True, bspatch_pat self.ignore_block_size = ignore_block_size self.block_size = payload.manifest.block_size self.minor_version = payload.manifest.minor_version + self.arch = platform.machine() + if self.arch == 'x86_64' or self.arch == 'x86': + self.bspatch_path = "./bspatch" or bspatch_path + self.puffpatch_path = "./puffin" or puffpatch_path + else: + self.bspatch_path = "/system/bin/bspatch" or bspatch_path + self.puffpatch_path = "/system/bin/puffin" or puffpatch_path + self.bsdiff_in_place = bsdiff_in_place - self.bspatch_path = bspatch_path or 'bspatch' - self.puffpatch_path = puffpatch_path or 'puffin' self.truncate_to_expected_size = truncate_to_expected_size def _ApplyReplaceOperation(self, op, op_name, out_data, part_file, part_size): From ba8e3c6443bdc62b2b51ab25a7116a07e89e6839 Mon Sep 17 00:00:00 2001 From: Captain Throwback Date: Sat, 24 Aug 2024 13:05:19 -0400 Subject: [PATCH 7/7] Update paths to /data/local Using system/bin creates an overlay which can be detected. Module support zip has also been updated --- update_payload/applier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update_payload/applier.py b/update_payload/applier.py index 14cb0aa..2b205b3 100644 --- a/update_payload/applier.py +++ b/update_payload/applier.py @@ -230,8 +230,8 @@ def __init__(self, payload, ignore_block_size, arch='x86_64', bspatch_path='bspa self.bspatch_path = "./bspatch" or bspatch_path self.puffpatch_path = "./puffin" or puffpatch_path else: - self.bspatch_path = "/system/bin/bspatch" or bspatch_path - self.puffpatch_path = "/system/bin/puffin" or puffpatch_path + self.bspatch_path = "/data/local/bin/bspatch" or bspatch_path + self.puffpatch_path = "/data/local/bin/puffin" or puffpatch_path self.bsdiff_in_place = bsdiff_in_place self.truncate_to_expected_size = truncate_to_expected_size