diff --git a/utils/zx7/CLI/Makefile b/utils/zx7/CLI/Makefile new file mode 100644 index 0000000..649c4e1 --- /dev/null +++ b/utils/zx7/CLI/Makefile @@ -0,0 +1,75 @@ +#--------------------------------------------------------------------------------- +# Clear the implicit built in rules +#--------------------------------------------------------------------------------- +.SUFFIXES: + +CXX=g++ +AR=ar + +#--------------------------------------------------------------------------------- +# TARGET is the name of the output +# BUILD is the directory where object files & intermediate files will be placed +# SOURCES is a list of directories containing source code +# INCLUDES is a list of directories containing extra header files +#--------------------------------------------------------------------------------- +TARGET := $(notdir $(CURDIR)) +SOURCES := ../src src +INCLUDES := + +#--------------------------------------------------------------------------------- +# options for code and library generation +#--------------------------------------------------------------------------------- + +CFLAGS = -O2 \ + -Wall \ + -funroll-loops \ + -fno-trapping-math \ + -fno-trapv \ + -Wno-switch \ + -UTARGET_PRIZM \ + $(INCLUDE) + +# ASFLAGS = $(CFLAGS) + +# LDFLAGS = -O2 -Wl,-static -Wl,-gc-sections + +#--------------------------------------------------------------------------------- +# list of directories containing libraries, this must be the top level containing +# include and lib +#--------------------------------------------------------------------------------- +LIBDIRS := + +OUTPUT := $(CURDIR)/zx7 + +#--------------------------------------------------------------------------------- +# automatically build a list of object files for our project +#--------------------------------------------------------------------------------- +CXXFILES := $(foreach dir,$(SOURCES),$(wildcard $(dir)/*.cpp)) +CFILES := $(foreach dir,$(SOURCES),$(wildcard $(dir)/*.c)) + +#--------------------------------------------------------------------------------- +# use CXX for linking C++ projects, CC for standard C +#--------------------------------------------------------------------------------- +LD := $(CXX) + +#--------------------------------------------------------------------------------- +# build a list of include paths +#--------------------------------------------------------------------------------- +INCLUDE := $(foreach dir,$(INCLUDES), -iquote $(CURDIR)/$(dir)) \ + $(foreach dir,$(LIBDIRS),-I$(dir)/include) \ + -I$(CURDIR)/$(BUILD) -I$(CURDIR)/.. -I/usr/include/ + +export VPATH := $(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) + +.PHONY: clean + +#--------------------------------------------------------------------------------- +$(OUTPUT): + $(CXX) $(CFLAGS) $(CXXFILES) $(CFILES) $< -o $(OUTPUT) + +#--------------------------------------------------------------------------------- +export CYGWIN := nodosfilewarning +clean: + $(RM) -f $(OUTPUT) + +#--------------------------------------------------------------------------------- diff --git a/utils/zx7/CLI/src/main.cpp b/utils/zx7/CLI/src/main.cpp new file mode 100644 index 0000000..dd4e609 --- /dev/null +++ b/utils/zx7/CLI/src/main.cpp @@ -0,0 +1,165 @@ +#include +#include + +#include +#include +#include +#include + +#include "zx7.h" + +void printHelp() { + std::cout << "Usage: zx7 [OPTION]... INPUT [OUTPUT]" << std::endl + << "Compress INPUT with zx7 to OUTPUT." << std::endl + << std::endl + << " -d, --decompress\tdecompress INPUT instead of compressing " + "it, outputting to stdout if OUTPUT not specified" + << std::endl + << " -h, --help\t\tdisplay this help and exit" << std::endl; +} + +class InputParser { +public: + InputParser(int &argc, char **argv) { + for (int i = 1; i < argc; ++i) + this->tokens.push_back(std::string(argv[i])); + } + + const std::string &getCmdOption(const std::string &option, bool pop = true) { + std::vector::const_iterator itr = + std::find(this->tokens.begin(), this->tokens.end(), option); + + if (itr != this->tokens.end() && ++itr != this->tokens.end()) { + if (pop) + this->tokens.erase(itr); + + return *itr; + } + + static const std::string empty_string(""); + return empty_string; + } + + bool cmdOptionExists(const std::string &option, bool pop = false) { + std::vector::const_iterator itr = + std::find(this->tokens.begin(), this->tokens.end(), option); + + if (itr == this->tokens.end()) + return false; + + if (pop) + this->tokens.erase(itr); + + return true; + } + + std::vector getRemainingOptions() const { return this->tokens; } + +private: + std::vector tokens; +}; + +// int main(int argc, char **argv) { +// InputParser input(argc, argv); +// if (input.cmdOptionExists("-h")) { +// // Do stuff +// } +// const std::string &filename = input.getCmdOption("-f"); +// if (!filename.empty()) { +// // Do interesting things ... +// } +// return 0; +// } + +enum mode { COMPRESS, DECOMPRESS }; + +int main(int argc, char **argv) { + InputParser input(argc, argv); + + try { + if (argc == 1) + throw "No arguments provided"; + + if (input.cmdOptionExists("--help") || input.cmdOptionExists("-h")) + throw "--help"; + + mode compression_mode = COMPRESS; + + // + is the same as ||, but evaluates both (i.e. removes --decompress and -d + // if both are used) + if (input.cmdOptionExists("-d", true) + + input.cmdOptionExists("--decompress", true)) + compression_mode = DECOMPRESS; + + const std::vector files = input.getRemainingOptions(); + + if (files.size() == 0) + throw "No input file"; + + if (files.size() == 1 && compression_mode == COMPRESS) + throw "No output file"; + + std::ifstream input_file(files[0], std::ios::binary); + const std::string input_data((std::istreambuf_iterator(input_file)), + std::istreambuf_iterator()); + const int input_size = input_data.size(); + + unsigned char *out_data; + unsigned int out_size; + + if (compression_mode == COMPRESS) { + out_size = ZX7Compress((unsigned char *)input_data.data(), input_size, + &out_data); + } else { + out_size = ZX7GetDecompressedSize((unsigned char *)input_data.data()); + out_data = (unsigned char *)malloc(out_size); + if (out_data == NULL) + throw std::runtime_error("Could not allocate output memory."); + + ZX7Decompress((unsigned char *)input_data.data(), out_data, out_size); + } + + if (files.size() == 1) { + std::cout << out_data; + } else { + std::ofstream output_file(files[1], std::ios::binary); + + output_file.write((char *)out_data, out_size); + } + } catch (const char *err) { + if (strcmp(err, "--help") != 0) + std::cout << "Error: " << err << "." << std::endl; + + printHelp(); + } + + // std::ifstream compress_file; + + // if (argc == 0) + + // compress_file.open() + + // char buff[] + + // compress_file = fopen("") + + // const char *srcData = "Bonjour, j'mappel docteur cahlos!"; + // unsigned char *outData; + // int compressedSize = ZX7Compress((const unsigned char *)srcData, + // strlen(srcData) + 1, &outData); + + // for (int i = 0; i < compressedSize; i++) { + // printf("%d,", outData[i]); + + // if (i % 64 == 63) + // printf("\n\t"); + // } + + // unsigned char destData[compressedSize]; + // ZX7Decompress(outData, destData, compressedSize); + + // printf("\n%s\n", destData); + + // return 0; + return 0; +} diff --git a/utils/zx7/LICENSE b/utils/zx7/LICENSE new file mode 100644 index 0000000..9aa236e --- /dev/null +++ b/utils/zx7/LICENSE @@ -0,0 +1,24 @@ +/* + * (c) Copyright 2012 by Einar Saukas. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * The name of its author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ \ No newline at end of file diff --git a/utils/zx7/Makefile b/utils/zx7/Makefile new file mode 100644 index 0000000..f4d2a4e --- /dev/null +++ b/utils/zx7/Makefile @@ -0,0 +1,114 @@ +#--------------------------------------------------------------------------------- +# Clear the implicit built in rules +#--------------------------------------------------------------------------------- +.SUFFIXES: + +ifeq ($(strip $(FXCGSDK)),) +export FXCGSDK := $(realpath ../../) +endif + +include $(FXCGSDK)/toolchain/prizm_rules + + +#--------------------------------------------------------------------------------- +# TARGET is the name of the output +# BUILD is the directory where object files & intermediate files will be placed +# SOURCES is a list of directories containing source code +# INCLUDES is a list of directories containing extra header files +#--------------------------------------------------------------------------------- +TARGET := $(notdir $(CURDIR)) +BUILD := build +SOURCES := src +INCLUDES := + +#--------------------------------------------------------------------------------- +# options for code and library generation +#--------------------------------------------------------------------------------- + +CFLAGS = -Os \ + -Wall \ + -funroll-loops \ + -fno-trapping-math \ + -fno-trapv \ + -Wno-switch \ + $(MACHDEP) $(INCLUDE) $(DEFINES) + +CXXFLAGS = $(CFLAGS) \ + -fpermissive \ + -fno-rtti \ + -fno-exceptions \ + -fno-threadsafe-statics \ + -fno-use-cxa-get-exception-ptr + +ASFLAGS = $(CFLAGS) + +LDFLAGS = $(MACHDEP) -O2 -T$(FXCGSDK)/toolchain/prizm.x -Wl,-static -Wl,-gc-sections + +#--------------------------------------------------------------------------------- +# list of directories containing libraries, this must be the top level containing +# include and lib +#--------------------------------------------------------------------------------- +LIBDIRS := + +ifneq ($(BUILD),$(notdir $(CURDIR))) + +export OUTPUT := $(CURDIR)/../../lib/lib$(TARGET) +export DEPSDIR := $(CURDIR)/$(BUILD) + +#--------------------------------------------------------------------------------- +# automatically build a list of object files for our project +#--------------------------------------------------------------------------------- +CFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.c))) +CPPFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.cpp))) +SFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.S))) + +#--------------------------------------------------------------------------------- +# use CXX for linking C++ projects, CC for standard C +#--------------------------------------------------------------------------------- +ifeq ($(strip $(CPPFILES)),) + export LD := $(CC) +else + export LD := $(CXX) +endif + +#--------------------------------------------------------------------------------- +# build a list of include paths +#--------------------------------------------------------------------------------- +export INCLUDE := $(foreach dir,$(INCLUDES), -iquote $(CURDIR)/$(dir)) \ + $(foreach dir,$(LIBDIRS),-I$(dir)/include) \ + -I$(CURDIR)/$(BUILD) -I$(LIBFXCG_INC) -I$(CURDIR) + +export VPATH := $(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) + +export OFILES := $(addsuffix .o,$(BINFILES)) \ + $(CPPFILES:.cpp=.o) $(CFILES:.c=.o) \ + $(SFILES:.S=.o) +.PHONY: $(BUILD) clean + +#--------------------------------------------------------------------------------- +all: $(BUILD) + @make --no-print-directory -C $(BUILD) -f $(CURDIR)/Makefile + +$(BUILD): + @[ -d $@ ] || mkdir $@ + +#--------------------------------------------------------------------------------- +export CYGWIN := nodosfilewarning +clean: + $(RM) -fr $(BUILD) $(OUTPUT).a $(OFILES) + +#--------------------------------------------------------------------------------- +else + +DEPENDS := $(OFILES:.o=.d) + +#--------------------------------------------------------------------------------- +# main targets +#--------------------------------------------------------------------------------- +$(OUTPUT).a: $(OFILES) + +-include $(DEPENDS) + +#--------------------------------------------------------------------------------- +endif + diff --git a/utils/zx7/README.md b/utils/zx7/README.md new file mode 100644 index 0000000..daa4f08 --- /dev/null +++ b/utils/zx7/README.md @@ -0,0 +1,9 @@ +# zx7 utility + +zx7 implementation by Einar Saukas allows for low impact decompression routines with very thorough offline compression. See the unmodified zx7.txt for more information. + +Use the compress routine in an offline tool to generate your data, and then decompress it on Prizm devices by using the ZX7Decompress function. + +Link to libzx7 by using -lzx7 in your project makefile, and include zx7/zx7.h to use. + +The CLI folder contains the source and Makefile for a CLI app to compress and decompress files with zx7. diff --git a/utils/zx7/src/compress.c b/utils/zx7/src/compress.c new file mode 100644 index 0000000..fb0e195 --- /dev/null +++ b/utils/zx7/src/compress.c @@ -0,0 +1,154 @@ +/* + * (c) Copyright 2012-2016 by Einar Saukas. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * The name of its author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#if !TARGET_PRIZM + +#include "zx7.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +static unsigned char* c_output_data; +static unsigned int c_output_index; +static unsigned int c_bit_index; +static int c_bit_mask; + +static inline void c_write_byte(int value) { + c_output_data[c_output_index++] = value; +} + +static void c_write_bit(int value) { + if (c_bit_mask == 0) { + c_bit_mask = 128; + c_bit_index = c_output_index; + c_write_byte(0); + } + if (value > 0) { + c_output_data[c_bit_index] |= c_bit_mask; + } + c_bit_mask >>= 1; +} + +static void write_elias_gamma(int value) { + int i; + + for (i = 2; i <= value; i <<= 1) { + c_write_bit(0); + } + while ((i >>= 1) > 0) { + c_write_bit(value & i); + } +} + +unsigned char *compress(Optimal *optimal, const unsigned char *input_data, unsigned int input_size, long skip, unsigned int *output_size) { + unsigned int input_index; + unsigned int input_prev; + int offset1; + int mask; + int i; + + /* calculate and allocate output buffer */ + input_index = input_size-1; + *output_size = (optimal[input_index].bits+18+7)/8 + 3; + unsigned char* ret = (unsigned char *)malloc(*output_size); + if (!ret) { + return 0; + } + + c_output_data = ret + 3; + + /* un-reverse optimal sequence */ + optimal[input_index].bits = 0; + while (input_index != skip) { + input_prev = input_index - (optimal[input_index].len > 0 ? optimal[input_index].len : 1); + optimal[input_prev].bits = input_index; + input_index = input_prev; + } + + c_output_index = 0; + c_bit_mask = 0; + + /* first byte is always literal */ + c_write_byte(input_data[input_index]); + + /* process remaining bytes */ + while ((input_index = optimal[input_index].bits) > 0) { + if (optimal[input_index].len == 0) { + + /* literal indicator */ + c_write_bit(0); + + /* literal value */ + c_write_byte(input_data[input_index]); + + } else { + + /* sequence indicator */ + c_write_bit(1); + + /* sequence length */ + write_elias_gamma(optimal[input_index].len-1); + + /* sequence offset */ + offset1 = optimal[input_index].offset-1; + if (offset1 < 128) { + c_write_byte(offset1); + } else { + offset1 -= 128; + c_write_byte((offset1 & 127) | 128); + for (mask = 1024; mask > 127; mask >>= 1) { + c_write_bit(offset1 & mask); + } + } + } + } + + /* sequence indicator */ + c_write_bit(1); + + /* end marker > MAX_LEN */ + for (i = 0; i < 16; i++) { + c_write_bit(0); + } + c_write_bit(1); + + // decompressed size is first three bytes + ret[0] = (input_size & 0xFF0000) >> 16; + ret[1] = (input_size & 0x00FF00) >> 8; + ret[2] = (input_size & 0x0000FF); + + free(optimal); + + return ret; +} + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/utils/zx7/src/dzx7.c b/utils/zx7/src/dzx7.c new file mode 100644 index 0000000..4a81268 --- /dev/null +++ b/utils/zx7/src/dzx7.c @@ -0,0 +1,134 @@ +/* + * (c) Copyright 2015 by Einar Saukas. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * The name of its author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "zx7.h" + +#ifdef __cplusplus +extern "C" { +#endif + +const unsigned char *d_input_data; +unsigned char *d_output_data; +unsigned int d_input_index; +unsigned int d_output_index; +unsigned int d_input_size; +int bit_mask; +int bit_value; + +static inline int d_read_byte() { + return d_input_data[d_input_index++]; +} + +static inline int d_read_bit() { + bit_mask >>= 1; + if (bit_mask == 0) { + bit_mask = 128; + bit_value = d_read_byte(); + } + return bit_value & bit_mask ? 1 : 0; +} + +static inline int read_elias_gamma() { + int i; + int value; + + i = 0; + while (!d_read_bit()) { + i++; + } + if (i > 15) { + return -1; + } + value = 1; + while (i--) { + value = value << 1 | d_read_bit(); + } + return value; +} + +static int read_offset() { + int value; + int i; + + value = d_read_byte(); + if (value < 128) { + return value; + } else { + i = d_read_bit(); + i = i << 1 | d_read_bit(); + i = i << 1 | d_read_bit(); + i = i << 1 | d_read_bit(); + return ((value & 127) | (i << 7)) + 128; + } +} + +static inline void d_write_byte(int value) { + d_output_data[d_output_index++] = value; +} + +void d_write_bytes(int offset, int length) { + int i; + while (length-- > 0) { + i = d_output_index-offset; + d_write_byte(d_output_data[i]); + } +} + +unsigned int ZX7GetDecompressedSize(const unsigned char* compressedData) { + return compressedData[0] * 65536 + compressedData[1] * 256 + compressedData[2]; +} + +int ZX7Decompress(const unsigned char* srcData, unsigned char* destData, unsigned int destLength) { + if (destLength < ZX7GetDecompressedSize(srcData) || !srcData || !destData) { + return -1; + } + + int length; + + d_input_data = srcData + 3; + d_output_data = destData; + + d_input_size = 0; + d_input_index = 0; + d_output_index = 0; + bit_mask = 0; + + d_write_byte(d_read_byte()); + while (1) { + if (!d_read_bit()) { + d_write_byte(d_read_byte()); + } else { + length = read_elias_gamma()+1; + if (length == 0) { + return 0; + } + d_write_bytes(read_offset()+1, length); + } + } +} + +#ifdef __cplusplus +} +#endif diff --git a/utils/zx7/src/optimize.c b/utils/zx7/src/optimize.c new file mode 100644 index 0000000..9e990e2 --- /dev/null +++ b/utils/zx7/src/optimize.c @@ -0,0 +1,139 @@ +/* + * (c) Copyright 2012-2016 by Einar Saukas. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * The name of its author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#if !TARGET_PRIZM + +#include "zx7.h" + +#ifdef __cplusplus +extern "C" { +#endif + + +#include + +static int elias_gamma_bits(int value) { + int bits; + + bits = 1; + while (value > 1) { + bits += 2; + value >>= 1; + } + return bits; +} + +static int count_bits(int offset, int len) { + return 1 + (offset > 128 ? 12 : 8) + elias_gamma_bits(len-1); +} + +Optimal* optimize(const unsigned char *input_data, unsigned int input_size, unsigned long skip) { + unsigned int *min; + unsigned int *max; + unsigned int *matches; + unsigned int *match_slots; + Optimal *optimal; + unsigned int *match; + int match_index; + int offset; + unsigned int len; + unsigned int best_len; + unsigned int bits; + unsigned int i; + + /* allocate all data structures at once */ + min = (unsigned int *)calloc(MAX_OFFSET+1, sizeof(unsigned int)); + max = (unsigned int *)calloc(MAX_OFFSET+1, sizeof(unsigned int)); + matches = (unsigned int *)calloc(256*256, sizeof(unsigned int)); + match_slots = (unsigned int *)calloc(input_size, sizeof(unsigned int)); + optimal = (Optimal *)calloc(input_size, sizeof(Optimal)); + + if (!min || !max || !matches || !match_slots || !optimal) { + return 0; + } + + /* index skipped bytes */ + for (i = 1; i <= skip; i++) { + match_index = input_data[i-1] << 8 | input_data[i]; + match_slots[i] = matches[match_index]; + matches[match_index] = i; + } + + /* first byte is always literal */ + optimal[skip].bits = 8; + + /* process remaining bytes */ + for (; i < input_size; i++) { + + optimal[i].bits = optimal[i-1].bits + 9; + match_index = input_data[i-1] << 8 | input_data[i]; + best_len = 1; + for (match = &matches[match_index]; *match != 0 && best_len < MAX_LEN; match = &match_slots[*match]) { + offset = i - *match; + if (offset > MAX_OFFSET) { + *match = 0; + break; + } + + for (len = 2; len <= MAX_LEN && i >= skip+len; len++) { + if (len > best_len) { + best_len = len; + bits = optimal[i-len].bits + count_bits(offset, len); + if (optimal[i].bits > bits) { + optimal[i].bits = bits; + optimal[i].offset = offset; + optimal[i].len = len; + } + } else if (max[offset] != 0 && i+1 == max[offset]+len) { + len = i-min[offset]; + if (len > best_len) { + len = best_len; + } + } + if (i < offset+len || input_data[i-len] != input_data[i-len-offset]) { + break; + } + } + min[offset] = i+1-len; + max[offset] = i; + } + match_slots[i] = matches[match_index]; + matches[match_index] = i; + } + + free(match_slots); + free(min); + free(max); + free(matches); + + return optimal; +} + + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/utils/zx7/src/zx7.c b/utils/zx7/src/zx7.c new file mode 100644 index 0000000..71b1c2a --- /dev/null +++ b/utils/zx7/src/zx7.c @@ -0,0 +1,46 @@ +/* + * (c) Copyright 2012-2016 by Einar Saukas. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * The name of its author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "zx7.h" + +#if !TARGET_PRIZM + +#ifdef __cplusplus +extern "C" { +#endif + +// ZX7 Compress the given data, outData is malloc'd and the return value is the length (first 3 bytes of data will be 24-bit size result for convenience) +unsigned int ZX7Compress(const unsigned char *srcData, unsigned int inLength, unsigned char** outData) { + unsigned int output_size; + *outData = compress(optimize(srcData, inLength, 0), srcData, inLength, 0, &output_size); + + return output_size; +} + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/utils/zx7/zx7.h b/utils/zx7/zx7.h new file mode 100644 index 0000000..fe3356f --- /dev/null +++ b/utils/zx7/zx7.h @@ -0,0 +1,62 @@ +/* + * (c) Copyright 2012-2016 by Einar Saukas. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * The name of its author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#if !TARGET_PRIZM + +typedef struct optimal_t { + unsigned int bits; + int offset; + int len; +} Optimal; + +#define MAX_OFFSET 2176 /* range 1..2176 */ +#define MAX_LEN 65536 /* range 2..65536 */ + +Optimal *optimize(const unsigned char *input_data, unsigned int input_size, unsigned long skip); + +unsigned char *compress(Optimal *optimal, const unsigned char *input_data, unsigned int input_size, long skip, unsigned int *output_size); + +// THOMAS : added these for my use: + +// ZX7 Compress the given data, outData is malloc'd and the return value is the length (first 3 bytes of data will be 24-bit size result for convenience) +unsigned int ZX7Compress(const unsigned char *srcData, unsigned int inLength, unsigned char** outData); + +#endif + +// Get decompressed size of ZX7Compress'd data +unsigned int ZX7GetDecompressedSize(const unsigned char* compressedData); + +// Decompress the given data. Returns 0 with no errors +int ZX7Decompress(const unsigned char* srcData, unsigned char* destData, unsigned int destLength); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/utils/zx7/zx7.txt b/utils/zx7/zx7.txt new file mode 100644 index 0000000..75dec9f --- /dev/null +++ b/utils/zx7/zx7.txt @@ -0,0 +1,353 @@ +======================= +"ZX7" - by Einar Saukas +======================= + +"ZX7" is an optimal LZ77/LZSS data compressor for all platforms, including the +ZX-Spectrum. + +Available implementations of standard LZ77/LZSS compression algorithm use either +a "greedy" or "flexible parsing" strategy, that cannot always guarantee the best +possible encoding. In comparison, "ZX7" provides a highly efficient compression +algorithm that always generate perfectly optimal LZ77/LZSS encoding. + + +===== +USAGE +===== + +To compress a file, use the command-line compressor as follows: + + zx7 Cobra.scr + +This will generate a compressed file called "Cobra.scr.zx7". + +Afterwards choose a decompressor routine in assembly Z80 (for the ZX-Spectrum), +according to your requirements for speed and size: + + * "Standard" routine: 69 bytes only + + * "Turbo" routine: 88 bytes, about 25% faster + + * "Mega" routine: 244 bytes, about 30% faster + +Finally compile the chosen decompressor routine and load the compressed file +somewhere in memory. To decompress data, just call the routine specifying the +source address of compressed data in HL and the target address in DE. + +For instance, if you compile the decompressor routine to address 65000, load +"Cobra.scr.zx7" at address 51200, and want to decompress it directly to the +screen, then execute the following code: + + LD HL, 51200 ; source address (put "Cobra.scr.zx7" there) + LD DE, 16384 ; target address (screen memory in this case) + CALL 65000 ; decompress routine compiled at this address + +It's also possible to decompress data into a memory area that partially overlaps +the compressed data itself (only if you won't need to decompress it again later, +obviously). In this case, the last address of compressed data must be at least +"delta" bytes higher than the last address of decompressed data. The exact value +of "delta" for each case is reported by ZX7 during compression. See image below: + + |------------------| compressed data + |---------------------------------| decompressed data + start >> <---> + delta + +For convenience, there's also a command-line decompressor that works as follows: + + dzx7 Cobra.scr.zx7 + + +===== +NOTES +===== + +For simplicity, provided implementation of ZX7 compressor (in C) will load all +data in memory at once, process everything, then write the output file directly. +If you want to compress a very large file (over 1Gb), it should take just a few +seconds (only a few times more than the time it takes to load the file itself +from disk), but it will need a modern computer with lots of memory. More +specifically, compressing n bytes of data requires approximately 17n bytes of +free memory (plus a small constant overhead). Technically it means compressing +within asymptotically optimal space O(n), asymptotically optimal expected time +O(n), and asymptotically optimal worst case time O(n*w) only. + +The provided ZX7 decompressor (in C) works even better. It writes the output +file while reading the compressed file, without keeping it in memory. Therefore +it always use the same amount of memory, regardless of file size. Thus even very +large compressed files (over 1Gb) can always be decompressed using very small +computers with limited memory, even if it took a lot of memory to compress it +originally. It means decompressing within asymptotically optimal space and time +O(n) only, although in this case it means storage space O(n) for input and +output files, and much smaller memory space O(w) for processing. + +As a matter of fact, it would be trivial to modify the ZX7 compressor to operate +with limited memory too. Theoretically, the ZX7 algorithm only requires optimal +memory space O(w) and optimal storage space O(n) for both compressing and +decompression n bytes of data with a sliding window of w bytes. However, such +compressor implementation would have to write and read intermediate files twice. +Since current ZX7 data format is focused on 8-bits Z80 machines using small data +blocks (typically under 64Kb), modifying the ZX7 compressor to generate +intermediate files is simply not worth it. + + +====== +EXTRAS +====== + +ZX7 compressor now contains a few extra "hidden" features, that are slightly +harder to use properly, and not supported by the ZX7 decompressor in C. These +features are only intended for very specific special cases, so you can safely +skip this entire section and simply keep using ZX7 regular features as usual. + +If you really want to use these new "hidden" features detailed in this section, +keep in mind that only certain ZX7 Assembly routines will be able to decompress +your data afterwards, and they will take additional steps to make it work. +Please read carefully these instructions before attempting to use any of them! + + +1. COMPRESSING BACKWARDS + +When using ZX7 for "in-place" decompression (decompressing data to overlap the +same memory area storing the compressed data), you must always leave a small +margin of "delta" bytes of compressed data at the end. However it won't work to +decompress some large data that will occupy all the upper memory until the last +memory address, since there won't be even a couple bytes left at the end. + +A possible workaround is to compress and decompress data backwards, starting at +the last memory address. Therefore you will only need to leave a small margin of +"delta" bytes of compressed data at the beginning instead. Technically, it will +require that lowest address of compressed data should be at least "delta" bytes +lower than lowest address of decompressed data. See image below: + + compressed data |------------------| + decompressed data |---------------------------------| + <---> << start + delta + +To compress a file backwards, use the command-line compressor as follows: + + zx7 -b Cobra.scr + +To decompress it later, you must call one of the supplied "backwards" variants +of the Assembly decompressor, specifying last source address of compressed data +in HL and last target address in DE. + +For instance, if you compile a "backwards" Assembly decompressor routine to +address 64000, load backwards compressed file "Cobra.scr.zx7" (with size 2450 +bytes) to address 51200, and want to decompress it directly to the ZX-Spectrum +screen (with 6912 bytes), then execute the following code: + + LD HL, 51200+2450-1 ; source (last address of "Cobra.scr.zx7") + LD DE, 16384+6912-1 ; target (last address of screen memory) + CALL 64000 ; backwards decompress routine + +Notice that compressing backwards may sometimes produce slightly smaller +compressed files in certain cases, slightly larger compressed files in others. +Overall it shouldn't make much difference either way. + + +2. COMPRESSING WITH PREFIX + +The LZ77/LZSS compression is achieved by "abbreviating repetitions", such that +certain sequences of bytes are replaced with much shorter references to previous +occurrences of these same sequences. For this reason, it's harder to get very +good compression ratio on very short files, or in the initial parts of larger +files, due to lack of choices for previous sequences that could be referenced. + +A possible improvement is to compress data while also taking into account what +else will be already stored in memory during decompression later. Thus the +compressed data may even contain shorter references to repetitions stored in +some previous "prefix" memory area, instead of just repetitions within the +decompressed area itself. + +An input file may contain both some prefix data to be referenced only, and the +actual data to be compressed. An optional parameter can specify how many bytes +must be skipped before compression. See below: + + compressed data + |-------------------| + prefix decompressed data + |--------------|---------------------------------| + start >> + <--------------> <---> + skip delta + +As usual, if you want to decompress data into a memory area that partially +overlaps the compressed data itself, the last address of compressed data must be +at least "delta" bytes higher than the last address of decompressed data. + +For instance, if you want the first 6144 bytes of a certain file to be skipped +(not compressed but possibly referenced), then use the command-line compressor +as follows: + + zx7 +6144 Cobra.cbr + +In practice, suppose an action game uses a few generic sprites that are common +for all levels (such as player graphics), and other sprites are specific for +each level (such as enemies). All generic sprites must stay always accessible at +a certain memory area, but any level specific data can be only decompressed as +needed, to the memory area immediately following it. In this case, the generic +sprites area could be used as prefix when compressing and decompressing each +level, in an attempt to improve compression. For instance, suppose generic +graphics are loaded from file "generic.gfx" to address 56000, occupying 2500 +bytes, and level specific graphics will be decompressed immediately afterwards, +to address 58500. To compress each level using "generic.gfx" as a 2500 bytes +prefix, use the command-line compressor as follows: + + copy /b generic.gfx+level_1.gfx prefixed_level_1.gfx + zx7 +2500 prefixed_level_1.gfx + + copy /b generic.gfx+level_2.gfx prefixed_level_2.gfx + zx7 +2500 prefixed_level_2.gfx + + copy /b generic.gfx+level_3.gfx prefixed_level_3.gfx + zx7 +2500 prefixed_level_3.gfx + +To decompress it later, you simply need to use one of the normal variants of the +Assembly decompressor, as usual. In this case, if you loaded compressed file +"prefixed_level_1.gfx.zx7" to address 48000 for instance, decompressing it will +require the following code: + + LD HL, 48000 ; source address (put "prefixed_level_1.gfx.zx7" there) + LD DE, 58500 ; target address (level specific memory area in this case) + CALL 65000 ; decompress routine compiled at this address + +However decompression will only work properly if exactly the same prefix data is +present in the memory area immediately preceding the decompression address. +Therefore you must be extremely careful to ensure the prefix area does not store +variables, self-modifying code, or anything else that may change prefix content +between compression and decompression. Also don't forget to recompress your +files whenever you modify a prefix! + +In certain cases, compressing with a prefix may considerably help compression. +In others, it may not even make any difference. It mostly depends on how much +similarity exists between data to be compressed and its provided prefix. + + +3. COMPRESSING BACKWARDS WITH SUFIX + +Both features above can be used together. A file can be compressed backwards, +with an optional parameter to specify how many bytes should be skipped (not +compressed but possibly referenced) from the end of the input file instead. See +below: + + compressed data + |-------------------| + decompressed data sufix + |---------------------------------|--------------| + << start + <---> <--------------> + delta skip + +As usual, if you want to decompress data into a memory area that partially +overlaps the compressed data itself, lowest address of compressed data must be +at least "delta" bytes lower than lowest address of decompressed data. + +For instance, if you want to skip the last 768 bytes of a certain input file and +compress everything else (possibly referencing this "sufix" of 768 bytes), then +use the command-line compressor as follows: + + zx7 -b +768 Cobra.cbr + +In previous example, suppose the action game now stores level-specific sprites +in the memory area from address 33000 to 33511 (512 bytes), just before generic +sprites that are stored from address 33512 to 34535 (1024 bytes). In this case, +these generic sprites could be used as sufix when compressing and decompressing +level-specific data as needed, in an attempt to improve compression. To compress +each level using "generic.gfx" as a 1024 bytes sufix, use the command-line +compressor as follows: + + copy /b "level_1.gfx+generic.gfx level_1_sufixed.gfx + zx7 -b +1024 level_1_sufixed.gfx + + copy /b "level_2.gfx+generic.gfx level_2_sufixed.gfx + zx7 -b +1024 level_2_sufixed.gfx + + copy /b "level_3.gfx+generic.gfx level_3_sufixed.gfx + zx7 -b +1024 level_3_sufixed.gfx + +To decompress it later, use the backwards variant of the Assembly decompressor. +In this case, if you compile a "backwards" decompressor routine to address +64000, and load compressed file "level_1_sufixed.gfx.zx7" (with 217 bytes) to +address 39000 for instance, decompressing it will require the following code: + + LD HL, 39000+217-1 ; source (last address of "level_1_sufixed.gfx.zx7") + LD DE, 33000+512-1 ; target (last address of level-specific data) + CALL 64000 ; backwards decompress routine + +Analogously, decompression will only work properly if exactly the same sufix +data is present in the memory area immediately following the decompression area. +Therefore you must be extremely careful to ensure the sufix area does not store +variables, self-modifying code, or anything else that may change sufix content +between compression and decompression. Also don't forget to recompress your +files whenever you modify a sufix! + +Also if you are using "in-place" decompression, you must leave a small margin of +"delta" bytes of compressed data just before the decompression area. + + +======= +LICENSE +======= + +The optimal C compressor is available under the "BSD-3" license. In practice, +this is relevant only if you want to modify its source code and/or incorporate +the compressor within your own products. Otherwise, if you just execute it to +compress files, you can simply ignore these conditions. + +The Z80 assembly decompressors can be used freely within your own ZX-Spectrum +programs (or any other Z80 platform), even for commercial releases. The only +condition is that you must indicate somehow in your documentation that you have +used "ZX7". + + +======= +HISTORY +======= + +2012-12-30: Initial release (with optimal compressor and Z80 decompressors) + +2012-12-31: Minor changes to source code since it wasn't strictly ANSI C (thanks + to Metalbrain!) + +2013-01-03: Minor changes to data format increasing maximum offset by 6% thus + improving compression, and some improvements in the "Standard" Z80 + routine (thanks to Antonio Villena!) + +2013-01-05: Further improvements in the "Standard" Z80 routine (thanks to + Metalbrain!) + +2013-02-25: Further improvements in the "Turbo" Z80 routine, released libraries + for z88dk and Boriel's ZX BASIC. + +2015-09-01: Implemented C decompressor, and updated compressor to calculate and + display "delta" + +2016-01-21: Improved documentation. + +2016-02-11: Added extra features. + +2016-02-12: Improved documentation once more. + +2016-06-10: Added command-line option "-f" + + +======= +CREDITS +======= + +The compressed file format is directly based (although slightly improved) on + Team Bomba's Bitbuster - http://www.teambomba.net/bombaman/downloadd26a.html + and Gasman's Bitbuster Extreme - www.west.co.tt/matt/speccy/apology/ + +Some of the size improvements used in "Standard" version were suggested by + Antonio Villena and Metalbrain. + +The main speed improvement used in "Turbo" version was originally suggested by + Urusergi for Magnus Lind's Exomizer - http://hem.bredband.net/magli143/exo/ + +The optimal LZ77/LZSS compress algorithm was invented by myself (Einar Saukas). + To the best of my knowledge, there was no similar high-performance solution + available. I thereby present this implementation as evidence of "prior art", + thus preventing anyone from ever patenting it. Software patents are evil!!!