From 08e1c01a71a391132c3af7d1e15f8d3df4155b62 Mon Sep 17 00:00:00 2001 From: figment Date: Sat, 21 Nov 2015 10:53:28 -0800 Subject: [PATCH 1/6] Add support for hfloat --- pyffi/formats/nif/__init__.py | 1 + pyffi/object_models/common.py | 67 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/pyffi/formats/nif/__init__.py b/pyffi/formats/nif/__init__.py index 6ce653800..4132402f6 100644 --- a/pyffi/formats/nif/__init__.py +++ b/pyffi/formats/nif/__init__.py @@ -396,6 +396,7 @@ class NifFormat(FileFormat): char = pyffi.object_models.common.Char short = pyffi.object_models.common.Short ushort = pyffi.object_models.common.UShort + hfloat = pyffi.object_models.common.HFloat float = pyffi.object_models.common.Float BlockTypeIndex = pyffi.object_models.common.UShort StringIndex = pyffi.object_models.common.UInt diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index 9757a4072..d933e940c 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -414,6 +414,73 @@ def get_hash(self, data=None): """ return int(self.get_value()*200) + +class HFloat(Float, EditableFloatSpinBox): + """Implementation of a 32-bit float.""" + + def __init__(self, **kwargs): + """Initialize the float.""" + super(HFloat, self).__init__(**kwargs) + self._value = float() + + @classmethod + def toFloat(cls, bom, value): + exponent = value & 0x00007C00 + if not exponent: return float() + bits = ((value & 32768) << 16) | \ + ((exponent + 0x0001C000) | (value & 0x3ff)) << 13 + return struct.unpack(bom+"f", struct.pack(bom+"I", bits))[0] + + @classmethod + def fromFloat(cls, bom, value): + if value > 131008.000: + bits = 0x47FFE000 + else: + bits = struct.unpack(bom+"I", struct.pack(bom+"f", value))[0] + if (bits & 0x7FFFFFFF) < 0x38800000: return int() + result = ((bits + 0x48000000) & ~0x3ff) | (bits & 0x3ff) + return ((result >> 13) & 0xFFFF) | ((bits & 0x80000000) >> 16) + + def read(self, stream, data): + """Read value from stream. + + :param stream: The stream to read from. + :type stream: file + """ + bom = data._byte_order + boi = bom + "H" + self._value = HFloat.toFloat(bom, struct.unpack(boi,stream.read(2))[0]) + + def write(self, stream, data): + """Write value to stream. + + :param stream: The stream to write to. + :type stream: file + """ + bom = data._byte_order + boi = bom + "H" + try: + stream.write(struct.pack(bof, HFloat.fromFloat(self._value))) + except OverflowError: + logger = logging.getLogger("pyffi.object_models") + logger.warn("float value overflow, writing NaN") + stream.write(struct.pack(boi, 0x7fff)) + + def get_size(self, data=None): + """Return number of bytes this type occupies in a file. + + :return: Number of bytes. + """ + return 2 + + def get_hash(self, data=None): + """Return a hash value for this value. Currently implemented + with precision 1/200. + + :return: An immutable object that can be used as a hash. + """ + return HFloat.fromFloat(self.get_value()) + class ZString(BasicBase, EditableLineEdit): """String of variable length (null terminated). From fc54f849290d2435d24c519864b0b89c1a059821 Mon Sep 17 00:00:00 2001 From: figment Date: Sat, 21 Nov 2015 10:53:49 -0800 Subject: [PATCH 2/6] Fix html report for python 3 --- pyffi/spells/nif/dump.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyffi/spells/nif/dump.py b/pyffi/spells/nif/dump.py index f24461318..c9e5fd1c1 100644 --- a/pyffi/spells/nif/dump.py +++ b/pyffi/spells/nif/dump.py @@ -256,6 +256,8 @@ def branchentry(self, branch): def toastexit(cls, toaster): if toaster.reports_per_blocktype: rows = [] + rows.append( "" ) + rows.append( "" ) rows.append( "" ) rows.append( "Report" ) rows.append( "" ) @@ -275,6 +277,7 @@ def toastexit(cls, toaster): @classmethod def browser(cls, htmlstr): + htmlbytes = bytes(htmlstr,'UTF-8') """Display html in the default web browser without creating a temp file. @@ -282,10 +285,15 @@ def browser(cls, htmlstr): with a URL to retrieve html from that server. """ class RequestHandler(http.server.BaseHTTPRequestHandler): + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() def do_GET(self): + self.do_HEAD() bufferSize = 1024*1024 - for i in range(0, len(htmlstr), bufferSize): - self.wfile.write(htmlstr[i:i+bufferSize]) + for i in range(0, len(htmlbytes), bufferSize): + self.wfile.write(htmlbytes[i:i+bufferSize]) server = http.server.HTTPServer(('127.0.0.1', 0), RequestHandler) webbrowser.open('http://127.0.0.1:%s' % server.server_port) From 21b4e002fb2b626dd926c95ce74b86559871afce Mon Sep 17 00:00:00 2001 From: figment Date: Sat, 21 Nov 2015 11:11:30 -0800 Subject: [PATCH 3/6] Fix documentation strings --- pyffi/object_models/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index d933e940c..2d58025f5 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -416,7 +416,7 @@ def get_hash(self, data=None): class HFloat(Float, EditableFloatSpinBox): - """Implementation of a 32-bit float.""" + """Implementation of a 16-bit float.""" def __init__(self, **kwargs): """Initialize the float.""" @@ -475,7 +475,7 @@ def get_size(self, data=None): def get_hash(self, data=None): """Return a hash value for this value. Currently implemented - with precision 1/200. + with the short form. :return: An immutable object that can be used as a hash. """ From 2f3da0ef212a1ee13638d2e201f3262c9590d4f2 Mon Sep 17 00:00:00 2001 From: figment Date: Sat, 21 Nov 2015 16:24:14 -0800 Subject: [PATCH 4/6] Fix byte ordermark code --- pyffi/object_models/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index 2d58025f5..35b2221ff 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -460,7 +460,7 @@ def write(self, stream, data): bom = data._byte_order boi = bom + "H" try: - stream.write(struct.pack(bof, HFloat.fromFloat(self._value))) + stream.write(struct.pack(boi, HFloat.fromFloat(bom, self._value))) except OverflowError: logger = logging.getLogger("pyffi.object_models") logger.warn("float value overflow, writing NaN") From 026df7bbc2f72edac70bcd07b6a6507e2cf52d77 Mon Sep 17 00:00:00 2001 From: figment Date: Sat, 21 Nov 2015 21:24:26 -0800 Subject: [PATCH 5/6] Implement hfloat accurate which handles small and large values more accurately --- pyffi/object_models/common.py | 164 ++++++++++++++++++++++++++++++++-- 1 file changed, 159 insertions(+), 5 deletions(-) diff --git a/pyffi/object_models/common.py b/pyffi/object_models/common.py index 35b2221ff..ec9948936 100644 --- a/pyffi/object_models/common.py +++ b/pyffi/object_models/common.py @@ -423,16 +423,17 @@ def __init__(self, **kwargs): super(HFloat, self).__init__(**kwargs) self._value = float() - @classmethod - def toFloat(cls, bom, value): + @staticmethod + def toFloatFast(bom, value): exponent = value & 0x00007C00 + mantissa = value & 0x3ff if not exponent: return float() bits = ((value & 32768) << 16) | \ - ((exponent + 0x0001C000) | (value & 0x3ff)) << 13 + ((exponent + 0x0001C000) | mantissa) << 13 return struct.unpack(bom+"f", struct.pack(bom+"I", bits))[0] - @classmethod - def fromFloat(cls, bom, value): + @staticmethod + def fromFloatFast(bom, value): if value > 131008.000: bits = 0x47FFE000 else: @@ -441,6 +442,159 @@ def fromFloat(cls, bom, value): result = ((bits + 0x48000000) & ~0x3ff) | (bits & 0x3ff) return ((result >> 13) & 0xFFFF) | ((bits & 0x80000000) >> 16) + @staticmethod + def half_sub( ha, hb ): + return half_add( ha, hb ^ 0x8000 ); + + @staticmethod + def h_sels( test, a, b ): + mask = test >> 31 + return (a & mask) | (b & ~mask) + + @staticmethod + def h_selb( mask, a, b ): + return (a & b) | (b & ~mask) + + @staticmethod + def h_cntlz( x ): + x1 = x | (x >> 1) + x2 = x1 >> 2 + x3 = x1 | x2 + x4 = x3 >> 4 + x5 = x3 | x4 + x6 = x5 >> 8 + x7 = x5 | x6 + x8 = x7 >> 16 + x9 = x7 | x8 + xA = ~x9 + xB = xA >> 1 + xC = xB & 0x55555555 + xD = xA - xC + xE = xD & 0x33333333 + xF = xD >> 2 + x10 = xF & 0x33333333 + x11 = xE + x10 + x12 = x11 >> 4 + x13 = x11 + x12 + x14 = x13 & 0x0f0f0f0f + x15 = x14 >> 8 + x16 = x14 + x15 + x19 = (x16 + (x16 >> 16)) & 0x0000003f + return ( x19 ) + + @staticmethod + def toFloatAccurate(bom, value): + h = int(value) + h_e_mask = 0x00007c00 + h_m_mask = 0x000003ff + h_s_mask = 0x00008000 + h_f_s_pos_offset = 0x00000010 + h_f_e_pos_offset = 0x0000000d + h_f_bias_offset = 0x0001c000 + f_e_mask = 0x7f800000 + f_m_mask = 0x007fffff + h_f_e_denorm_bias = 0x0000007e + h_f_m_denorm_sa_bias = 0x00000008 + f_e_pos = 0x00000017 + h_e_mask_minus_one = 0x00007bff + h_e = h & h_e_mask; + h_m = h & h_m_mask + h_s = h & h_s_mask + h_e_f_bias = h_e + h_f_bias_offset + h_m_nlz = HFloat.h_cntlz( h_m ) + f_s = h_s << h_f_s_pos_offset + f_e = h_e_f_bias << h_f_e_pos_offset + f_m = h_m << h_f_e_pos_offset + f_em = f_e | f_m + h_f_m_sa = h_m_nlz - h_f_m_denorm_sa_bias + f_e_denorm_unpacked = h_f_e_denorm_bias - h_f_m_sa + h_f_m = h_m << h_f_m_sa if h_m else 0 + f_m_denorm = h_f_m & f_m_mask + f_e_denorm = f_e_denorm_unpacked << f_e_pos + f_em_denorm = f_e_denorm | f_m_denorm + f_em_nan = f_e_mask | f_m + is_e_eqz_msb = h_e - 1 + is_m_nez_msb = -h_m + is_e_flagged_msb = h_e_mask_minus_one - h_e + is_zero_msb = is_e_eqz_msb & ~is_m_nez_msb + is_inf_msb = is_e_flagged_msb & ~is_m_nez_msb + is_denorm_msb = is_m_nez_msb & is_e_eqz_msb + is_nan_msb = is_e_flagged_msb & is_m_nez_msb + is_zero = is_zero_msb >> 31 + f_zero_result = f_em & ~is_zero + f_denorm_result = HFloat.h_sels( is_denorm_msb, f_em_denorm, f_zero_result ) + f_inf_result = HFloat.h_sels( is_inf_msb, f_e_mask, f_denorm_result) + f_nan_result = HFloat.h_sels( is_nan_msb, f_em_nan, f_inf_result ) + f_result = f_s | f_nan_result + return struct.unpack(bom+"f", struct.pack(bom+"I", f_result))[0] + + @staticmethod + def fromFloatAccurate(bom, value): + f = struct.unpack(bom+"I", struct.pack(bom+"f", value))[0] + one = 0x00000001 + f_s_mask = 0x80000000 + f_e_mask = 0x7f800000 + f_m_mask = 0x007fffff + f_m_hidden_bit = 0x00800000 + f_m_round_bit = 0x00001000 + f_snan_mask = 0x7fc00000 + f_e_pos = 0x00000017 + h_e_pos = 0x0000000a + h_e_mask = 0x00007c00 + h_snan_mask = 0x00007e00 + h_e_mask_value = 0x0000001f + f_h_s_pos_offset = 0x00000010 + f_h_bias_offset = 0x00000070 + f_h_m_pos_offset = 0x0000000d + h_nan_min = 0x00007c01 + f_h_e_biased_flag = 0x0000008f + f_s = f & f_s_mask + f_e = f & f_e_mask + h_s = f_s >> f_h_s_pos_offset + f_m = f & f_m_mask + f_e_amount = f_e >> f_e_pos + f_e_half_bias = f_e_amount - f_h_bias_offset + f_snan = f & f_snan_mask + f_m_round_mask = f_m & f_m_round_bit + f_m_round_offset = f_m_round_mask << one + f_m_rounded = f_m + f_m_round_offset + f_m_denorm_sa = one - f_e_half_bias + f_m_with_hidden = f_m_rounded | f_m_hidden_bit + f_m_denorm = f_m_with_hidden >> f_m_denorm_sa if f_m_denorm_sa >= 0 else 0 + h_m_denorm = f_m_denorm >> f_h_m_pos_offset + f_m_rounded_overflow = f_m_rounded & f_m_hidden_bit + m_nan = f_m >> f_h_m_pos_offset + h_em_nan = h_e_mask | m_nan + h_e_norm_overflow_offset = f_e_half_bias + 1 + h_e_norm_overflow = h_e_norm_overflow_offset << h_e_pos + h_e_norm = f_e_half_bias << h_e_pos + h_m_norm = f_m_rounded >> f_h_m_pos_offset + h_em_norm = h_e_norm | h_m_norm + is_h_ndenorm_msb = f_h_bias_offset - f_e_amount + is_f_e_flagged_msb = f_h_e_biased_flag - f_e_half_bias + is_h_denorm_msb = ~is_h_ndenorm_msb + is_f_m_eqz_msb = f_m - 1 + is_h_nan_eqz_msb = m_nan - 1 + is_f_inf_msb = is_f_e_flagged_msb & is_f_m_eqz_msb + is_f_nan_underflow_msb = is_f_e_flagged_msb & is_h_nan_eqz_msb + is_e_overflow_msb = h_e_mask_value - f_e_half_bias + is_h_inf_msb = is_e_overflow_msb | is_f_inf_msb + is_f_nsnan_msb = f_snan - f_snan_mask + is_m_norm_overflow_msb = -f_m_rounded_overflow + is_f_snan_msb = ~is_f_nsnan_msb + h_em_overflow_result = HFloat.h_sels( is_m_norm_overflow_msb, h_e_norm_overflow, h_em_norm ) + h_em_nan_result = HFloat.h_sels( is_f_e_flagged_msb, h_em_nan, h_em_overflow_result ) + h_em_nan_underflow_result = HFloat.h_sels( is_f_nan_underflow_msb, h_nan_min, h_em_nan_result ) + h_em_inf_result = HFloat.h_sels( is_h_inf_msb, h_e_mask, h_em_nan_underflow_result ) + h_em_denorm_result = HFloat.h_sels( is_h_denorm_msb, h_m_denorm, h_em_inf_result ) + h_em_snan_result = HFloat.h_sels( is_f_snan_msb, h_snan_mask, h_em_denorm_result ) + h_result = h_s | h_em_snan_result + return h_result + + toFloat = toFloatAccurate + + fromFloat = fromFloatAccurate + def read(self, stream, data): """Read value from stream. From 86d6a2df81f12fc4b9664074577ceb4311bec4fb Mon Sep 17 00:00:00 2001 From: figment Date: Sun, 22 Nov 2015 07:16:05 -0800 Subject: [PATCH 6/6] Add hfloat license --- LICENSE.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/LICENSE.rst b/LICENSE.rst index 4548dd0fc..5951359ca 100644 --- a/LICENSE.rst +++ b/LICENSE.rst @@ -46,3 +46,26 @@ All rights reserved. See http://www.havok.com for details. PyFFI uses xdelta3. Copyright 2001-2010 Joshua P. MacDonald xdelta3 is released under the GPLv2. See external/xdelta3.0z/COPYING for details. + +======================================= + +// Branch-free implementation of half-precision (16 bit) floating point +// Copyright 2006 Mike Acton +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE