PB Floating point variables

[MikeTrader]

Hello Charles,

I would like to create a function that recovers the value stored in a PowerBASIC DOUBLE.
http://www.rit.edu/~meseec/eecc250-w.../IEEE-754.html

The first step for me would be to code this in BASIC
Then I would like to develop a function for converting a DOUBLE to text.
Lastly perhaps create and ASM function to replace FORMAT$()

The DOUBLE is of interest here because all floating point numbers are returned by SQL lite as DOUBLE's.
I am not having success in accuratly recovering them.

[Charles Pegge]

Hi Mike,
I am puzzled because I thought the bit format of a DOUBLE, SINGLE etc is a universal standard. Can you explain what sort of errors you are getting?
Are they rounding errors or do they look like type conversion problems?

PS: couldn't get the link to work but this describes the standard well:
http://en.wikipedia.org/wiki/IEEE_754

[MikeTrader]

Yes, these are rounding errors.
After beating this around on the PB forum, I now realize that what i need is a BCD floating point variable, as the ieee binary spec is unable to represent some floats exactly.

As I think about this I realize this might mean writing a class of functions to handle addition, subtraction and multiplication and division. How insane an idea is that?

[José Roca]

 
As a former programmer of business applications, I used BCD in DOS times. Today I will use the DECIMAL data type. Here is my definition of the data type and the declarations of the available API functions:

Code Select Expand


TYPE DECIMAL
   wReserved AS WORD
   scale     AS BYTE    ' // The number of decimal places for the number.
                        ' // Valid values are from 0 to 28. So 12.345 is represented
                        ' // as 12345 with a scale of 3.
   sign      AS BYTE    ' // 0 for positive numbers or %DECIMAL_NEG (&H80) for
                        ' // negative numbers. So -1 is represented as
                        ' // 1 with the %DECIMAL_NEG bit set.
   Hi32      AS DWORD   ' // The high 32 bits of your number
   Lo64      AS QUAD    ' // The low 64 bits of your number. This is an _int64 (QUAD in PB)
END TYPE

%DECIMAL_NEG = &H80?

DECLARE FUNCTION VarUI1FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarUI1FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF BYTE _                        ' [out] BYTE * pbOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarI2FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarI2FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF INTEGER _                     ' [out] SHORT * psOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarI4FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarI4FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF LONG _                        ' [out] LONG * plOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarI8FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarI8FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF QUAD _                        ' [out] LONG64 FAR* pi64Out
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarR4FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarR4FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF SINGLE _                      ' [out] FLOAT *pfltOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarR8FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarR8FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL pdecIn
, BYREF DOUBLE _                      ' [out] double pdblOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDateFromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarDateFromDec" ( _
   BYREF ANY _                         ' DECIMAL *pdecIn
, BYREF DOUBLE _                      ' DATE *pdateOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarCyFromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarCyFromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL pdecIn
, BYREF CUR _                         ' [out] CY * pcyOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarBstrFromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarBstrFromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYVAL DWORD _                       ' [in] LCID lcid
, BYVAL DWORD _                       ' [in] ULONG dwFlags
, BYREF STRING _                      ' [out] BSTR *pbstrOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarBoolFromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarBoolFromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF INTEGER _                     ' [out] VARIANT_BOOL * pboolOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarI1FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarI1FromDec" ( _
   BYREF ANY _                         ' __in DECIMAL *pdecIn
, BYREF BYTE _                        ' __out CHAR *pcOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarUI2FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarUI2FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF WORD _                        ' [out] USHORT *puiOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarUI4FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarUI4FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF DWORD _                       ' [out] ULONG *pulOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarUI8FromDec _
   LIB "OLEAUT32.DLL" ALIAS "VarUI8FromDec" ( _
   BYREF ANY _                         ' [in] DECIMAL *pdecIn
, BYREF QUAD _                        ' [out] ULONG64 FAR* pi64Out
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromUI1 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromUI1" ( _
   BYVAL BYTE _                        ' [in] BYTE bIn
, BYREF ANY _                         ' [out] DECIMAL * pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromI2 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromI2" ( _
   BYVAL INTEGER _                     ' [in] SHORT uiIn
, BYREF ANY _                         ' [out] DECIMAL * pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromI4 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromI4" ( _
   BYVAL LONG _                        ' [in] LONG lIn
, BYREF ANY _                         ' [out] DECIMAL * pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromI8 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromI8" ( _
   BYVAL QUAD _                        ' [in] i64In
, BYREF ANY _                         ' [out] DECIMAL * pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromR4 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromR4" ( _
   BYVAL SINGLE _                      ' [in] FLOAT fltIn
, BYREF ANY _                         ' [out] DECIMAL * pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromR8 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromR8" ( _
   BYVAL DOUBLE _                      ' [in] double dblIn
, BYREF ANY _                         ' [out] DECIMAL * pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromDate _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromDate" ( _
   BYVAL DOUBLE _                      ' [in] DATE dateIn
, BYREF ANY _                         ' [out] DECIMAL * pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromCy _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromCy" ( _
   BYVAL CUR _                         ' [in] CURRENCY cyIn
, BYREF ANY _                         ' [out] DECIMAL pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromStr _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromStr" ( _
   BYVAL DWORD _                       ' OLECHAR *strIn
, BYVAL DWORD _                       ' LCID lcid
, BYVAL DWORD _                       ' ULONG dwFlags
, BYREF ANY _                         ' DECIMAL *pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromDisp _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromDisp" ( _
   BYVAL IDispatch _                   ' IDispatch *pdispIn
, BYVAL DWORD _                       ' LCID lcid
, BYREF ANY _                         ' DECIMAL *pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromBool _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromBool" ( _
   BYVAL INTEGER _                     ' VARIANT_BOOL boolIn
, BYREF ANY _                         ' DECIMAL *pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromI1 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromI1" ( _
   BYVAL BYTE _                        ' CHAR cIn
, BYREF ANY _                         ' DECIMAL *pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromUI2 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromUI2" ( _
   BYVAL WORD _                        ' USHORT uiIn
, BYREF ANY _                         ' DECIMAL *pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromUI4 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromUI4" ( _
   BYVAL DWORD _                       ' ULONG ulIn
, BYREF ANY _                         ' DECIMAL *pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFromUI8 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFromUI8" ( _
   BYVAL QUAD _                        ' ULONG64 ui64In
, BYREF ANY _                         ' DECIMAL *pdecOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarI4FromI8 _
   LIB "OLEAUT32.DLL" ALIAS "VarI4FromI8" ( _
   BYVAL QUAD _                        ' LONG64 i64In
, BYREF LONG _                        ' LONG *plOut
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarI4FromUI8 _
   LIB "OLEAUT32.DLL" ALIAS "VarI4FromUI8" ( _
   BYVAL QUAD _                        ' LONG64 i64In
, BYREF LONG _                        ' LONG *plOut
   ) AS LONG                           ' HRESULT

'// Decimal math
'//
DECLARE FUNCTION VarDecAdd _
   LIB "OLEAUT32.DLL" ALIAS "VarDecAdd" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecLeft
, BYREF ANY _                         ' [in] LPDECIMAL pdecRight
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecDiv _
   LIB "OLEAUT32.DLL" ALIAS "VarDecDiv" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecLeft
, BYREF ANY _                         ' [in] LPDECIMAL pdecRight
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecMul _
   LIB "OLEAUT32.DLL" ALIAS "VarDecMul" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecLeft
, BYREF ANY _                         ' [in] LPDECIMAL pdecRight
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecSub _
   LIB "OLEAUT32.DLL" ALIAS "VarDecSub" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecLeft
, BYREF ANY _                         ' [in] LPDECIMAL pdecRight
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecAbs _
   LIB "OLEAUT32.DLL" ALIAS "VarDecAbs" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecIn
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecFix _
   LIB "OLEAUT32.DLL" ALIAS "VarDecFix" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecIn
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecInt _
   LIB "OLEAUT32.DLL" ALIAS "VarDecInt" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecIn
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecNeg _
   LIB "OLEAUT32.DLL" ALIAS "VarDecNeg" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecIn
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecRound _
   LIB "OLEAUT32.DLL" ALIAS "VarDecRound" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecIn
, BYVAL LONG _                        ' [in] int cDecimals
, BYREF ANY _                         ' [out] LPDECIMAL pdecResult
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecCmp _
   LIB "OLEAUT32.DLL" ALIAS "VarDecCmp" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecLeft
, BYREF ANY _                         ' [in] LPDECIMAL pdecRight
   ) AS LONG                           ' HRESULT

DECLARE FUNCTION VarDecCmpR8 _
   LIB "OLEAUT32.DLL" ALIAS "VarDecCmpR8" ( _
   BYREF ANY _                         ' [in] LPDECIMAL pdecIn
, BYVAL DOUBLE _                      ' [in] double dblRight
   ) AS LONG                           ' HRESULT


[Charles Pegge]

You could have a pair of functions to interconvert an Extended Precision number to a 20 byte hexadecimal coding which could be stored in a Sqlite string field.

Take the binary nybbles and add 65 so that the code looks something like this: ABDPFEGAHIKEFDNOMAEJ with characters between A and P.

To able to sort on such a field correctly, the characters must be stored in reverse order so that the most significant is the leftmost.

This encoding/decoding can be done efficiently in assembler. I can come up with the code if you think this is a practical way to go.

If Extended Precision proves to be insufficient then yes, its going to be customised arithmetic functions.

[MikeTrader]

Wow. This gives me a lot to work with. Thank you guys.

[MikeTrader]

Code Select Expand


   - Corrected exponential ranges to those specified in IEEE-754.
   - Correction of decimal exponent field size.
   - Made correction so that decimal significand is displayed even when
     its value is 0.
   - decBinVal array sizes corrected for small values (those near 1.0*2**-126
     for 32-bit or 1.0*2**-1022 for 64-bit).
   - Corrected ieee32.Convert2Dec() to ieee64.Convert2Dec() in the ieee64
     section.
   - Added support for zero and unnormalized numbers (those less than
     1.0*2**-126 for 32-bit or 1.0*2**-1022 for 64-bit).
   - Modified significand input field to hold the notationally largest range
     input value (with the most significant digits, exponent, and signs),
     -4.94065645841246544E-324 .
   - Use only 1 set of intBinVal and decBinVal arrays so that 32-bit results
     will be marked invalid when 64-bit values greater than 3.40282347E38
     are entered.
   - Joined arrays intBinVal and decBinVal into one array, BinVal, in order
     to ease the manipulations involved in rounding.

   - Added IEEE-754 round-to-nearest value rounding mode.

   - Added input echo field which displays how the host machine sees the
     input value (round-off, number of significant digits, max and min
     exponentials, etc.).
   - For both precisions, added an echo field which displays the input value
     accurate to the number of bits in that precision's significand.
   - Added action so that when an input value overflows a precision, set that
     precision's outputs to the precision's Infinity values instead of having
     the bits be replaced by "#"s.
   - For both precisions, added a status field indicating from high to low:
     overflow, normal, unnormalized, underflow, and normal for zero.
   - Run Convert2Bin() only once per precision (not for each substring).
   - Added the ability to round or not to round by providing 2 activation
     buttons labeled as such.
   - Reduced exponent input field to hold up to the notationally largest
     exponent value (with the most significant digits and sign), -324, unless
     someone wants to oddly decimalize (add fractional/decimal parts to) the
     exponent.
   - Replaced calls to round() with tests and corrections for rounding up to
     calls to floor() which always rounds down.

   1998/09/28 to 1998/09/30

   - Made BinVal external to Convert2Bin (and therefore local to function ieee)
     as this.BinVal as is required in IEEE-754hex64.html and IEEE-754hex32.html
     so that Unix 'diff' has some chance at comparing this file with
     IEEE-754hex64.html or IEEE-754hex32.html .
   - General clean-ups.
   - Removed the optional "Exponent:" input field due to its problem of causing
     the entire input to underflow to zero when its value is -324, the problems
     encountered when trying to correct this deficiency programatically, and
     its superfluousness due to the ability of entering scientific notation
     (4.94065645841246544E-324) into the "Significand:" field alone.
   - Renamed the "Significand:" input field to "Decimal Floating-Point:".

   1998/10/06 to 1998/10/07

   - Added removal of input leading and trailing blanks by adding RemoveBlanks
     routine.
   - Added converting all floating-point inputs into canonical form (scientific
     notation, +x.xxE+xxxx form) with constant exponential width and plus sign
     usage by adding the Canonical routine, in order to simplify the string
     number comparisons involved in the 1.7976931348623157E+308 overflow check.
   - Found and corrected bug in both binary outputs created by some unknown
     JavaScript scoping problem for the symbol 'sOut' by introduction of the
     Canonical routine (never in a released version).

   - Floating-point input is now hand parsed and then its magnitude is compared
     against +1.7976931348623157E+00308 in the OvfCheck routine.  If the
     magnitude is greater than this value (compared as a string), set all
     outputs to the appropriate Infinity values.  The JavaScript implementation
     of IEEE-754 64-bit precision clips such values at 1.7976931348623157E+308.

   - Greatly improved the efficiency of the Convert2Hex routine.

   1998/10/20

   - Allow power of 10 indicator in numStrClipOff to be "E" as well as "e" in
     case not all browsers use "e".
   - Reordered 'numerals' in OvfCheck so that their index (value) is the same
     as that which the numeral represents.

   1998/10/23

   - With hand parsed floating-point input, compare its magnitude against
     +2.4703282292062328E-00324 in the UndfCheck routine.  If the magnitude is
     less than this value (compared as a string), set 'this.StatCond' to
     "underflow".  The JavaScript implementation of IEEE-754 64-bit precision
     underflows such values to zero.
   - The above required all settings of 'this.StatCond' to "normal" to have to
     be removed, "normal" rather than "error" is now the default.
   - With hand parsed input, set the output sign bit from its minus sign
     character before the input is turned into a numeric.  This supports input
     of negative zeros, "-0", and those values which underflow to it.

   1998/10/28

   - The central testing section of OvfCheck and UndfCheck were joined together
     as the single routine A_gt_B.
   - The translation which moves an input's base 10 exponential (the sign and
     value to the right of the "E") to its left end as its sign and most
     significant digits is now performed by the MostSigOrder routine.
   - Due to tests now involving negative exponentials in A_gt_B (via UndfCheck),
     a bias of 50,000 is added to all exponentials, when moved by MostSigOrder,
     in order to simplify numeric compares performed as iterative comparisons of
     individual digits in strings.

   1998/10/29

   - Floating-point input is now hand parsed, manipulated, and placed into the
     this.BinVal array only once by introduction of the Dec2Bin routine.
     As a result, the Convert2Bin routine now used is quite similar to that of
     IEEE-754hex64.html and IEEE-754hex32.html .

   32-bit:
     3.4028234663852886E+38   down to        (1.9999998 * 2**127)
     1.1754942807573643E-38   normalized     (1.0 * 2**-126)
     1.4012984643248170E-45   unnormalized   (1.0 * 2**-149)
    (7.0064923216240862E-46)       "         (1.0 * 2**-150)

   64-bit:
     1.7976931348623157E+308  down to        (1.9999999999999996 * 2**1023)
     2.2250738585072016E-308  normalized     (1.0 * 2**-1022)
     4.9406564584124654E-324  unnormalized   (1.0 * 2**-1074)
    (2.4703282292062328E-324)      "         (1.0 * 2**-1075)


                           Float Values (b = bias)

         Sign Exponent (e)  Mantissa (m)            Value
         ----------------------------------------------------------
                              11..11                Quiet
          1      11..11          :                  -NaN
                              10..01
         ----------------------------------------------------------
          1      11..11       10..00            Indeterminate
         ----------------------------------------------------------
                              01..11              Signaling
          1      11..11          :                  -NaN
                              00..01
         ----------------------------------------------------------
          1      11..11       00..00              -Infinity
         ----------------------------------------------------------
                 11..10       11..11      Negative Normalized Real
          1         :            :            -1.m * 2**(e-b)
                 00..01       00..00
         ----------------------------------------------------------
                              11..11     Negative Denormalized Real
          1      00..00          :            -0.m * 2**(-b+1)
                              00..01
         ----------------------------------------------------------
          1      00..00       00..00                 -0
         ----------------------------------------------------------
          0      00..00       00..00                 +0
         ----------------------------------------------------------
                              00..01     Positive Denormalized Real
          0      00..00          :             0.m * 2**(-b+1)
                              11..11
         ----------------------------------------------------------
                 00..01       00..00      Positive Normalized Real
          0         :            :             1.m * 2**(e-b)
                 11..10       11..11
         ----------------------------------------------------------
          0      11..11       00..00              +Infinity
         ----------------------------------------------------------
                              00..01              Signaling
          0      11..11          :                  +NaN
                              01..11
         ----------------------------------------------------------
                              10..00                Quiet
          0      11..11          :                  +NaN
                              11..11
         ----------------------------------------------------------


  In the following, 's' stands for the sign bit, 'e' stands for the exponent,
  and 'm' stands for the fractional part of the mantissa.  The symbol 'x' stands
  for a "don't care" bit (either a 0 or 1).

For single-precision (32-bit) floating point numbers:

        Type                  s (1 bit)   e (8 bits)    m (23 bits)
        ----                  ---------   ----------    -----------
        signaling NaN         x           255 (max)     .0xxxxx---x
                                                        (with at least
                                                         one 1 bit)

        quiet NaN             x           255 (max)     .1xxxxx---x

        Indeterminate         1           255 (max)     .100000---0

        negative infinity     1           255 (max)     .000000---0

        positive infinity     0           255 (max)     .000000---0

        negative zero         1           0             .000000---0

        positive zero         0           0             .000000---0


For double-precision (64-bit) floating point numbers:

        Type                  s (1 bit)   e (11 bits)   m (52 bits)
        ----                  ---------   -----------   -----------
        signaling NaN         x           2047 (max)    .0xxxxx---x
                                                        (with at least
                                                         one 1 bit)

        quiet NaN             x           2047 (max)    .1xxxxx---x

        Indeterminate         1           2047 (max)    .100000---0

        negative infinity     1           2047 (max)    .000000---0

        positive infinity     0           2047 (max)    .000000---0

        negative zero         1           0             .000000---0

        positive zero         0           0             .000000---0



This code is only tested for 64bit floating point variables.
The conversion to hex is not exactly correct

[Charles Pegge]

Mike, does extended precision arithmetic meet your needs? This is the native format of the FPU, so using variables of this type ensures you can work without losses by intermediate variables of lesser precision.

One point to note is that PB requires the extended form of STR$ to convert them properly into text: STR$(n,18).

[MikeTrader]

Well I could use extended precision or even in fact a double precision as we all have for years, but at this point I want to explore alternatives to the ieee format.

Next I think I will do some speed tests with Jose's Decimal data type.

[Charles Pegge]

There is a Blob Data type in SqLite. You could use it to store the binary of any arbitrary number format. It all depends on what properties this data type has. Whether you can sort or select ranges if you need to.

If blob data is inadequate then we can pursue the text encoding idea.

[MikeTrader]

Yes the blob is great for this. The problem is with the queries. I don't know if they are going to want to use the values in a query. If they do I will have to write a UDF to convert each value first. It would be better to have this done as part of the data prep, but that means using one of the underlying datatypes QUAD or DOUBLE. The other thing to consider is some clever use of text that would sort etc correctly, but that is can of worms I would rather not open.

Apart from that, how these numbers are handled once returned is another matter. I still don't know what the range of data is. There is no sample data and great potential for large values.

Anyway, for now I am playing with the DECIMAL.

Code Select Expand


       

#COMPILE EXE
#DIM ALL
     
#INCLUDE "Decimal.inc"

GLOBAL hDbg AS LONG             
               
             


'¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤'
SUB time_stamp_count(tick AS QUAD) ' CPU Clock count    Charles E V Pegge

  '---------------------------'
  '                           ' approx because it is not a serialised instruction
  '                           ' it may execute before or after other instructions
  '                           ' in the pipeline.
  ! mov ebx,tick              ' var address where count is to be stored.
  ! db  &h0f,&h31             ' RDTSC read time-stamp counter into edx:eax hi lo.
  ! mov [ebx],eax             ' save low order 4 bytes.
  ! mov [ebx+4],edx           ' save high order 4 bytes.
  '---------------------------'

END SUB
           

'¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
FUNCTION PBMAIN
               
  LOCAL i, Rounding, nLoops, RetVal, LONGvar AS LONG 
  LOCAL cBeg, cEnd AS QUAD  ' for time stamp, measuring cpu clock cycles
  LOCAL d AS DOUBLE
  LOCAL s, sTemp AS STRING
  LOCAL Dec1,   Dec2,  Dec3 AS Decimal   
  LOCAL pDec1, pDec2, pDec3 AS Decimal PTR
               
  hDbg = FREEFILE '
  OPEN "DecimalDebug.txt" FOR OUTPUT LOCK WRITE AS hDbg ' PRINT #hDbg, "MetersToFt="+STR$(MetersToFt)
             
    pDec1 = VARPTR(Dec1)
    pDec2 = VARPTR(Dec2)
    pDec3 = VARPTR(Dec3)
 
    nLoops = 4 ' 10000000
                           
           
    d = 523.34#
   
    time_stamp_count(cBeg) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
      FOR i = 1 TO nLoops
        d = d * 3 
        d = d / 3
      NEXT
    time_stamp_count(cEnd) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
    s = s + "VAL =" + STR$(d) + ",   Clock Cycles="+STR$( (cEnd-cBeg)\nLoops ) + $CRLF + $CRLF
    '=======================
                 




    LONGvar = 52334         
    RetVal = VarDecFromI4(LONGvar, pDec1) ' store 52334 (Integer) in Dec1

    LONGvar = 100
    RetVal = VarDecFromI4(LONGvar,   pDec2) ' store 100 (Integer) in Dec3 

    RetVal = VarDecDiv(pDec1, pDec2, pDec3) ' Divide them to get the float "523.34"
    RetVal = VarR8FromDec(pDec3, d) 
    s = s + "523.34 =" + STR$(d) + $CRLF

    Dec1 = Dec3 ' xfer to Dec 1 
    LONGvar = 3
    RetVal = VarDecFromI4(LONGvar,   pDec2) ' prepare Dec2 

             
    time_stamp_count(cBeg) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
      FOR i = 1 TO nLoops
        RetVal = VarDecMul(pDec1, pDec2, pDec3) ' 523.34*3
        RetVal = VarDecDiv(pDec1, pDec2, pDec3) ' 523.34/3
      NEXT '
    time_stamp_count(cEnd) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
    RetVal = VarR8FromDec(pDec3, d) 
    s = s + "Result =" + STR$(d) + ",   Clock Cycles="+STR$( (cEnd-cBeg)\nLoops ) + $CRLF + $CRLF
    '=======================
               
       
'   BYREF ANY _                         ' [in] DECIMAL *pdecIn
' , BYVAL DWORD _                       ' [in] LCID lcid
' , BYVAL DWORD _                       ' [in] ULONG dwFlags
' , BYREF STRING _                      ' [out] BSTR *pbstrOut
'   ) AS LONG                           ' HRESULT   

    sTemp = "          " ' guess
    CALL VarBstrFromDec(pDec3, LEN(sTemp), BYVAL 0, sTemp )
    s = s + "VarBstrFromDec =" + sTemp


PRINT #hDbg, s
                   
MSGBOX s,64,"All Done"  : EXIT FUNCTION


  CLOSE hDbg

END FUNCTION

'¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤


I cant figure out why the values are all wrong

[José Roca]

 

I cant figure out why the values are all wrong

Lack of proper understanding about parameter passing conventions? What are you doing using VARPTR to pass the address of a DECIMAL structure if the parameter has been declared as BYREF ANY? This only will work if you use BYVAL; otherwise, you are passing the address of an address. Also, as VarBstrFromDec returns an UNICODE string, you have to use ACODE$ with your message box.

An small example:

Code Select Expand


FUNCTION PBMAIN
               
   LOCAL hr AS LONG
   LOCAL Dec1, Dec2, Dec3 AS DECIMAL
   LOCAL strVal AS STRING

   hr = VarDecFromI4(52334, Dec1) ' store 52334 (Integer) in Dec1
   hr = VarDecFromI4(100, Dec2)   ' store 100 (Integer) in Dec3 
   hr = VarDecDiv(Dec1, Dec2, Dec3) ' Divide them to get the float "523.34"
   VarBstrFromDec(Dec3, 0, 0, strVal)
   MSGBOX ACODE$(strVal)

END FUNCTION


And the lcid parameter is not the length of the string, but the locale identifier (0 for default locale). For a list of locale identifiers, see:

http://msdn2.microsoft.com/en-us/library/ms221219.aspx

[Charles Pegge]

Re: Extended precision numbers:

whole numbers are accurate to 18 digits but this deteriorates with fractional numbers down to around 15 digits, as I have discovered. It will be interesting to compare this with José's Decimal code.

The loss of resolution occurs when decimalising the numbers.

Re: Text encoding numbers:

Before reversing the order of hexadigits and encoding them as text, the sign bit must be inverted, when converting extended precision numbers. Exponent and mantissa are in the correct order of priority so there is no need to treat them separately.

Signed integers are a little more complicated: they must be converted from twos complement  format to an absolute value and an inverted sign bit, similar to the above.

But once this is done, the numbers will be ordered correctly, when the database treats them as text,
obeying all the rules governing text fields.

I have not spotted any other worms so far.

[MikeTrader]

What are you doing using VARPTR to pass the address of a DECIMAL structure if the parameter has been declared as BYREF ANY?

Ah... a casualty of developing on four fronts at the same time... one of which is all pointers. The last few weeks I see pointers when I close my eyes. Thx Jose.

Well timing results indicate DECIMAL operations to be about 5x slower than DOUBLE operations for mutiplication and division. Not as bad as the order of magnitude described in some threads I read in the last week.

Code Select Expand

     

#COMPILE EXE
#DIM ALL
     
#INCLUDE "Decimal.inc"

GLOBAL hDbg AS LONG             
               
             


'¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤'
SUB time_stamp_count(tick AS QUAD) ' CPU Clock count    Charles E V Pegge

  '---------------------------'
  '                           ' approx because it is not a serialised instruction
  '                           ' it may execute before or after other instructions
  '                           ' in the pipeline.
  ! mov ebx,tick              ' var address where count is to be stored.
  ! db  &h0f,&h31             ' RDTSC read time-stamp counter into edx:eax hi lo.
  ! mov [ebx],eax             ' save low order 4 bytes.
  ! mov [ebx+4],edx           ' save high order 4 bytes.
  '---------------------------'

END SUB
           

'¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤
FUNCTION PBMAIN
               
  LOCAL i, Rounding, nLoops, RetVal, LONGvar AS LONG 
  LOCAL cBeg, cEnd AS QUAD  ' for time stamp, measuring cpu clock cycles
  LOCAL d AS DOUBLE
  LOCAL s, sTemp AS STRING
  LOCAL Dec1,   Dec2,  Dec3 AS Decimal   

               
  hDbg = FREEFILE '
  OPEN "DecimalDebug.txt" FOR OUTPUT LOCK WRITE AS hDbg ' PRINT #hDbg, "MetersToFt="+STR$(MetersToFt)
             

    nLoops = 10000000
                               
    d = 523.34#
   
    time_stamp_count(cBeg) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
      FOR i = 1 TO nLoops
        d = d * 3 
        d = d / 3
      NEXT
    time_stamp_count(cEnd) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
    s = s + "VAL =" + STR$(d) + ",   Clock Cycles="+STR$( (cEnd-cBeg)\nLoops ) + $CRLF + $CRLF
    '=======================
                 




    LONGvar = 52334         
    RetVal = VarDecFromI4(LONGvar, Dec1) ' store 52334 (Integer) in Dec1

    LONGvar = 100
    RetVal = VarDecFromI4(LONGvar,   Dec2) ' store 100 (Integer) in Dec3 

    RetVal = VarDecDiv(Dec1, Dec2, Dec3) ' Divide them to get the float "523.34"

    Dec1 = Dec3 ' xfer to Dec 1 
    LONGvar = 3
    RetVal = VarDecFromI4(LONGvar,   Dec2) ' prepare Dec2 

             
    time_stamp_count(cBeg) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
      FOR i = 1 TO nLoops
        RetVal = VarDecMul(Dec1, Dec2, Dec3) ' 523.34*3
        RetVal = VarDecDiv(Dec1, Dec2, Dec3) ' 523.34/3
      NEXT '
    time_stamp_count(cEnd) ' measuring cpu clock cycles. The overhead just for making this call is about 25 clocks
    CALL VarBstrFromDec(Dec3, 0, 0, sTemp )
    s = s + "Result =" + ACODE$(sTemp) + ",   Clock Cycles="+STR$( (cEnd-cBeg)\nLoops ) + $CRLF + $CRLF
    '=======================
               


PRINT #hDbg, s
                   
MSGBOX s,64,"All Done"  : EXIT FUNCTION


  CLOSE hDbg

END FUNCTION

'¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤¤

Charles, how do you suggest I make the comparison

[Charles Pegge]

Well, your example shows 29 digit accuracy (26 decimal places) if I counted correctly. That should satisfy most cosmologists. But 15 digits should be good enough for most applications - even in Wall St

It's a direct trade-off between accuracy and performance, but if you can avoid handling fractions directly then 18 digits are possible with Extended Precision.