Плавающая точка в StrDec

Вам не нужно вводить новый счетчик. Вы анализируете temp_bcd снизу вверх, уменьшая ESI. Если ESI находится по адресу до начала temp_bcd, синтаксический анализ завершен. Небольшая проблема заключается в том, что вы уменьшаете ESI после загрузки AL. Таким образом, последний (правильный) AL подразумевает (неправильный) ESI за границей. Таким образом, ESI должно быть на 2 байта меньше, чтобы разорвать цикл.

Для шести чисел после запятой вы должны умножить MyNum на 1.000.000 и изменить ECX на 6 (в настоящее время 8).

INCLUDE masm32rt.inc

.data
     CaptionOut BYTE "Result: ", 0

     temp_bcd     dt     ?
     szdecimal    db     32 dup(0)

     MyNum REAL10 123.456789

.code
main PROC


    ; FPU to DecStr

    fld MyNum

;   pushd 100               ;use the stack as storage for this value
    pushd 1000000               ;use the stack as storage for this value
    fimul dword ptr[esp] ;converts 2 decimal places as 2 more integer digits
    fbstp temp_bcd          ;store the packed BCD integer value
    pop  eax                    ;clean the stack of the pushed 100

; ESI will be used for pointing to the packed BCD
; EDI will be used for pointing to the decimal string

    push esi
    push edi
    lea  esi,temp_bcd+9 ;point to the most significant byte
    lea  edi,szdecimal  ;point to the start of the decimal string buffer
    xor  eax,eax
    fwait                       ;to ascertain that the transfer of the
                                ;packed BCD is completed

    mov  al,[esi]           ;get the byte with the sign code
    dec  esi                    ;decrement to next most significant byte
    or    al,al             ;for checking the sign bit
    jns  @F                 ;jump if no sign bit
    mov  al,"-"             ;the value is negative
    stosb                       ;insert the negative sign

@@:

; The next 8 bytes (in this example) will contain the integer digits
; and the least significant byte will then contain the 2 decimal digits.
; No leading 0 will be included in the integer portion
; unless it would be the only integer digit.

;   mov  ecx,8              ;number of bytes to process for integer digits
    mov  ecx,6              ;number of bytes to process for integer digits

@@:

    mov  al,[esi]           ;get the next byte
    dec  esi                    ;adjust the pointer to the next one
    or    al,al             ;for checking if it is 0
    jnz  @F                 ;the starting integer digit is now in AL
    dec  ecx                    ;adjust the counter of integer bytes
    jnz  @B                 ;continue searching for the first integer digit

; If the loop terminates with ECX=0, the integer portion would be 0.
; A "0" character must be inserted before processing the decimal digits

    mov  al,"0"             ;the ASCII 0
    stosb                       ;insert it
    mov  al,[esi]           ;get the byte containing the decimal digits
    jmp  decimal_digits

@@:

; The first integer byte must be checked to determine
; if it contains 1 or 2 integer digits

    test al,0f0h            ;test if the H.O. nibble contains a digit
    jz    int_digit2        ;if not, process only the L.O. nibble

int_digit1:
    ror  ax,4               ;puts the H.O. nibble in the L.O. nibble position
                                ;and saves the L.O. nibble in AH
    add  al,30h             ;convert it to ASCII character
    stosb                       ;store this character
    shr  ax,12              ;restores the L.O. nibble in AL
                                ;and also resets the other bits of AX

int_digit2:
    add  al,30h             ;convert it to ASCII character
    stosb                       ;store this character
    mov  al,[esi]           ;get next byte
    dec  esi                    ;adjust the pointer to the next one
    dec  ecx                    ;adjust the counter of integer bytes
    jnz  int_digit1     ;continue processing the integer bytes

decimal_digits:
    mov  byte ptr [edi],"." ;insert the preferred decimal delimiter
    inc  edi

; Also, if the number of required decimal digits is not even, the code
; would have to be altered to insert the decimal delimiter at the
; proper location.

;   ror  ax,4               ;puts the H.O. nibble in the L.O. nibble position
                                ;and saves the L.O. nibble in AH
;   add  al,30h             ;convert it to ASCII character
;   stosb                       ;store this character
;   shr  ax,12              ;restores the L.O. nibble in AL
                                ;and also resets the other bits of AX
;   add  al,30h             ;convert it to ASCII character
;   stosw                       ;store this character + the terminating 0

@@:
    ror  ax,4            ;puts the H.O. nibble in the L.O. nibble position
                        ;and saves the L.O. nibble in AH
    add  al,30h          ;convert it to ASCII character
    stosb                ;store this character
    shr  ax,12           ;restores the L.O. nibble in AL
                        ;and also resets the other bits of AX
    add  al,30h          ;convert it to ASCII character
    stosb                ;store this character + the terminating 0

    mov  al,[esi]        ;get next byte
    dec  esi             ;adjust the pointer to the next one
    cmp esi, OFFSET temp_bcd - 1
    jae @B

    @@:
    mov BYTE PTR [edi], 0

    INVOKE MessageBoxA, 0, ADDR szdecimal, ADDR CaptionOut, 0

    INVOKE ExitProcess,0
main ENDP
END main

И, конечно же, я бы сделал это совсем по-другому: https://stackoverflow.com/a/30133492/3512216: -)