ruberon/source/AMD64.ob07

MODULE AMD64;

IMPORT IL, BIN, WR := WRITER, CHL := CHUNKLISTS, LISTS, PATHS, PROG, TARGETS,
       REG, UTILS, S := STRINGS, PE32, ELF, X86, ERRORS;


CONST

    rax = REG.R0;
    r10 = REG.R10;
    r11 = REG.R11;

    rcx = REG.R1;
    rdx = REG.R2;
    r8  = REG.R8;
    r9  = REG.R9;

    rsp = 4;
    rbp = 5;
    rsi = 6;
    rdi = 7;

    MAX_XMM = 5;

    je = 84H; jne = 85H; jl = 8CH; jge = 8DH; jle = 8EH; jg = 8FH; jb = 82H;

    sete = 94H; setne = 95H; setl = 9CH; setge = 9DH; setle = 9EH; setg = 9FH; setc = 92H; setnc = 93H;

    shl = IL.opLSL2; shr = IL.opLSR2; sar = IL.opASR2; ror = IL.opROR2;

    sCODE = BIN.PICCODE;
    sDATA = BIN.PICDATA;
    sBSS  = BIN.PICBSS;
    sIMP  = BIN.PICIMP;

    FPR_ERR = 41;


TYPE

    COMMAND = IL.COMMAND;

    Number = POINTER TO RECORD (LISTS.ITEM) value: INTEGER END;

    OPRR = PROCEDURE (reg1, reg2: INTEGER);


VAR

    R: REG.REGS;

    Numbers: LISTS.LIST;
    Numbers_Count: INTEGER;
    Numbers_Offs: INTEGER;

    prog: BIN.PROGRAM;

    tcount: INTEGER;

    dllret, sofinit: INTEGER;

    Win64RegPar: ARRAY 4 OF INTEGER;
    SystemVRegPar: ARRAY 6 OF INTEGER;

    Xmm: ARRAY 1000 OF INTEGER;

    fname: PATHS.PATH;


PROCEDURE OutByte (b: BYTE);
    BEGIN
        X86.OutByte(b)
    END OutByte;


PROCEDURE OutByte2 (a, b: BYTE);
    BEGIN
        X86.OutByte(a);
        X86.OutByte(b)
    END OutByte2;


PROCEDURE OutByte3 (a, b, c: BYTE);
    BEGIN
        X86.OutByte(a);
        X86.OutByte(b);
        X86.OutByte(c)
    END OutByte3;


PROCEDURE OutInt (n: INTEGER);
    BEGIN
        X86.OutByte(n MOD 256);
        X86.OutByte(UTILS.Byte(n, 1));
        X86.OutByte(UTILS.Byte(n, 2));
        X86.OutByte(UTILS.Byte(n, 3))
    END OutInt;


PROCEDURE short (n: INTEGER): INTEGER;
        RETURN 2 * ORD(X86.isByte(n))
    END short;


PROCEDURE long (n: INTEGER): INTEGER;
        RETURN 40H * ORD(~X86.isByte(n))
    END long;


PROCEDURE OutIntByte (n: INTEGER);
    BEGIN
        IF X86.isByte(n) THEN
            OutByte(n MOD 256)
        ELSE
            OutInt(n)
        END
    END OutIntByte;


PROCEDURE isLong (n: INTEGER): BOOLEAN;
        RETURN (n > UTILS.max32) OR (n < UTILS.min32)
    END isLong;


PROCEDURE NewNumber (value: INTEGER);
    VAR
        number: Number;

    BEGIN
        NEW(number);
        number.value := value;
        LISTS.push(Numbers, number);
        INC(Numbers_Count)
    END NewNumber;


PROCEDURE NewLabel (): INTEGER;
    BEGIN
        BIN.NewLabel(prog)
        RETURN IL.NewLabel()
    END NewLabel;


PROCEDURE Rex (reg1, reg2: INTEGER);
    BEGIN
        OutByte(48H + reg1 DIV 8 + 4 * (reg2 DIV 8))
    END Rex;


PROCEDURE lea (reg, offset, section: INTEGER);
    BEGIN
        Rex(0, reg);
        OutByte2(8DH, 05H + 8 * (reg MOD 8)); (* lea reg, [rip + offset] *)
        X86.Reloc(section, offset)
    END lea;


PROCEDURE oprr (op: BYTE; reg1, reg2: INTEGER); (* op reg1, reg2 *)
    BEGIN
        Rex(reg1, reg2);
        OutByte2(op, 0C0H + 8 * (reg2 MOD 8) + reg1 MOD 8)
    END oprr;


PROCEDURE oprr2 (op1, op2: BYTE; reg1, reg2: INTEGER); (* op reg1, reg2 *)
    BEGIN
        Rex(reg1, reg2);
        OutByte3(op1, op2, 0C0H + 8 * (reg2 MOD 8) + reg1 MOD 8)
    END oprr2;


PROCEDURE mov (reg1, reg2: INTEGER); (* mov reg1, reg2 *)
    BEGIN
        oprr(89H, reg1, reg2)
    END mov;


PROCEDURE xor (reg1, reg2: INTEGER); (* xor reg1, reg2 *)
    BEGIN
        oprr(31H, reg1, reg2)
    END xor;


PROCEDURE and (reg1, reg2: INTEGER); (* and reg1, reg2 *)
    BEGIN
        oprr(21H, reg1, reg2)
    END and;


PROCEDURE _or (reg1, reg2: INTEGER); (* or reg1, reg2 *)
    BEGIN
        oprr(09H, reg1, reg2)
    END _or;


PROCEDURE add (reg1, reg2: INTEGER); (* add reg1, reg2 *)
    BEGIN
        oprr(01H, reg1, reg2)
    END add;


PROCEDURE sub (reg1, reg2: INTEGER); (* sub reg1, reg2 *)
    BEGIN
        oprr(29H, reg1, reg2)
    END sub;


PROCEDURE xchg (reg1, reg2: INTEGER); (* xchg reg1, reg2 *)
    BEGIN
        IF rax IN {reg1, reg2} THEN
            Rex(reg1 + reg2, 0);
            OutByte(90H + (reg1 + reg2) MOD 8)
        ELSE
            oprr(87H, reg1, reg2)
        END
    END xchg;


PROCEDURE cmprr (reg1, reg2: INTEGER);  (* cmp reg1, reg2 *)
    BEGIN
        oprr(39H, reg1, reg2)
    END cmprr;


PROCEDURE pop (reg: INTEGER); (* pop reg *)
    BEGIN
        IF reg >= 8 THEN
            OutByte(41H)
        END;
        OutByte(58H + reg MOD 8)
    END pop;


PROCEDURE push (reg: INTEGER); (* push reg *)
    BEGIN
        IF reg >= 8 THEN
            OutByte(41H)
        END;
        OutByte(50H + reg MOD 8)
    END push;


PROCEDURE decr (reg: INTEGER);
    BEGIN
        Rex(reg, 0);
        OutByte2(0FFH, 0C8H + reg MOD 8) (* dec reg1 *)
    END decr;


PROCEDURE incr (reg: INTEGER);
    BEGIN
        Rex(reg, 0);
        OutByte2(0FFH, 0C0H + reg MOD 8) (* inc reg1 *)
    END incr;


PROCEDURE drop;
    BEGIN
        REG.Drop(R)
    END drop;


PROCEDURE GetAnyReg (): INTEGER;
        RETURN REG.GetAnyReg(R)
    END GetAnyReg;


PROCEDURE callimp (label: INTEGER);
    BEGIN
        OutByte2(0FFH, 15H);    (* call qword[rip + label + IMP] *)
        X86.Reloc(sIMP, label)
    END callimp;


PROCEDURE pushDA (offs: INTEGER);
    VAR
        reg: INTEGER;

    BEGIN
        reg := GetAnyReg();
        lea(reg, offs, sDATA);
        push(reg);
        drop
    END pushDA;


PROCEDURE CallRTL (proc: INTEGER);
    VAR
        label: INTEGER;

    BEGIN
        label := IL.codes.rtl[proc];
        IF label < 0 THEN
            callimp(-label)
        ELSE
            X86.call(label)
        END
    END CallRTL;


PROCEDURE UnOp (VAR reg: INTEGER);
    BEGIN
        REG.UnOp(R, reg)
    END UnOp;


PROCEDURE BinOp (VAR reg1, reg2: INTEGER);
    BEGIN
        REG.BinOp(R, reg1, reg2)
    END BinOp;


PROCEDURE PushAll (NumberOfParameters: INTEGER);
    BEGIN
        REG.PushAll(R);
        DEC(R.pushed, NumberOfParameters)
    END PushAll;


PROCEDURE movabs (reg, n: INTEGER);
    VAR
        i: INTEGER;

    BEGIN
        Rex(reg, 0);
        OutByte(0B8H + reg MOD 8); (* movabs reg, n *)
        FOR i := 0 TO 7 DO
            OutByte(UTILS.Byte(n, i))
        END
    END movabs;


PROCEDURE movrc (reg, n: INTEGER); (* mov reg, n *)
    BEGIN
        IF isLong(n) THEN
            movabs(reg, n)
        ELSIF n = 0 THEN
            xor(reg, reg)
        ELSE
            Rex(reg, 0);
            OutByte2(0C7H, 0C0H + reg MOD 8);
            OutInt(n)
        END
    END movrc;


PROCEDURE test (reg: INTEGER); (* test reg, reg *)
    BEGIN
        oprr(85H, reg, reg)
    END test;


PROCEDURE oprlongc (reg, n: INTEGER; oprr: OPRR);
    VAR
        reg2: INTEGER;

    BEGIN
        reg2 := GetAnyReg();
        ASSERT(reg2 # reg);
        movabs(reg2, n);
        oprr(reg, reg2);
        drop
    END oprlongc;


PROCEDURE oprc (op, reg, n: INTEGER; oprr: OPRR);
    BEGIN
        IF isLong(n) THEN
            oprlongc(reg, n, oprr)
        ELSE
            Rex(reg, 0);
            X86.oprc(op, reg, n)
        END
    END oprc;


PROCEDURE cmprc (reg, n: INTEGER); (* cmp reg, n *)
    BEGIN
        IF n = 0 THEN
            test(reg)
        ELSE
            oprc(0F8H, reg, n, cmprr)
        END
    END cmprc;


PROCEDURE addrc (reg, n: INTEGER); (* add reg, n *)
    BEGIN
        oprc(0C0H, reg, n, add)
    END addrc;


PROCEDURE subrc (reg, n: INTEGER); (* sub reg, n *)
    BEGIN
        oprc(0E8H, reg, n, sub)
    END subrc;


PROCEDURE andrc (reg, n: INTEGER); (* and reg, n *)
    BEGIN
        oprc(0E0H, reg, n, and)
    END andrc;


PROCEDURE orrc (reg, n: INTEGER); (* or reg, n *)
    BEGIN
        oprc(0C8H, reg, n, _or)
    END orrc;


PROCEDURE xorrc (reg, n: INTEGER); (* xor reg, n *)
    BEGIN
        oprc(0F0H, reg, n, xor)
    END xorrc;


PROCEDURE pushc (n: INTEGER);
    VAR
        reg2: INTEGER;

    BEGIN
        IF isLong(n) THEN
            reg2 := GetAnyReg();
            movabs(reg2, n);
            push(reg2);
            drop
        ELSE
            X86.pushc(n)
        END
    END pushc;


PROCEDURE not (reg: INTEGER); (* not reg *)
    BEGIN
        Rex(reg, 0);
        OutByte2(0F7H, 0D0H + reg MOD 8)
    END not;


PROCEDURE neg (reg: INTEGER); (* neg reg *)
    BEGIN
        Rex(reg, 0);
        OutByte2(0F7H, 0D8H + reg MOD 8)
    END neg;


PROCEDURE movzx (reg1, reg2, offs: INTEGER; word: BOOLEAN); (* movzx reg1, byte/word[reg2 + offs] *)
    BEGIN
        Rex(reg2, reg1);
        X86.movzx(reg1, reg2, offs, word)
    END movzx;


PROCEDURE movmr32 (reg1, offs, reg2: INTEGER); (* mov dword[reg1+offs], reg2_32 *)
    BEGIN
        X86._movrm(reg2, reg1, offs, 32, TRUE)
    END movmr32;


PROCEDURE movrm32 (reg1, reg2, offs: INTEGER); (* mov reg1_32, dword[reg2+offs] *)
    BEGIN
        X86._movrm(reg1, reg2, offs, 32, FALSE)
    END movrm32;


PROCEDURE movmr (reg1, offs, reg2: INTEGER); (* mov qword[reg1+offs], reg2 *)
    BEGIN
        X86._movrm(reg2, reg1, offs, 64, TRUE)
    END movmr;


PROCEDURE movrm (reg1, reg2, offs: INTEGER); (* mov reg1, qword[reg2+offs] *)
    BEGIN
        X86._movrm(reg1, reg2, offs, 64, FALSE)
    END movrm;


PROCEDURE comisd (xmm1, xmm2: INTEGER); (* comisd xmm1, xmm2 *)
    BEGIN
        OutByte(66H);
        IF (xmm1 >= 8) OR (xmm2 >= 8) THEN
            OutByte(40H + (xmm1 DIV 8) * 4 + xmm2 DIV 8)
        END;
        OutByte3(0FH, 2FH, 0C0H + (xmm1 MOD 8) * 8 + xmm2 MOD 8)
    END comisd;


PROCEDURE _movsdrm (xmm, reg, offs: INTEGER; mr: BOOLEAN);
    VAR
        b: BYTE;

    BEGIN
        OutByte(0F2H);
        IF (xmm >= 8) OR (reg >= 8) THEN
            OutByte(40H + (xmm DIV 8) * 4 + reg DIV 8)
        END;
        OutByte2(0FH, 10H + ORD(mr));
        IF (offs = 0) & (reg # rbp) THEN
            b := 0
        ELSE
            b := 40H + long(offs)
        END;
        OutByte(b + (xmm MOD 8) * 8 + reg MOD 8);
        IF reg = rsp THEN
            OutByte(24H)
        END;
        IF b # 0 THEN
            OutIntByte(offs)
        END
    END _movsdrm;


PROCEDURE movsdrm (xmm, reg, offs: INTEGER); (* movsd xmm, qword[reg+offs] *)
    BEGIN
        _movsdrm(xmm, reg, offs, FALSE)
    END movsdrm;


PROCEDURE movsdmr (reg, offs, xmm: INTEGER); (* movsd qword[reg+offs], xmm *)
    BEGIN
        _movsdrm(xmm, reg, offs, TRUE)
    END movsdmr;


PROCEDURE opxx (op, xmm1, xmm2: INTEGER);
    BEGIN
        OutByte(0F2H);
        IF (xmm1 >= 8) OR (xmm2 >= 8) THEN
            OutByte(40H + (xmm1 DIV 8) * 4 + xmm2 DIV 8)
        END;
        OutByte3(0FH, op, 0C0H + (xmm1 MOD 8) * 8 + xmm2 MOD 8)
    END opxx;


PROCEDURE jcc (cc, label: INTEGER); (* jcc label *)
    BEGIN
        X86.jcc(cc, label)
    END jcc;


PROCEDURE shiftrc (op, reg, n: INTEGER);
    BEGIN
        Rex(reg, 0);
        IF n = 1 THEN
            OutByte(0D1H)
        ELSE
            OutByte(0C1H)
        END;
        X86.shift(op, reg MOD 8);
        IF n # 1 THEN
            OutByte(n)
        END
    END shiftrc;


PROCEDURE GetRegA;
    BEGIN
        ASSERT(REG.GetReg(R, rax))
    END GetRegA;


PROCEDURE Win64Passing (params: INTEGER);
    VAR
        n, i: INTEGER;

    BEGIN
        n := params MOD 32;
        params := params DIV 32;
        FOR i := 0 TO n - 1 DO
            IF i IN BITS(params) THEN
                movsdrm(i, rsp, i * 8)
            ELSE
                movrm(Win64RegPar[i], rsp, i * 8)
            END
        END
    END Win64Passing;


PROCEDURE SysVPassing (params: INTEGER);
    VAR
        n, i, s, p, ofs: INTEGER;
        i_count, f_count: INTEGER;
        reg: BOOLEAN;

    BEGIN
        ASSERT(r11 IN R.regs);
        n := params MOD 32;
        params := params DIV 32;
        s := 0;

        i_count := 0;
        f_count := 0;
        FOR i := 0 TO n - 1 DO
            IF i IN BITS(params) THEN
                INC(f_count)
            ELSE
                INC(i_count)
            END
        END;

        s := MAX(0, f_count - 8) + MAX(0, i_count - 6);
        p := 0;

        subrc(rsp, s * 8);

        i_count := 0;
        f_count := 0;
        FOR i := 0 TO n - 1 DO
            ofs := (i + s) * 8;
            IF i IN BITS(params) THEN
                reg := f_count <= 7;
                IF reg THEN
                    movsdrm(f_count, rsp, ofs);
                    INC(f_count)
                END
            ELSE
                reg := i_count <= 5;
                IF reg THEN
                    movrm(SystemVRegPar[i_count], rsp, ofs);
                    INC(i_count)
                END
            END;

            IF ~reg THEN
                movrm(r11, rsp, ofs);
                movmr(rsp, p, r11);
                INC(p, 8)
            END
        END
    END SysVPassing;


PROCEDURE fcmp (op: INTEGER; xmm: INTEGER);
    VAR
        cc, reg: INTEGER;

    BEGIN
        reg := GetAnyReg();
        xor(reg, reg);
        CASE op OF
        |IL.opEQF:
            comisd(xmm - 1, xmm);
            cc := sete

        |IL.opNEF:
            comisd(xmm - 1, xmm);
            cc := setne

        |IL.opLTF:
            comisd(xmm - 1, xmm);
            cc := setc

        |IL.opGTF:
            comisd(xmm, xmm - 1);
            cc := setc

        |IL.opLEF:
            comisd(xmm, xmm - 1);
            cc := setnc

        |IL.opGEF:
            comisd(xmm - 1, xmm);
            cc := setnc
        END;
        OutByte2(7AH, 3 + reg DIV 8); (* jp L *)
        X86.setcc(cc, reg)
        (* L: *)
    END fcmp;


PROCEDURE translate (commands: LISTS.LIST; stroffs: INTEGER);
    VAR
        cmd, next: COMMAND;

        opcode, param1, param2, param3, a, b, c, n, label, L, i, cc: INTEGER;

        reg1, reg2, reg3, xmm: INTEGER;

        float: REAL;

    BEGIN
        xmm := -1;
        cmd := commands.first(COMMAND);
        WHILE cmd # NIL DO

            param1 := cmd.param1;
            param2 := cmd.param2;

            opcode := cmd.opcode;

            CASE opcode OF

            |IL.opJMP:
                X86.jmp(param1)

            |IL.opCALL, IL.opWIN64CALL, IL.opSYSVCALL:
                CASE opcode OF
                |IL.opCALL:
                |IL.opWIN64CALL: Win64Passing(param2)
                |IL.opSYSVCALL:  SysVPassing(param2)
                END;
                X86.call(param1)

            |IL.opCALLP, IL.opWIN64CALLP, IL.opSYSVCALLP:
                UnOp(reg1);
                IF reg1 # rax THEN
                    mov(rax, reg1)
                END;
                drop;
                CASE opcode OF
                |IL.opCALLP:
                |IL.opWIN64CALLP: Win64Passing(param2)
                |IL.opSYSVCALLP:  SysVPassing(param2)
                END;
                OutByte2(0FFH, 0D0H); (* call rax *)
                ASSERT(R.top = -1)

            |IL.opCALLI, IL.opWIN64CALLI, IL.opSYSVCALLI:
                CASE opcode OF
                |IL.opCALLI:
                |IL.opWIN64CALLI: Win64Passing(param2)
                |IL.opSYSVCALLI:  SysVPassing(param2)
                END;
                callimp(param1)

            |IL.opLABEL:
                X86.SetLabel(param1)

            |IL.opERR:
                CallRTL(IL._error)

            |IL.opONERR:
                pushc(param2);
                X86.jmp(param1)

            |IL.opPUSHC:
                pushc(param2)

            |IL.opPRECALL:
                PushAll(0);
                IF (param2 # 0) & (xmm >= 0) THEN
                    subrc(rsp, 8)
                END;
                INC(Xmm[0]);
                Xmm[Xmm[0]] := xmm + 1;
                WHILE xmm >= 0 DO
                    subrc(rsp, 8);
                    movsdmr(rsp, 0, xmm);
                    DEC(xmm)
                END;
                ASSERT(xmm = -1)

            |IL.opWIN64ALIGN16:
                ASSERT(rax IN R.regs);
                mov(rax, rsp);
                andrc(rsp, -16);
                push(rax);
                subrc(rsp, (MAX(param2 - 4, 0) MOD 2 + MAX(4 - param2, 0) + 1) * 8)

            |IL.opSYSVALIGN16:
                ASSERT(rax IN R.regs);
                mov(rax, rsp);
                andrc(rsp, -16);
                push(rax);
                IF ~ODD(param2) THEN
                    push(rax)
                END

            |IL.opRESF, IL.opRES:
                ASSERT(R.top = -1);
                ASSERT(xmm = -1);
                n := Xmm[Xmm[0]]; DEC(Xmm[0]);

                IF opcode = IL.opRESF THEN
                    INC(xmm);
                    IF n > 0 THEN
                        movsdmr(rsp, n * 8, 0);
                        DEC(xmm);
                        INC(n)
                    END;

                    IF xmm + n > MAX_XMM THEN
                        ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
                    END
                ELSE
                    GetRegA
                END;

                WHILE n > 0 DO
                    INC(xmm);
                    movsdrm(xmm, rsp, 0);
                    addrc(rsp, 8);
                    DEC(n)
                END

            |IL.opENTER:
                ASSERT(R.top = -1);

                X86.SetLabel(param1);

                param3 := cmd.param3;

                IF param3 > 0 THEN
                    push(rbp);
                    mov(rbp, rsp);

                    n := param3 MOD 32;
                    param3 := param3 DIV 32;

                    FOR i := 0 TO n - 1 DO
                        IF i IN BITS(param3) THEN
                            movsdmr(rbp, i * 8 + 16, i)
                        ELSE
                            movmr(rbp, i * 8 + 16, Win64RegPar[i])
                        END
                    END
                ELSIF param3 < 0 THEN
                    param3 := -param3;
                    n := (param3 MOD 32) * 8;
                    param3 := param3 DIV 32;
                    pop(r11);
                    subrc(rsp, n);
                    push(r11);
                    push(rbp);
                    mov(rbp, rsp);

                    a := 0;
                    b := 0;
                    c := 0;

                    INC(n, 16);

                    FOR i := 16 TO n - 8 BY 8 DO
                        IF ODD(param3) THEN
                            IF b <= 7 THEN
                                movsdmr(rbp, i, b);
                                INC(b)
                            ELSE
                                movrm(r11, rbp, n + c);
                                movmr(rbp, i, r11);
                                INC(c, 8)
                            END
                        ELSE
                            IF a <= 5 THEN
                                movmr(rbp, i, SystemVRegPar[a]);
                                INC(a)
                            ELSE
                                movrm(r11, rbp, n + c);
                                movmr(rbp, i, r11);
                                INC(c, 8)
                            END
                        END;
                        param3 := param3 DIV 2
                    END
                ELSE
                    push(rbp);
                    mov(rbp, rsp)
                END;

                n := param2;
                IF n > 4 THEN
                    movrc(rcx, n);
                                        (* L: *)
                    pushc(0);
                    OutByte2(0E2H, 0FCH) (* loop L *)
                ELSE
                    WHILE n > 0 DO
                        pushc(0);
                        DEC(n)
                    END
                END

            |IL.opLEAVE, IL.opLEAVER, IL.opLEAVEF:
                IF opcode = IL.opLEAVER THEN
                    UnOp(reg1);
                    IF reg1 # rax THEN
                        mov(rax, reg1)
                    END;
                    drop
                END;

                ASSERT(R.top = -1);

                IF opcode = IL.opLEAVEF THEN
                    DEC(xmm)
                END;

                ASSERT(xmm = -1);

                IF param1 > 0 THEN
                    mov(rsp, rbp)
                END;

                pop(rbp);
                IF param2 > 0 THEN
                    OutByte3(0C2H, (param2 * 8) MOD 256, (param2 * 8) DIV 256) (* ret param2*8 *)
                ELSE
                    X86.ret
                END

            |IL.opSAVES:
                UnOp(reg1);
                REG.PushAll_1(R);
                pushDA(stroffs + param2);
                push(reg1);
                drop;
                pushc(param1);
                CallRTL(IL._move)

            |IL.opSADR:
                lea(GetAnyReg(), stroffs + param2, sDATA)

            |IL.opLOAD8:
                UnOp(reg1);
                movzx(reg1, reg1, 0, FALSE)

            |IL.opLOAD16:
                UnOp(reg1);
                movzx(reg1, reg1, 0, TRUE)

            |IL.opLOAD32:
                UnOp(reg1);
                movrm32(reg1, reg1, 0);
                shiftrc(shl, reg1, 32);
                shiftrc(shr, reg1, 32)

            |IL.opLOAD64:
                UnOp(reg1);
                movrm(reg1, reg1, 0)

            |IL.opLLOAD64:
                reg1 := GetAnyReg();
                movrm(reg1, rbp, param2 * 8)

            |IL.opLLOAD8,
            IL.opLLOAD16:
                reg1 := GetAnyReg();
                movzx(reg1, rbp, param2 * 8, opcode = IL.opLLOAD16)

            |IL.opLLOAD32:
                reg1 := GetAnyReg();
                movrm32(reg1, rbp, param2 * 8);
                shiftrc(shl, reg1, 32);
                shiftrc(shr, reg1, 32)

            |IL.opGLOAD64:
                reg1 := GetAnyReg();
                Rex(0, reg1);  (* mov reg1, qword[rip + param2 + BSS] *)
                OutByte2(8BH, 05H + 8 * (reg1 MOD 8));
                X86.Reloc(sBSS, param2)

            |IL.opGLOAD8, IL.opGLOAD16:
                reg1 := GetAnyReg();
                Rex(0, reg1);  (* movzx reg1, byte/word[rip + param2 + BSS] *)
                OutByte3(0FH, 0B6H + ORD(opcode = IL.opGLOAD16), 05H + 8 * (reg1 MOD 8));
                X86.Reloc(sBSS, param2)

            |IL.opGLOAD32:
                reg1 := GetAnyReg();
                lea(reg1, param2, sBSS);
                movrm32(reg1, reg1, 0);
                shiftrc(shl, reg1, 32);
                shiftrc(shr, reg1, 32)

            |IL.opVLOAD64:
                reg1 := GetAnyReg();
                movrm(reg1, rbp, param2 * 8);
                movrm(reg1, reg1, 0)

            |IL.opVLOAD8,
            IL.opVLOAD16:
                reg1 := GetAnyReg();
                movrm(reg1, rbp, param2 * 8);
                movzx(reg1, reg1, 0, opcode = IL.opVLOAD16)

            |IL.opVLOAD32:
                reg1 := GetAnyReg();
                reg2 := GetAnyReg();
                movrm(reg2, rbp, param2 * 8);
                movrm32(reg1, reg2, 0);
                shiftrc(shl, reg1, 32);
                shiftrc(shr, reg1, 32);
                drop

            |IL.opLADR:
                n := param2 * 8;
                next := cmd.next(COMMAND);
                IF (next.opcode = IL.opSAVEF) OR (next.opcode = IL.opSAVEFI) THEN
                    ASSERT(xmm >= 0);
                    movsdmr(rbp, n, xmm);
                    DEC(xmm);
                    cmd := next
                ELSIF next.opcode = IL.opLOADF THEN
                    INC(xmm);
                    IF xmm > MAX_XMM THEN
                        ERRORS.ErrorMsg(fname, next.param1, next.param2, FPR_ERR)
                    END;
                    movsdrm(xmm, rbp, n);
                    cmd := next
                ELSE
                    IF (next.opcode = IL.opADDC) & ~isLong(n + next.param2) THEN
                        INC(n, next.param2);
                        cmd := next
                    END;
                    reg1 := GetAnyReg();
                    Rex(0, reg1);
                    OutByte2(8DH, 45H + long(n) + (reg1 MOD 8) * 8); (* lea reg1, qword[rbp+n] *)
                    OutIntByte(n)
                END

            |IL.opGADR:
                next := cmd.next(COMMAND);
                IF (next.opcode = IL.opADDC) & ~isLong(param2 + next.param2) THEN
                    INC(param2, next.param2);
                    cmd := next
                END;
                lea(GetAnyReg(), param2, sBSS)

            |IL.opVADR:
                movrm(GetAnyReg(), rbp, param2 * 8)

            |IL.opSAVE8C:
                UnOp(reg1);
                IF reg1 >= 8 THEN
                    OutByte(41H)
                END;
                OutByte3(0C6H, reg1 MOD 8, param2); (* mov byte[reg1], param2 *)
                drop

            |IL.opSAVE16C:
                UnOp(reg1);
                OutByte(66H);
                IF reg1 >= 8 THEN
                    OutByte(41H)
                END;
                OutByte2(0C7H, reg1 MOD 8);
                OutByte2(param2 MOD 256, param2 DIV 256); (* mov word[reg1], param2 *)
                drop

            |IL.opSAVEC:
                UnOp(reg1);
                IF isLong(param2) THEN
                    reg2 := GetAnyReg();
                    movrc(reg2, param2);
                    movmr(reg1, 0, reg2);
                    drop
                ELSE
                    Rex(reg1, 0);
                    OutByte2(0C7H, reg1 MOD 8); (* mov qword[reg1], param2 *)
                    OutInt(param2)
                END;
                drop

            |IL.opRSET:
                PushAll(2);
                CallRTL(IL._set);
                GetRegA

            |IL.opRSETR:
                PushAll(1);
                pushc(param2);
                CallRTL(IL._set);
                GetRegA

            |IL.opRSETL:
                UnOp(reg1);
                REG.PushAll_1(R);
                pushc(param2);
                push(reg1);
                drop;
                CallRTL(IL._set);
                GetRegA

            |IL.opRSET1:
                PushAll(1);
                CallRTL(IL._set1);
                GetRegA

            |IL.opINCL, IL.opEXCL:
                BinOp(reg1, reg2);
                cmprc(reg1, 64);
                OutByte2(73H, 04H); (* jnb L *)
                Rex(reg2, reg1);
                OutByte3(0FH, 0ABH + 8 * ORD(opcode = IL.opEXCL), 8 * (reg1 MOD 8) + reg2 MOD 8); (* bts/btr qword[reg2], reg1 *)
                (* L: *)
                drop;
                drop

            |IL.opINCLC, IL.opEXCLC:
                UnOp(reg1);
                Rex(reg1, 0);
                OutByte2(0FH, 0BAH); (* bts/btr qword[reg1], param2 *)
                OutByte2(28H + 8 * ORD(opcode = IL.opEXCLC) + reg1 MOD 8, param2);
                drop

            |IL.opEQS .. IL.opGES:
                PushAll(4);
                pushc(opcode - IL.opEQS);
                CallRTL(IL._strcmp);
                GetRegA

            |IL.opEQSW .. IL.opGESW:
                PushAll(4);
                pushc(opcode - IL.opEQSW);
                CallRTL(IL._strcmpw);
                GetRegA

            |IL.opCONST:
                movrc(GetAnyReg(), param2)

            |IL.opEQ..IL.opGE,
            IL.opEQC..IL.opGEC:

                IF (IL.opEQ <= opcode) & (opcode <= IL.opGE) THEN
                    BinOp(reg1, reg2);
                    cmprr(reg1, reg2);
                    drop
                ELSE
                    UnOp(reg1);
                    cmprc(reg1, param2)
                END;

                drop;
                cc := X86.cond(opcode);

                next := cmd.next(COMMAND);
                IF next.opcode = IL.opJNZ THEN
                    jcc(cc, next.param1);
                    cmd := next
                ELSIF next.opcode = IL.opJZ THEN
                    jcc(X86.inv0(cc), next.param1);
                    cmd := next
                ELSE
                    reg1 := GetAnyReg();
                    X86.setcc(cc + 16, reg1);
                    andrc(reg1, 1)
                END

            |IL.opCODE:
                OutByte(param2)

            |IL.opPUSHIP:
                reg1 := GetAnyReg();
                lea(reg1, param2, sIMP);
                movrm(reg1, reg1, 0)

            |IL.opPARAM:
                IF param2 = 1 THEN
                    UnOp(reg1);
                    push(reg1);
                    drop
                ELSE
                    ASSERT(R.top + 1 <= param2);
                    PushAll(param2)
                END

            |IL.opJNZ1:
                UnOp(reg1);
                test(reg1);
                jcc(jne, param1)

            |IL.opJG:
                UnOp(reg1);
                test(reg1);
                jcc(jg, param1)

            |IL.opJNZ:
                UnOp(reg1);
                test(reg1);
                jcc(jne, param1);
                drop

            |IL.opJZ:
                UnOp(reg1);
                test(reg1);
                jcc(je, param1);
                drop

            |IL.opIN, IL.opINR:
                IF opcode = IL.opINR THEN
                    reg2 := GetAnyReg();
                    movrc(reg2, param2)
                END;
                label := NewLabel();
                L := NewLabel();
                BinOp(reg1, reg2);
                cmprc(reg1, 64);
                jcc(jb, L);
                xor(reg1, reg1);
                X86.jmp(label);
                X86.SetLabel(L);
                Rex(reg2, reg1);
                OutByte3(0FH, 0A3H, 0C0H + 8 * (reg1 MOD 8) + reg2 MOD 8); (* bt reg2, reg1 *)
                X86.setcc(setc, reg1);
                andrc(reg1, 1);
                X86.SetLabel(label);
                drop

            |IL.opINL:
                UnOp(reg1);
                Rex(reg1, 0);
                OutByte2(0FH, 0BAH); (* bt reg1, param2 *)
                OutByte2(0E0H + reg1 MOD 8, param2);
                X86.setcc(setc, reg1);
                andrc(reg1, 1)

            |IL.opNOT:
                UnOp(reg1);
                test(reg1);
                X86.setcc(sete, reg1);
                andrc(reg1, 1)

            |IL.opORD:
                UnOp(reg1);
                test(reg1);
                X86.setcc(setne, reg1);
                andrc(reg1, 1)

            |IL.opABS:
                UnOp(reg1);
                test(reg1);
                OutByte2(7DH, 03H); (* jge L *)
                neg(reg1)
                (* L: *)

            |IL.opEQB, IL.opNEB:
                BinOp(reg1, reg2);
                drop;
                test(reg1);
                label := NewLabel();
                jcc(je, label);
                movrc(reg1, 1);
                X86.SetLabel(label);
                test(reg2);
                label := NewLabel();
                jcc(je, label);
                movrc(reg2, 1);
                X86.SetLabel(label);
                cmprr(reg1, reg2);
                IF opcode = IL.opEQB THEN
                    X86.setcc(sete, reg1)
                ELSE
                    X86.setcc(setne, reg1)
                END;
                andrc(reg1, 1)

            |IL.opMULSC:
                UnOp(reg1);
                andrc(reg1, param2)

            |IL.opDIVSC:
                UnOp(reg1);
                xorrc(reg1, param2)

            |IL.opADDSC:
                UnOp(reg1);
                orrc(reg1, param2)

            |IL.opSUBSL:
                UnOp(reg1);
                not(reg1);
                andrc(reg1, param2)

            |IL.opSUBSR:
                UnOp(reg1);
                andrc(reg1, ORD(-BITS(param2)))

            |IL.opMULS:
                BinOp(reg1, reg2);
                and(reg1, reg2);
                drop

            |IL.opDIVS:
                BinOp(reg1, reg2);
                xor(reg1, reg2);
                drop

            |IL.opUMINS:
                UnOp(reg1);
                not(reg1)

            |IL.opCOPY:
                IF (0 < param2) & (param2 <= 256) THEN
                    BinOp(reg1, reg2);
                    reg3 := GetAnyReg();
                    FOR n := 0 TO param2 - param2 MOD 8 - 1 BY 8 DO
                        movrm(reg3, reg1, n);
                        movmr(reg2, n, reg3)
                    END;
                    n := param2 - param2 MOD 8;
                    IF param2 MOD 8 >= 4 THEN
                        movrm32(reg3, reg1, n);
                        movmr32(reg2, n, reg3);
                        INC(n, 4);
                        DEC(param2, 4)
                    END;
                    IF param2 MOD 8 >= 2 THEN
                        X86.movrm16(reg3, reg1, n);
                        X86.movmr16(reg2, n, reg3);
                        INC(n, 2);
                        DEC(param2, 2)
                    END;
                    IF param2 MOD 8 = 1 THEN
                        X86.movrm8(reg3, reg1, n);
                        X86.movmr8(reg2, n, reg3);
                    END;
                    drop;
                    drop;
                    drop
                ELSE
                    PushAll(2);
                    pushc(param2);
                    CallRTL(IL._move)
                END

            |IL.opMOVE:
                PushAll(3);
                CallRTL(IL._move)

            |IL.opCOPYA:
                PushAll(4);
                pushc(param2);
                CallRTL(IL._arrcpy);
                GetRegA

            |IL.opCOPYS:
                PushAll(4);
                pushc(param2);
                CallRTL(IL._strcpy)

            |IL.opROT:
                PushAll(0);
                push(rsp);
                pushc(param2);
                CallRTL(IL._rot)

            |IL.opNEW:
                PushAll(1);
                n := param2 + 16;
                ASSERT(UTILS.Align(n, 16));
                pushc(n);
                pushc(param1);
                CallRTL(IL._new)

            |IL.opDISP:
                PushAll(1);
                CallRTL(IL._dispose)

            |IL.opPUSHT:
                UnOp(reg1);
                movrm(GetAnyReg(), reg1, -8)

            |IL.opISREC:
                PushAll(2);
                pushc(param2 * tcount);
                CallRTL(IL._isrec);
                GetRegA

            |IL.opIS:
                PushAll(1);
                pushc(param2 * tcount);
                CallRTL(IL._is);
                GetRegA

            |IL.opTYPEGR:
                PushAll(1);
                pushc(param2 * tcount);
                CallRTL(IL._guardrec);
                GetRegA

            |IL.opTYPEGP:
                UnOp(reg1);
                PushAll(0);
                push(reg1);
                pushc(param2 * tcount);
                CallRTL(IL._guard);
                GetRegA

            |IL.opTYPEGD:
                UnOp(reg1);
                PushAll(0);
                X86.pushm(reg1, -8);
                pushc(param2 * tcount);
                CallRTL(IL._guardrec);
                GetRegA

            |IL.opCASET:
                push(rcx);
                push(rcx);
                pushc(param2 * tcount);
                CallRTL(IL._guardrec);
                pop(rcx);
                test(rax);
                jcc(jne, param1)

            |IL.opSAVEP:
                UnOp(reg1);
                reg2 := GetAnyReg();
                lea(reg2, param2, sCODE);
                movmr(reg1, 0, reg2);
                drop;
                drop

            |IL.opPUSHP:
                lea(GetAnyReg(), param2, sCODE)

            |IL.opINC, IL.opDEC:
                BinOp(reg1, reg2);
                (* add/sub qword[reg2], reg1 *)
                Rex(reg2, reg1);
                OutByte2(01H + 28H * ORD(opcode = IL.opDEC), reg2 MOD 8 + (reg1 MOD 8) * 8);
                drop;
                drop

            |IL.opINCC:
                UnOp(reg1);
                IF isLong(param2) THEN
                    reg2 := GetAnyReg();
                    movrc(reg2, param2);
                    (* add qword[reg1], reg2 *)
                    Rex(reg1, reg2);
                    OutByte2(01H, reg1 MOD 8 + (reg2 MOD 8) * 8);
                    drop
                ELSIF ABS(param2) = 1 THEN
                    Rex(reg1, 0);
                    OutByte2(0FFH, reg1 MOD 8 + 8 * ORD(param2 = -1)) (* inc/dec qword[reg1] *)
                ELSE
                    (* add qword[reg1], param2 *)
                    Rex(reg1, 0);
                    OutByte2(81H + short(param2), reg1 MOD 8);
                    OutIntByte(param2)
                END;
                drop

            |IL.opDROP:
                UnOp(reg1);
                drop

            |IL.opSAVE, IL.opSAVE64:
                BinOp(reg2, reg1);
                movmr(reg1, 0, reg2);
                drop;
                drop

            |IL.opSAVE8:
                BinOp(reg2, reg1);
                X86.movmr8(reg1, 0, reg2);
                drop;
                drop

            |IL.opSAVE16:
                BinOp(reg2, reg1);
                X86.movmr16(reg1, 0, reg2);
                drop;
                drop

            |IL.opSAVE32:
                BinOp(reg2, reg1);
                movmr32(reg1, 0, reg2);
                drop;
                drop

            |IL.opMAX, IL.opMIN:
                BinOp(reg1, reg2);
                cmprr(reg1, reg2);
                OutByte2(7DH + ORD(opcode = IL.opMIN), 3); (* jge/jle L *)
                mov(reg1, reg2);
                (* L: *)
                drop

            |IL.opMAXC, IL.opMINC:
                UnOp(reg1);
                cmprc(reg1, param2);
                label := NewLabel();
                IF opcode = IL.opMINC THEN
                    cc := jle
                ELSE
                    cc := jge
                END;
                jcc(cc, label);
                movrc(reg1, param2);
                X86.SetLabel(label)

            |IL.opSBOOL:
                BinOp(reg2, reg1);
                test(reg2);
                IF reg1 >= 8 THEN
                    OutByte(41H)
                END;
                OutByte3(0FH, 95H, reg1 MOD 8); (* setne byte[reg1] *)
                drop;
                drop

            |IL.opSBOOLC:
                UnOp(reg1);
                IF reg1 >= 8 THEN
                    OutByte(41H)
                END;
                OutByte3(0C6H, reg1 MOD 8, ORD(param2 # 0)); (* mov byte[reg1], 0/1 *)
                drop

            |IL.opUMINUS:
                UnOp(reg1);
                neg(reg1)

            |IL.opADD:
                BinOp(reg1, reg2);
                add(reg1, reg2);
                drop

            |IL.opSUB:
                BinOp(reg1, reg2);
                sub(reg1, reg2);
                drop

            |IL.opSUBR, IL.opSUBL:
                UnOp(reg1);
                IF param2 = 1 THEN
                    decr(reg1)
                ELSIF param2 = -1 THEN
                    incr(reg1)
                ELSIF param2 # 0 THEN
                    subrc(reg1, param2)
                END;
                IF opcode = IL.opSUBL THEN
                    neg(reg1)
                END

            |IL.opADDC:
                IF (param2 # 0) & ~isLong(param2) THEN
                    UnOp(reg1);
                    next := cmd.next(COMMAND);
                    CASE next.opcode OF
                    |IL.opLOAD64:
                        movrm(reg1, reg1, param2);
                        cmd := next
                    |IL.opLOAD32:
                        movrm32(reg1, reg1, param2);
                        shiftrc(shl, reg1, 32);
                        shiftrc(shr, reg1, 32);
                        cmd := next
                    |IL.opLOAD16:
                        movzx(reg1, reg1, param2, TRUE);
                        cmd := next
                    |IL.opLOAD8:
                        movzx(reg1, reg1, param2, FALSE);
                        cmd := next
                    |IL.opLOAD64_PARAM:
                        X86.pushm(reg1, param2);
                        drop;
                        cmd := next
                    ELSE
                        IF param2 = 1 THEN
                            incr(reg1)
                        ELSIF param2 = -1 THEN
                            decr(reg1)
                        ELSE
                            addrc(reg1, param2)
                        END
                    END
                ELSIF isLong(param2) THEN
                    UnOp(reg1);
                    addrc(reg1, param2)
                END

            |IL.opDIV:
                PushAll(2);
                CallRTL(IL._divmod);
                GetRegA

            |IL.opDIVR:
                n := UTILS.Log2(param2);
                IF n > 0 THEN
                    UnOp(reg1);
                    shiftrc(sar, reg1, n)
                ELSIF n < 0 THEN
                    PushAll(1);
                    pushc(param2);
                    CallRTL(IL._divmod);
                    GetRegA
                END

            |IL.opDIVL:
                UnOp(reg1);
                REG.PushAll_1(R);
                pushc(param2);
                push(reg1);
                drop;
                CallRTL(IL._divmod);
                GetRegA

            |IL.opMOD:
                PushAll(2);
                CallRTL(IL._divmod);
                mov(rax, rdx);
                GetRegA

            |IL.opMODR:
                n := UTILS.Log2(param2);
                IF n > 0 THEN
                    UnOp(reg1);
                    andrc(reg1, param2 - 1);
                ELSIF n < 0 THEN
                    PushAll(1);
                    pushc(param2);
                    CallRTL(IL._divmod);
                    mov(rax, rdx);
                    GetRegA
                ELSE
                    UnOp(reg1);
                    xor(reg1, reg1)
                END

            |IL.opMODL:
                UnOp(reg1);
                REG.PushAll_1(R);
                pushc(param2);
                push(reg1);
                drop;
                CallRTL(IL._divmod);
                mov(rax, rdx);
                GetRegA

            |IL.opMUL:
                BinOp(reg1, reg2);
                oprr2(0FH, 0AFH, reg2, reg1); (* imul reg1, reg2 *)
                drop

            |IL.opMULC:
                IF (cmd.next(COMMAND).opcode = IL.opADD) & ((param2 = 2) OR (param2 = 4) OR (param2 = 8)) THEN
                    BinOp(reg1, reg2);
                    OutByte2(48H + 5 * (reg1 DIV 8) + 2 * (reg2 DIV 8), 8DH); (* lea reg1, [reg1 + reg2 * param2] *)
                    reg1 := reg1 MOD 8;
                    reg2 := reg2 MOD 8;
                    OutByte2(04H + reg1 * 8, reg1 + reg2 * 8 + 40H * UTILS.Log2(param2));
                    drop;
                    cmd := cmd.next(COMMAND)
                ELSE
                    UnOp(reg1);

                    a := param2;
                    IF a > 1 THEN
                        n := UTILS.Log2(a)
                    ELSIF a < -1 THEN
                        n := UTILS.Log2(-a)
                    ELSE
                        n := -1
                    END;

                    IF a = 1 THEN

                    ELSIF a = -1 THEN
                        neg(reg1)
                    ELSIF a = 0 THEN
                        xor(reg1, reg1)
                    ELSE
                        IF n > 0 THEN
                            IF a < 0 THEN
                                neg(reg1)
                            END;
                            shiftrc(shl, reg1, n)
                        ELSE
                            IF isLong(a) THEN
                                reg2 := GetAnyReg();
                                movabs(reg2, a);
                                ASSERT(reg1 # reg2);
                                oprr2(0FH, 0AFH, reg2, reg1); (* imul reg1, reg2 *)
                                drop
                            ELSE
                                (* imul reg1, a *)
                                Rex(reg1, reg1);
                                OutByte2(69H + short(a), 0C0H + (reg1 MOD 8) * 9);
                                OutIntByte(a)
                            END
                        END
                    END
                END

            |IL.opADDS:
                BinOp(reg1, reg2);
                _or(reg1, reg2);
                drop

            |IL.opSUBS:
                BinOp(reg1, reg2);
                not(reg2);
                and(reg1, reg2);
                drop

            |IL.opNOP, IL.opAND, IL.opOR:

            |IL.opSWITCH:
                UnOp(reg1);
                IF param2 = 0 THEN
                    reg2 := rax
                ELSE
                    reg2 := rcx
                END;
                IF reg1 # reg2 THEN
                    ASSERT(REG.GetReg(R, reg2));
                    ASSERT(REG.Exchange(R, reg1, reg2));
                    drop
                END;
                drop

            |IL.opENDSW:

            |IL.opCASEL:
                GetRegA;
                cmprc(rax, param1);
                jcc(jl, param2);
                drop

            |IL.opCASER:
                GetRegA;
                cmprc(rax, param1);
                jcc(jg, param2);
                drop

            |IL.opCASELR:
                GetRegA;
                cmprc(rax, param1);
                IF param2 = cmd.param3 THEN
                    jcc(jne, param2)
                ELSE
                    jcc(jl, param2);
                    jcc(jg, cmd.param3)
                END;
                drop

            |IL.opASR, IL.opROR, IL.opLSL, IL.opLSR:
                UnOp(reg1);
                IF reg1 # rcx THEN
                    ASSERT(REG.GetReg(R, rcx));
                    ASSERT(REG.Exchange(R, reg1, rcx));
                    drop
                END;

                BinOp(reg1, reg2);
                ASSERT(reg2 = rcx);
                Rex(reg1, 0);
                OutByte(0D3H);
                X86.shift(opcode, reg1 MOD 8); (* shift reg1, cl *)
                drop

            |IL.opASR1, IL.opROR1, IL.opLSL1, IL.opLSR1:
                UnOp(reg1);
                IF reg1 # rcx THEN
                    ASSERT(REG.GetReg(R, rcx));
                    ASSERT(REG.Exchange(R, reg1, rcx));
                    drop
                END;

                reg1 := GetAnyReg();
                movrc(reg1, param2);
                BinOp(reg1, reg2);
                ASSERT(reg1 = rcx);
                Rex(reg2, 0);
                OutByte(0D3H);
                X86.shift(opcode, reg2 MOD 8); (* shift reg2, cl *)
                drop;
                drop;
                ASSERT(REG.GetReg(R, reg2))

            |IL.opASR2, IL.opROR2, IL.opLSL2, IL.opLSR2:
                UnOp(reg1);
                shiftrc(opcode, reg1, param2 MOD 64)

            |IL.opGET, IL.opGETC:
                IF opcode = IL.opGET THEN
                    BinOp(reg1, reg2)
                ELSIF opcode = IL.opGETC THEN
                    UnOp(reg2);
                    reg1 := GetAnyReg();
                    movrc(reg1, param1)
                END;
                drop;
                drop;
                X86._movrm(reg1, reg1, 0, param2 * 8, FALSE);
                X86._movrm(reg1, reg2, 0, param2 * 8, TRUE)

            |IL.opCHKIDX:
                UnOp(reg1);
                cmprc(reg1, param2);
                jcc(jb, param1)

            |IL.opCHKIDX2:
                BinOp(reg1, reg2);
                IF param2 # -1 THEN
                    cmprr(reg2, reg1);
                    jcc(jb, param1);
                END;
                INCL(R.regs, reg1);
                DEC(R.top);
                R.stk[R.top] := reg2

            |IL.opLENGTH:
                PushAll(2);
                CallRTL(IL._length);
                GetRegA

            |IL.opLENGTHW:
                PushAll(2);
                CallRTL(IL._lengthw);
                GetRegA

            |IL.opLEN:
                n := param2;
                UnOp(reg1);
                drop;
                EXCL(R.regs, reg1);

                WHILE n > 0 DO
                    UnOp(reg2);
                    drop;
                    DEC(n)
                END;

                INCL(R.regs, reg1);
                ASSERT(REG.GetReg(R, reg1))

            |IL.opEQP, IL.opNEP, IL.opEQIP, IL.opNEIP:
                UnOp(reg1);
                reg2 := GetAnyReg();

                CASE opcode OF
                |IL.opEQP, IL.opNEP:
                    lea(reg2, param1, sCODE)

                |IL.opEQIP, IL.opNEIP:
                    lea(reg2, param1, sIMP);
                    movrm(reg2, reg2, 0)
                END;

                cmprr(reg1, reg2);
                drop;
                drop;
                reg1 := GetAnyReg();

                CASE opcode OF
                |IL.opEQP, IL.opEQIP: X86.setcc(sete,  reg1)
                |IL.opNEP, IL.opNEIP: X86.setcc(setne, reg1)
                END;

                andrc(reg1, 1)

            |IL.opINCCB, IL.opDECCB:
                UnOp(reg1);
                IF reg1 >= 8 THEN
                    OutByte(41H)
                END;
                OutByte3(80H, 28H * ORD(opcode = IL.opDECCB) + reg1 MOD 8, param2 MOD 256); (* add/sub byte[reg1], param2 MOD 256 *)
                drop

            |IL.opINCB, IL.opDECB:
                BinOp(reg1, reg2);
                IF (reg1 >= 8) OR (reg2 >= 8) THEN
                    OutByte(40H + reg2 DIV 8 + 4 * (reg1 DIV 8))
                END;
                OutByte2(28H * ORD(opcode = IL.opDECB), reg2 MOD 8 + 8 * (reg1 MOD 8)); (* add/sub byte[reg2], reg1_8 *)
                drop;
                drop

            |IL.opSAVEIP:
                UnOp(reg1);
                reg2 := GetAnyReg();
                lea(reg2, param2, sIMP);
                movrm(reg2, reg2, 0);
                push(reg2);
                drop;
                IF reg1 >= 8 THEN
                    OutByte(41H)
                END;
                OutByte2(8FH, reg1 MOD 8); (* pop qword[reg1] *)
                drop

            |IL.opCLEANUP:
                IF param2 # 0 THEN
                    addrc(rsp, param2 * 8)
                END

            |IL.opPOPSP:
                pop(rsp)

            |IL.opLOADF:
                UnOp(reg1);
                INC(xmm);
                IF xmm > MAX_XMM THEN
                    ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
                END;
                movsdrm(xmm, reg1, 0);
                drop

            |IL.opPUSHF:
                ASSERT(xmm >= 0);
                subrc(rsp, 8);
                movsdmr(rsp, 0, xmm);
                DEC(xmm)

            |IL.opCONSTF:
                float := cmd.float;
                INC(xmm);
                IF xmm > MAX_XMM THEN
                    ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
                END;
                (* movsd xmm, qword ptr [rip + Numbers_Offs + Numbers_Count * 8 + DATA] *)
                OutByte(0F2H);
                IF xmm >= 8 THEN
                    OutByte(44H)
                END;
                OutByte3(0FH, 10H, 05H + 8 * (xmm MOD 8));
                X86.Reloc(sDATA, Numbers_Offs + Numbers_Count * 8);
                NewNumber(UTILS.splitf(float, a, b))

            |IL.opSAVEF, IL.opSAVEFI:
                ASSERT(xmm >= 0);
                UnOp(reg1);
                movsdmr(reg1, 0, xmm);
                DEC(xmm);
                drop

            |IL.opADDF:
                ASSERT(xmm >= 1);
                opxx(58H, xmm - 1, xmm);
                DEC(xmm)

            |IL.opSUBF:
                ASSERT(xmm >= 1);
                opxx(5CH, xmm - 1, xmm);
                DEC(xmm)

            |IL.opSUBFI:
                ASSERT(xmm >= 1);
                opxx(5CH, xmm, xmm - 1);
                opxx(10H, xmm - 1, xmm);
                DEC(xmm)

            |IL.opMULF:
                ASSERT(xmm >= 1);
                opxx(59H, xmm - 1, xmm);
                DEC(xmm)

            |IL.opDIVF:
                ASSERT(xmm >= 1);
                opxx(5EH, xmm - 1, xmm);
                DEC(xmm)

            |IL.opDIVFI:
                ASSERT(xmm >= 1);
                opxx(5EH, xmm, xmm - 1);
                opxx(10H, xmm - 1, xmm);
                DEC(xmm)

            |IL.opFABS, IL.opUMINF: (* andpd/xorpd xmm, xmmword[rip + Numbers_Offs + (16) + DATA] *)
                ASSERT(xmm >= 0);
                OutByte(66H);
                IF xmm >= 8 THEN
                    OutByte(44H)
                END;
                OutByte3(0FH, 54H + 3 * ORD(opcode = IL.opUMINF), 05H + (xmm MOD 8) * 8);
                X86.Reloc(sDATA, Numbers_Offs + 16 * ORD(opcode = IL.opFABS))

            |IL.opFLT:
                UnOp(reg1);
                INC(xmm);
                IF xmm > MAX_XMM THEN
                    ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
                END;
                OutByte(0F2H); Rex(reg1, xmm); OutByte(0FH); (* cvtsi2sd xmm, reg1 *)
                OutByte2(2AH, 0C0H + (xmm MOD 8) * 8 + reg1 MOD 8);
                drop

            |IL.opFLOOR:
                ASSERT(xmm >= 0);
                reg1 := GetAnyReg();
                subrc(rsp, 8);
                OutByte3(00FH, 0AEH, 05CH); OutByte2(024H, 004H);                       (* stmxcsr dword[rsp+4];                              *)
                OutByte2(00FH, 0AEH); OutByte2(01CH, 024H);                             (* stmxcsr dword[rsp];                                *)
                OutByte3(081H, 024H, 024H); OutByte2(0FFH, 09FH); OutByte2(0FFH, 0FFH); (* and dword[rsp],11111111111111111001111111111111b;  *)
                OutByte3(081H, 00CH, 024H); OutByte2(000H, 020H); OutByte2(000H, 000H); (* or dword[rsp],00000000000000000010000000000000b;   *)
                OutByte2(00FH, 0AEH); OutByte2(014H, 024H);                             (* ldmxcsr dword[rsp];                                *)
                OutByte(0F2H); Rex(xmm, reg1); OutByte(0FH);                            (* cvtsd2si reg1, xmm                                 *)
                OutByte2(2DH, 0C0H + xmm MOD 8 + (reg1 MOD 8) * 8);
                OutByte3(00FH, 0AEH, 054H); OutByte2(024H, 004H);                       (* ldmxcsr dword[rsp+4];                              *)
                addrc(rsp, 8);
                DEC(xmm)

            |IL.opEQF .. IL.opGEF:
                ASSERT(xmm >= 1);
                fcmp(opcode, xmm);
                DEC(xmm, 2)

            |IL.opINF:
                INC(xmm);
                IF xmm > MAX_XMM THEN
                    ERRORS.ErrorMsg(fname, param1, param2, FPR_ERR)
                END;
                (* movsd xmm, qword ptr [rip + Numbers_Offs + 32 + DATA] *)
                OutByte(0F2H);
                IF xmm >= 8 THEN
                    OutByte(44H)
                END;
                OutByte3(0FH, 10H, 05H + 8 * (xmm MOD 8));
                X86.Reloc(sDATA, Numbers_Offs + 32)

            |IL.opPACK, IL.opPACKC:
                IF opcode = IL.opPACK THEN
                    BinOp(reg1, reg2)
                ELSE
                    UnOp(reg1);
                    reg2 := GetAnyReg();
                    movrc(reg2, param2)
                END;
                push(reg1);
                movrm(reg1, reg1, 0);
                shiftrc(shl, reg1, 1);
                shiftrc(shr, reg1, 53);
                add(reg1, reg2);
                andrc(reg1, ORD({0..10}));
                shiftrc(shl, reg1, 52);
                movrm(reg2, rsp, 0);
                movrm(reg2, reg2, 0);

                push(reg1);
                lea(reg1, Numbers_Offs + 40, sDATA); (* {0..51, 63} *)
                movrm(reg1, reg1, 0);
                and(reg2, reg1);
                pop(reg1);

                _or(reg2, reg1);
                pop(reg1);
                movmr(reg1, 0, reg2);
                drop;
                drop

            |IL.opUNPK, IL.opLADR_UNPK:

                IF opcode = IL.opLADR_UNPK THEN
                    n := param2 * 8;
                    UnOp(reg1);
                    reg2 := GetAnyReg();
                    Rex(0, reg2);
                    OutByte2(8DH, 45H + long(n) + (reg2 MOD 8) * 8); (* lea reg2, qword[rbp+n] *)
                    OutIntByte(n)
                ELSE
                    BinOp(reg1, reg2)
                END;

                push(reg1);
                movrm(reg1, reg1, 0);
                shiftrc(shl, reg1, 1);
                shiftrc(shr, reg1, 53);
                subrc(reg1, 1023);

                movmr(reg2, 0, reg1);

                pop(reg2);
                movrm(reg1, reg2, 0);

                push(reg2);
                lea(reg2, Numbers_Offs + 48, sDATA); (* {52..61} *)
                movrm(reg2, reg2, 0);
                _or(reg1, reg2);
                pop(reg2);

                Rex(reg1, 0);
                OutByte2(0FH, 0BAH);
                OutByte2(0F0H + reg1 MOD 8, 3EH); (* btr reg1, 62 *)
                movmr(reg2, 0, reg1);
                drop;
                drop

            |IL.opSADR_PARAM:
                pushDA(stroffs + param2)

            |IL.opVADR_PARAM:
                X86.pushm(rbp, param2 * 8)

            |IL.opLOAD64_PARAM:
                UnOp(reg1);
                X86.pushm(reg1, 0);
                drop

            |IL.opLLOAD64_PARAM:
                X86.pushm(rbp, param2 * 8)

            |IL.opGLOAD64_PARAM:
                OutByte2(0FFH, 35H); (* push qword[rip + param2 + BSS] *)
                X86.Reloc(sBSS, param2)

            |IL.opCONST_PARAM:
                pushc(param2)

            |IL.opGLOAD32_PARAM, IL.opLOAD32_PARAM:
                IF opcode = IL.opGLOAD32_PARAM THEN
                    reg1 := GetAnyReg();
                    lea(reg1, param2, sBSS)
                ELSE
                    UnOp(reg1)
                END;
                movrm32(reg1, reg1, 0);
                shiftrc(shl, reg1, 32);
                shiftrc(shr, reg1, 32);
                push(reg1);
                drop

            |IL.opLLOAD32_PARAM:
                reg1 := GetAnyReg();
                movrm32(reg1, rbp, param2 * 8);
                shiftrc(shl, reg1, 32);
                shiftrc(shr, reg1, 32);
                push(reg1);
                drop

            |IL.opLADR_SAVEC:
                n := param1 * 8;
                IF isLong(param2) THEN
                    reg2 := GetAnyReg();
                    movrc(reg2, param2);
                    movmr(rbp, n, reg2);
                    drop
                ELSE
                    OutByte3(48H, 0C7H, 45H + long(n)); (* mov qword[rbp+n], param2 *)
                    OutIntByte(n);
                    OutInt(param2)
                END

            |IL.opGADR_SAVEC:
                IF isLong(param2) THEN
                    reg1 := GetAnyReg();
                    movrc(reg1, param2);
                    reg2 := GetAnyReg();
                    lea(reg2, param1, sBSS);
                    movmr(reg2, 0, reg1);
                    drop;
                    drop
                ELSE
                    (* mov qword[rip + param1 - 4 + BSS], param2 *)
                    OutByte3(48H, 0C7H, 05H);
                    X86.Reloc(sBSS, param1 - 4);
                    OutInt(param2)
                END

            |IL.opLADR_SAVE:
                UnOp(reg1);
                movmr(rbp, param2 * 8, reg1);
                drop

            |IL.opLADR_INCC:
                IF isLong(param2) THEN
                    reg2 := GetAnyReg();
                    movrc(reg2, param2);
                    n := param1 * 8;
                    Rex(0, reg2);
                    OutByte2(01H, 45H + long(n) + (reg2 MOD 8) * 8);
                    OutIntByte(n); (* add qword[rbp+n], reg2 *)
                    drop
                ELSIF ABS(param2) = 1 THEN
                    n := param1 * 8;
                    OutByte3(48H, 0FFH, 45H + 8 * ORD(param2 = -1) + long(n)); (* inc/dec qword[rbp+n] *)
                    OutIntByte(n)
                ELSE
                    n := param1 * 8;
                    OutByte3(48H, 81H + short(param2), 45H + long(n));
                    OutIntByte(n);
                    OutIntByte(param2) (* add qword[rbp+n], param2 *)
                END

            |IL.opLADR_INCCB, IL.opLADR_DECCB:
                param2 := param2 MOD 256;
                n := param1 * 8;
                OutByte2(80H, 45H + long(n) + 28H * ORD(opcode = IL.opLADR_DECCB));
                OutIntByte(n);
                OutByte(param2) (* add/sub byte[rbp+n], param2 *)

            |IL.opLADR_INC, IL.opLADR_DEC:
                UnOp(reg1);
                n := param2 * 8;
                Rex(0, reg1);
                OutByte2(01H + 28H * ORD(opcode = IL.opLADR_DEC), 45H + long(n) + (reg1 MOD 8) * 8);
                OutIntByte(n); (* add/sub qword[rbp+n], reg1 *)
                drop

            |IL.opLADR_INCB, IL.opLADR_DECB:
                UnOp(reg1);
                n := param2 * 8;
                IF reg1 >= 8 THEN
                    OutByte(44H)
                END;
                OutByte2(28H * ORD(opcode = IL.opLADR_DECB), 45H + long(n) + 8 * (reg1 MOD 8));
                OutIntByte(n); (* add/sub byte[rbp+n], reg1_8 *)
                drop

            |IL.opLADR_INCL, IL.opLADR_EXCL:
                UnOp(reg1);
                cmprc(reg1, 64);
                n := param2 * 8;
                OutByte2(73H, 5 + 3 * ORD(~X86.isByte(n))); (* jnb L *)
                Rex(0, reg1);
                OutByte3(0FH, 0ABH + 8 * ORD(opcode = IL.opLADR_EXCL), 45H + long(n) + 8 * (reg1 MOD 8));
                OutIntByte(n); (* bts/btr qword[rbp+n], reg1 *)
                (* L: *)
                drop

            |IL.opLADR_INCLC, IL.opLADR_EXCLC:
                n := param1 * 8;
                OutByte3(48H, 0FH, 0BAH); (* bts/btr qword[rbp+n], param2 *)
                OutByte(6DH + long(n) + 8 * ORD(opcode = IL.opLADR_EXCLC));
                OutIntByte(n);
                OutByte(param2)

            |IL.opFNAME:
                fname := cmd(IL.FNAMECMD).fname

            END;

            cmd := cmd.next(COMMAND)
        END;

        ASSERT(R.pushed = 0);
        ASSERT(R.top = -1);
        ASSERT(xmm = -1)
    END translate;


PROCEDURE prolog (modname: ARRAY OF CHAR; target, stack_size: INTEGER);
    VAR
        ModName_Offs, entry, L: INTEGER;

    BEGIN
        ModName_Offs := tcount * 8 + CHL.Length(IL.codes.data);
        Numbers_Offs := ModName_Offs + LENGTH(modname) + 1;
        ASSERT(UTILS.Align(Numbers_Offs, 16));

        entry := NewLabel();
        X86.SetLabel(entry);

        IF target = TARGETS.Win64DLL THEN
            dllret := NewLabel();
            push(r8);
            push(rdx);
            push(rcx);
            CallRTL(IL._dllentry);
            test(rax);
            jcc(je, dllret);
            pushc(0)
        ELSIF target = TARGETS.Linux64 THEN
            push(rsp)
        ELSE
            pushc(0)
        END;

        lea(rax, entry, sCODE);
        push(rax);
        pushDA(0); (* TYPES *)
        pushc(tcount);
        pushDA(ModName_Offs); (* MODNAME *)
        CallRTL(IL._init);

        IF target IN {TARGETS.Win64C, TARGETS.Win64GUI, TARGETS.Linux64} THEN
            L := NewLabel();
            pushc(0);
            push(rsp);
            pushc(1024 * 1024 * stack_size);
            pushc(0);
            CallRTL(IL._new);
            pop(rax);
            test(rax);
            jcc(je, L);
            GetRegA;
            addrc(rax, 1024 * 1024 * stack_size - 8);
            drop;
            mov(rsp, rax);
            X86.SetLabel(L)
        END
    END prolog;


PROCEDURE epilog (modname: ARRAY OF CHAR; target: INTEGER);
    VAR
        i, n: INTEGER;
        number: Number;
        exp: IL.EXPORT_PROC;


        PROCEDURE _import (imp: LISTS.LIST);
        VAR
            lib:  IL.IMPORT_LIB;
            proc: IL.IMPORT_PROC;

        BEGIN

            lib := imp.first(IL.IMPORT_LIB);
            WHILE lib # NIL DO
                BIN.Import(prog, lib.name, 0);
                proc := lib.procs.first(IL.IMPORT_PROC);
                WHILE proc # NIL DO
                    BIN.Import(prog, proc.name, proc.label);
                    proc := proc.next(IL.IMPORT_PROC)
                END;
                lib := lib.next(IL.IMPORT_LIB)
            END

        END _import;


    BEGIN
        IF target = TARGETS.Win64DLL THEN
            X86.SetLabel(dllret);
            X86.ret
        ELSIF target = TARGETS.Linux64SO THEN
            sofinit := NewLabel();
            X86.ret;
            X86.SetLabel(sofinit);
            CallRTL(IL._sofinit);
            X86.ret
        ELSE
            pushc(0);
            CallRTL(IL._exit)
        END;

        X86.fixup;

        i := 0;
        WHILE i < tcount DO
            BIN.PutData64LE(prog, CHL.GetInt(IL.codes.types, i));
            INC(i)
        END;

        i := 0;
        WHILE i < CHL.Length(IL.codes.data) DO
            BIN.PutData(prog, CHL.GetByte(IL.codes.data, i));
            INC(i)
        END;

        BIN.PutDataStr(prog, modname);
        BIN.PutData(prog, 0);
        n := CHL.Length(prog.data);
        ASSERT(UTILS.Align(n, 16));
        i := n - CHL.Length(prog.data);
        WHILE i > 0 DO
            BIN.PutData(prog, 0);
            DEC(i)
        END;
        number := Numbers.first(Number);
        FOR i := 0 TO Numbers_Count - 1 DO
            BIN.PutData64LE(prog, number.value);
            number := number.next(Number)
        END;

        exp := IL.codes.export.first(IL.EXPORT_PROC);
        WHILE exp # NIL DO
            BIN.Export(prog, exp.name, exp.label);
            exp := exp.next(IL.EXPORT_PROC)
        END;

        _import(IL.codes._import)
    END epilog;


PROCEDURE CodeGen* (outname: ARRAY OF CHAR; target: INTEGER; options: PROG.OPTIONS);
    VAR
        path, modname, ext: PATHS.PATH;

    BEGIN
        Xmm[0] := 0;
        X86.align16(TRUE);
        tcount := CHL.Length(IL.codes.types);

        Win64RegPar[0] := rcx;
        Win64RegPar[1] := rdx;
        Win64RegPar[2] := r8;
        Win64RegPar[3] := r9;

        SystemVRegPar[0] := rdi;
        SystemVRegPar[1] := rsi;
        SystemVRegPar[2] := rdx;
        SystemVRegPar[3] := rcx;
        SystemVRegPar[4] := r8;
        SystemVRegPar[5] := r9;

        PATHS.split(outname, path, modname, ext);
        S.append(modname, ext);

        REG.Init(R, push, pop, mov, xchg, {rax, rcx, rdx, r8, r9, r10, r11});

        IL.set_bss(MAX(IL.codes.bss, MAX(IL.codes.dmin - CHL.Length(IL.codes.data), 8)));

        Numbers := LISTS.create(NIL);
        Numbers_Count := 0;
        NewNumber(ROR(1, 1));      (* 8000000000000000H *)
        NewNumber(0);
        NewNumber(ROR(-2, 1));     (* 7FFFFFFFFFFFFFFFH *)
        NewNumber(-1);
        NewNumber(ROR(7FFH, 12));  (* +Infinity *)
        NewNumber(ORD(-BITS(LSR(ASR(ROR(1, 1), 10), 1))));  (* {0..51, 63} *)
        NewNumber(LSR(ASR(ROR(1, 1), 9), 2));  (* {52..61} *)

        prog := BIN.create(IL.codes.lcount);
        BIN.SetParams(prog, IL.codes.bss, 1, WCHR(1), WCHR(0));

        X86.SetProgram(prog);

        prolog(modname, target, options.stack);
        translate(IL.codes.commands, tcount * 8);
        epilog(modname, target);

        BIN.fixup(prog);
        IF TARGETS.OS = TARGETS.osWIN64 THEN
            PE32.write(prog, outname, target = TARGETS.Win64C, target = TARGETS.Win64DLL, TRUE, options.PE32FileAlignment)
        ELSIF TARGETS.OS = TARGETS.osLINUX64 THEN
            ELF.write(prog, outname, sofinit, target = TARGETS.Linux64SO, TRUE)
        END
    END CodeGen;


END AMD64.