path: root/src/kmk/kmkbuiltin/kSubmit.c

/* $Id: kSubmit.c 3413 2020-08-20 08:20:15Z bird $ */
/** @file
 * kMk Builtin command - submit job to a kWorker.
 */

/*
 * Copyright (c) 2007-2016 knut st. osmundsen <bird-kBuild-spamx@anduin.net>
 *
 * This file is part of kBuild.
 *
 * kBuild is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * kBuild is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with kBuild.  If not, see <http://www.gnu.org/licenses/>
 *
 */

/*******************************************************************************
*   Header Files                                                               *
*******************************************************************************/
#ifdef __APPLE__
# define _POSIX_C_SOURCE 1 /* 10.4 sdk and unsetenv */
#endif
#include "makeint.h"
#include "job.h"
#include "variable.h"
#include "pathstuff.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#ifdef HAVE_ALLOCA_H
# include <alloca.h>
#endif
#if defined(_MSC_VER)
# include <ctype.h>
# include <io.h>
# include <direct.h>
# include <process.h>
#else
# include <unistd.h>
#endif
#ifdef KBUILD_OS_WINDOWS
# ifndef CONFIG_NEW_WIN_CHILDREN
#  include "sub_proc.h"
# else
#  include "../w32/winchildren.h"
# endif
# include "nt/nt_child_inject_standard_handles.h"
#endif

#include "kbuild.h"
#include "kmkbuiltin.h"
#include "err.h"


/*********************************************************************************************************************************
*   Defined Constants And Macros                                                                                                 *
*********************************************************************************************************************************/
/** Hashes a pid. */
#define KWORKER_PID_HASH(a_pid) ((size_t)(a_pid) % 61)

#define TUPLE(a_sz)     a_sz, sizeof(a_sz) - 1


/*********************************************************************************************************************************
*   Structures and Typedefs                                                                                                      *
*********************************************************************************************************************************/
typedef struct WORKERINSTANCE *PWORKERINSTANCE;
typedef struct WORKERINSTANCE
{
    /** Pointer to the next worker instance. */
    PWORKERINSTANCE         pNext;
    /** Pointer to the previous worker instance. */
    PWORKERINSTANCE         pPrev;
    /** Pointer to the next worker with the same pid hash slot. */
    PWORKERINSTANCE         pNextPidHash;
    /** 32 or 64. */
    unsigned                cBits;
    /** The process ID of the kWorker process. */
    pid_t                   pid;
    union
    {
        struct
        {
            /** The exit code. */
            int32_t         rcExit;
            /** Set to 1 if the worker is exiting. */
            uint8_t         bWorkerExiting;
            uint8_t         abUnused[3];
        } s;
        uint8_t             ab[8];
    } Result;
    /** Number of result bytes read alread.  */
    size_t                  cbResultRead;

#ifdef KBUILD_OS_WINDOWS
    /** The process handle. */
    HANDLE                  hProcess;
    /** The bi-directional pipe we use to talk to the kWorker process. */
    HANDLE                  hPipe;
    /** For overlapped read (have valid event semaphore). */
    OVERLAPPED              OverlappedRead;
# ifdef CONFIG_NEW_WIN_CHILDREN
    /** Standard output catcher (reused). */
    PWINCCWPIPE             pStdOut;
    /** Standard error catcher (reused). */
    PWINCCWPIPE             pStdErr;
# endif
#else
    /** The socket descriptor we use to talk to the kWorker process. */
    int                     fdSocket;
#endif

    /** --debug-dump-history-on-failure.   */
    int                     fDebugDumpHistoryOnFailure;
    /** Current history index (must mod with aHistory element count). */
    unsigned                iHistory;
    /** History.   */
    struct
    {
        /** Pointer to the message, NULL if none. */
        void               *pvMsg;
        /** The message size, zero if not present. */
        size_t              cbMsg;
    } aHistory[4];

    /** What it's busy with.  NULL if idle. */
    struct child           *pBusyWith;
} WORKERINSTANCE;


typedef struct WORKERLIST
{
    /** The head of the list.  NULL if empty. */
    PWORKERINSTANCE         pHead;
    /** The tail of the list.  NULL if empty. */
    PWORKERINSTANCE         pTail;
    /** Number of list entries. */
    size_t                  cEntries;
} WORKERLIST;
typedef WORKERLIST *PWORKERLIST;


/*********************************************************************************************************************************
*   Global Variables                                                                                                             *
*********************************************************************************************************************************/
/** List of idle worker.*/
static WORKERLIST           g_IdleList;
/** List of busy workers. */
static WORKERLIST           g_BusyList;
/** PID hash table for the workers.
 * @sa KWORKER_PID_HASH() */
static PWORKERINSTANCE      g_apPidHash[61];

#ifdef KBUILD_OS_WINDOWS
/** For naming the pipes.
 * Also indicates how many worker instances we've spawned. */
static unsigned             g_uWorkerSeqNo = 0;
#endif
/** Set if we've registred the atexit handler already. */
static int                  g_fAtExitRegistered = 0;

/** @var g_cArchBits
 * The bit count of the architecture this binary is compiled for. */
/** @var g_szArch
 * The name of the architecture this binary is compiled for. */
/** @var g_cArchBits
 * The bit count of the alternative architecture. */
/** @var g_szAltArch
 * The name of the alternative architecture. */
#if defined(KBUILD_ARCH_AMD64)
static unsigned             g_cArchBits    = 64;
static char const           g_szArch[]     = "amd64";
static unsigned             g_cAltArchBits = 32;
static char const           g_szAltArch[]  = "x86";
#elif defined(KBUILD_ARCH_X86)
static unsigned             g_cArchBits    = 32;
static char const           g_szArch[]     = "x86";
static unsigned             g_cAltArchBits = 64;
static char const           g_szAltArch[]  = "amd64";
#else
# error "Port me!"
#endif

#ifdef KBUILD_OS_WINDOWS
/** The processor group allocator state. */
static MKWINCHILDCPUGROUPALLOCSTATE g_SubmitProcessorGroupAllocator;
# if K_ARCH_BITS == 64
/** The processor group allocator state for 32-bit processes. */
static MKWINCHILDCPUGROUPALLOCSTATE g_SubmitProcessorGroupAllocator32;
# endif
#endif

#ifdef KBUILD_OS_WINDOWS
/** Pointer to kernel32!SetThreadGroupAffinity. */
static BOOL (WINAPI *g_pfnSetThreadGroupAffinity)(HANDLE, const GROUP_AFFINITY*, GROUP_AFFINITY *);
#endif



/**
 * Unlinks a worker instance from a list.
 *
 * @param   pList               The list.
 * @param   pWorker             The worker.
 */
static void kSubmitListUnlink(PWORKERLIST pList, PWORKERINSTANCE pWorker)
{
    PWORKERINSTANCE pNext = pWorker->pNext;
    PWORKERINSTANCE pPrev = pWorker->pPrev;

    if (pNext)
    {
        assert(pNext->pPrev == pWorker);
        pNext->pPrev = pPrev;
    }
    else
    {
        assert(pList->pTail == pWorker);
        pList->pTail = pPrev;
    }

    if (pPrev)
    {
        assert(pPrev->pNext == pWorker);
        pPrev->pNext = pNext;
    }
    else
    {
        assert(pList->pHead == pWorker);
        pList->pHead = pNext;
    }

    assert(!pList->pHead || pList->pHead->pPrev == NULL);
    assert(!pList->pTail || pList->pTail->pNext == NULL);

    assert(pList->cEntries > 0);
    pList->cEntries--;

    pWorker->pNext = NULL;
    pWorker->pPrev = NULL;
}


/**
 * Appends a worker instance to the tail of a list.
 *
 * @param   pList               The list.
 * @param   pWorker             The worker.
 */
static void kSubmitListAppend(PWORKERLIST pList, PWORKERINSTANCE pWorker)
{
    PWORKERINSTANCE pTail = pList->pTail;

    assert(pTail != pWorker);
    assert(pList->pHead != pWorker);

    pWorker->pNext = NULL;
    pWorker->pPrev = pTail;
    if (pTail != NULL)
    {
        assert(pTail->pNext == NULL);
        pTail->pNext = pWorker;
    }
    else
    {
        assert(pList->pHead == NULL);
        pList->pHead = pWorker;
    }
    pList->pTail = pWorker;

    assert(pList->pHead->pPrev == NULL);
    assert(pList->pTail->pNext == NULL);

    pList->cEntries++;
}


/**
 * Remove worker from the process ID hash table.
 *
 * @param   pWorker             The worker.
 */
static void kSubmitPidHashRemove(PWORKERINSTANCE pWorker)
{
    size_t idxHash = KWORKER_PID_HASH(pWorker->pid);
    if (g_apPidHash[idxHash] == pWorker)
        g_apPidHash[idxHash] = pWorker->pNext;
    else
    {
        PWORKERINSTANCE pPrev = g_apPidHash[idxHash];
        while (pPrev && pPrev->pNext != pWorker)
            pPrev = pPrev->pNext;
        assert(pPrev != NULL);
        if (pPrev)
            pPrev->pNext = pWorker->pNext;
    }
    pWorker->pid = -1;
}


/**
 * Looks up a worker by its process ID.
 *
 * @returns Pointer to the worker instance if found. NULL if not.
 * @param   pid                 The process ID of the worker.
 */
static PWORKERINSTANCE kSubmitFindWorkerByPid(pid_t pid)
{
    PWORKERINSTANCE pWorker = g_apPidHash[KWORKER_PID_HASH(pid)];
    while (pWorker && pWorker->pid != pid)
        pWorker = pWorker->pNextPidHash;
    return pWorker;
}


/**
 * Calcs the path to the kWorker binary for the worker.
 *
 * @returns
 * @param   pCtx            The command execution context.
 * @param   pWorker         The worker (for its bitcount).
 * @param   pszExecutable   The output buffer.
 * @param   cbExecutable    The output buffer size.
 */
static int kSubmitCalcExecutablePath(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, char *pszExecutable, size_t cbExecutable)
{
#if defined(KBUILD_OS_WINDOWS) || defined(KBUILD_OS_OS2)
    static const char   s_szWorkerName[] = "kWorker.exe";
#else
    static const char   s_szWorkerName[] = "kWorker";
#endif
    const char         *pszBinPath = get_kbuild_bin_path();
    size_t const        cchBinPath = strlen(pszBinPath);
    size_t              cchExecutable;
    if (   pWorker->cBits == g_cArchBits
        ?  cchBinPath + 1 + sizeof(s_szWorkerName) <= cbExecutable
        :  cchBinPath + 1 - sizeof(g_szArch) + sizeof(g_szAltArch) + sizeof(s_szWorkerName) <= cbExecutable )
    {
        memcpy(pszExecutable, pszBinPath, cchBinPath);
        cchExecutable = cchBinPath;

        /* Replace the arch bin directory extension with the alternative one if requested. */
        if (pWorker->cBits != g_cArchBits)
        {
            if (   cchBinPath < sizeof(g_szArch)
                || memcmp(&pszExecutable[cchBinPath - sizeof(g_szArch) + 1], g_szArch, sizeof(g_szArch) - 1) != 0)
                return errx(pCtx, 1, "KBUILD_BIN_PATH does not end with main architecture (%s) as expected: %s",
                            pszBinPath, g_szArch);
            cchExecutable -= sizeof(g_szArch) - 1;
            memcpy(&pszExecutable[cchExecutable], g_szAltArch, sizeof(g_szAltArch) - 1);
            cchExecutable += sizeof(g_szAltArch) - 1;
        }

        /* Append a slash and the worker name. */
        pszExecutable[cchExecutable++] = '/';
        memcpy(&pszExecutable[cchExecutable], s_szWorkerName, sizeof(s_szWorkerName));
        return 0;
    }
    return errx(pCtx, 1, "KBUILD_BIN_PATH is too long");
}


#ifdef KBUILD_OS_WINDOWS
/**
 * Calcs the UTF-16 path to the kWorker binary for the worker.
 *
 * @returns
 * @param   pCtx            The command execution context.
 * @param   pWorker         The worker (for its bitcount).
 * @param   pwszExecutable  The output buffer.
 * @param   cwcExecutable   The output buffer size.
 */
static int kSubmitCalcExecutablePathW(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, wchar_t *pwszExecutable, size_t cwcExecutable)
{
    char szExecutable[MAX_PATH];
    int rc = kSubmitCalcExecutablePath(pCtx, pWorker, szExecutable, sizeof(szExecutable));
    if (rc == 0)
    {
        int cwc = MultiByteToWideChar(CP_ACP, 0 /*fFlags*/, szExecutable, strlen(szExecutable) + 1,
                                      pwszExecutable, cwcExecutable);
        if (cwc > 0)
            return 0;
        return errx(pCtx, 1, "MultiByteToWideChar failed on '%s': %u", szExecutable, GetLastError());
    }
    return rc;
}
#endif


/**
 * Creates a new worker process.
 *
 * @returns 0 on success, non-zero value on failure.
 * @param   pCtx                The command execution context.
 * @param   pWorker             The worker structure.  Caller does the linking
 *                              (as we might be reusing an existing worker
 *                              instance because a worker shut itself down due
 *                              to high resource leak level).
 * @param   cVerbosity          The verbosity level.
 */
static int kSubmitSpawnWorker(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, int cVerbosity)
{
    int rc;
#ifdef KBUILD_OS_WINDOWS
    wchar_t wszExecutable[MAX_PATH];
#else
    PATH_VAR(szExecutable);
#endif

    /*
     * Get the output path so it can be passed on as a volatile.
     */
    const char      *pszVarVolatile;
    struct variable *pVarVolatile = lookup_variable(TUPLE("PATH_OUT"));
    if (pVarVolatile)
        pszVarVolatile = "PATH_OUT";
    else
    {
        pVarVolatile = lookup_variable(TUPLE("PATH_OUT_BASE"));
        if (pVarVolatile)
            pszVarVolatile = "PATH_OUT_BASE";
        else
            warn(pCtx, "Neither PATH_OUT_BASE nor PATH_OUT was found.");
    }
    if (pVarVolatile && strchr(pVarVolatile->value, '"'))
        return errx(pCtx, -1, "%s contains double quotes.", pszVarVolatile);
    if (pVarVolatile && strlen(pVarVolatile->value) >= GET_PATH_MAX)
        return errx(pCtx, -1, "%s is too long (max %u)", pszVarVolatile, GET_PATH_MAX);

    /*
     * Construct the executable path.
     */
#ifdef KBUILD_OS_WINDOWS
    rc = kSubmitCalcExecutablePathW(pCtx, pWorker, wszExecutable, K_ELEMENTS(wszExecutable));
#else
    rc = kSubmitCalcExecutablePath(pCtx, pWorker, szExecutable, GET_PATH_MAX);
#endif
    if (rc == 0)
    {
#ifdef KBUILD_OS_WINDOWS
        static DWORD    s_fDenyRemoteClients = ~(DWORD)0;
        wchar_t         wszPipeName[128];
        HANDLE          hWorkerPipe;
        int             iProcessorGroup;

# if K_ARCH_BITS == 64
        /** @todo make it return -1 if not applicable (e.g only one group).  */
        if (pWorker->cBits != 32)
            iProcessorGroup = MkWinChildAllocateCpuGroup(&g_SubmitProcessorGroupAllocator);
        else
            iProcessorGroup = MkWinChildAllocateCpuGroup(&g_SubmitProcessorGroupAllocator32);
# else
        iProcessorGroup = MkWinChildAllocateCpuGroup(&g_SubmitProcessorGroupAllocator);
# endif

        /*
         * Create the bi-directional pipe with overlapping I/O enabled.
         */
        if (s_fDenyRemoteClients == ~(DWORD)0)
            s_fDenyRemoteClients = GetVersion() >= 0x60000 ? PIPE_REJECT_REMOTE_CLIENTS : 0;
        _snwprintf(wszPipeName, sizeof(wszPipeName), L"\\\\.\\pipe\\kmk-%u-kWorker-%u-%u",
                   GetCurrentProcessId(), g_uWorkerSeqNo++, GetTickCount());
        hWorkerPipe = CreateNamedPipeW(wszPipeName,
                                       PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED | FILE_FLAG_FIRST_PIPE_INSTANCE /* win2k sp2+ */,
                                       PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | s_fDenyRemoteClients,
                                       1 /* cMaxInstances */,
                                       64 /*cbOutBuffer*/,
                                       65536 /*cbInBuffer*/,
                                       0 /*cMsDefaultTimeout -> 50ms*/,
                                       NULL /* pSecAttr - no inherit */);
        if (hWorkerPipe != INVALID_HANDLE_VALUE)
        {
            pWorker->hPipe = CreateFileW(wszPipeName,
                                         GENERIC_READ | GENERIC_WRITE,
                                         0 /* dwShareMode - no sharing */,
                                         NULL /*pSecAttr - no inherit */,
                                         OPEN_EXISTING,
                                         FILE_FLAG_OVERLAPPED,
                                         NULL /*hTemplate*/);
            if (pWorker->hPipe != INVALID_HANDLE_VALUE)
            {
                pWorker->OverlappedRead.hEvent = CreateEventW(NULL /*pSecAttrs - no inherit*/, TRUE /*bManualReset*/,
                                                              TRUE /*bInitialState*/, NULL /*pwszName*/);
                if (pWorker->OverlappedRead.hEvent != NULL)
                {
                    extern int          process_priority; /* main.c */
                    wchar_t             wszCommandLine[MAX_PATH * 3 + 32];
                    wchar_t            *pwszDst = wszCommandLine;
                    size_t              cwcDst = K_ELEMENTS(wszCommandLine);
                    int                 cwc;
                    DWORD               fFlags;
                    STARTUPINFOW        StartupInfo;
                    PROCESS_INFORMATION ProcInfo = { NULL, NULL, 0, 0 };

                    /*
                     * Compose the command line.
                     */
                    cwc = _snwprintf(pwszDst, cwcDst, L"\"%s\" ", wszExecutable);
                    assert(cwc > 0 && cwc < cwcDst);
                    pwszDst += cwc;
                    cwcDst  -= cwc;
                    if (pVarVolatile && *pVarVolatile->value)
                    {
                        char chEnd = strchr(pVarVolatile->value, '\0')[-1];
                        if (chEnd == '\\')
                            cwc = _snwprintf(pwszDst, cwcDst, L" --volatile \"%S.\"", pVarVolatile->value);
                        else
                            cwc = _snwprintf(pwszDst, cwcDst, L" --volatile \"%S\"", pVarVolatile->value);
                        assert(cwc > 0 && cwc < cwcDst);
                        pwszDst += cwc;
                        cwcDst  -= cwc;
                    }
                    if (iProcessorGroup >= 0)
                    {
                        cwc = _snwprintf(pwszDst, cwcDst, L" --group %d", iProcessorGroup);
                        assert(cwc > 0 && cwc < cwcDst);
                        pwszDst += cwc;
                        cwcDst  -= cwc;
                    }
                    *pwszDst = '\0';

                    /*
                     * Fill in the startup information.
                     */
                    memset(&StartupInfo, 0, sizeof(StartupInfo));
                    StartupInfo.cb = sizeof(StartupInfo);
                    GetStartupInfoW(&StartupInfo);
                    StartupInfo.dwFlags    &= ~STARTF_USESTDHANDLES;
                    StartupInfo.lpReserved2 = NULL;
                    StartupInfo.cbReserved2 = 0;

                    /*
                     * Flags and such.
                     */
                    fFlags = CREATE_SUSPENDED;
                    switch (process_priority)
                    {
                        case 1: fFlags |= CREATE_SUSPENDED | IDLE_PRIORITY_CLASS; break;
                        case 2: fFlags |= CREATE_SUSPENDED | BELOW_NORMAL_PRIORITY_CLASS; break;
                        case 3: fFlags |= CREATE_SUSPENDED | NORMAL_PRIORITY_CLASS; break;
                        case 4: fFlags |= CREATE_SUSPENDED | HIGH_PRIORITY_CLASS; break;
                        case 5: fFlags |= CREATE_SUSPENDED | REALTIME_PRIORITY_CLASS; break;
                    }

                    /*
                     * Create the worker process.
                     */
                    if (CreateProcessW(wszExecutable, wszCommandLine, NULL /*pProcSecAttr*/, NULL /*pThreadSecAttr*/,
                                       FALSE /*fInheritHandles*/, fFlags, NULL /*pwszzEnvironment*/,
                                       NULL /*pwszCwd*/, &StartupInfo, &ProcInfo))
                    {
                        char   szErrMsg[256];
                        BOOL   afReplace[3] = { TRUE, FALSE, FALSE };
                        HANDLE ahReplace[3] = { hWorkerPipe, INVALID_HANDLE_VALUE, INVALID_HANDLE_VALUE };
                        if (pWorker->pStdOut)
                        {
                            afReplace[1] = TRUE;
                            afReplace[2] = TRUE;
                            ahReplace[1] = pWorker->pStdOut->hPipeChild;
                            ahReplace[2] = pWorker->pStdErr->hPipeChild;
                        }

                        rc = nt_child_inject_standard_handles(ProcInfo.hProcess, afReplace, ahReplace, szErrMsg, sizeof(szErrMsg));
                        if (rc == 0)
                        {
                            BOOL fRet;
                            switch (process_priority)
                            {
                                case 1: fRet = SetThreadPriority(ProcInfo.hThread, THREAD_PRIORITY_IDLE); break;
                                case 2: fRet = SetThreadPriority(ProcInfo.hThread, THREAD_PRIORITY_BELOW_NORMAL); break;
                                case 3: fRet = SetThreadPriority(ProcInfo.hThread, THREAD_PRIORITY_NORMAL); break;
                                case 4: fRet = SetThreadPriority(ProcInfo.hThread, THREAD_PRIORITY_HIGHEST); break;
                                case 5: fRet = SetThreadPriority(ProcInfo.hThread, THREAD_PRIORITY_TIME_CRITICAL); break;
                                default: fRet = TRUE;
                            }
                            if (!fRet)
                                warnx(pCtx, "warning: failed to set kWorker thread priority: %u\n", GetLastError());

                            if (iProcessorGroup >= 0 && g_pfnSetThreadGroupAffinity)
                            {
                                GROUP_AFFINITY OldAff = { 0,                                                    0, 0, 0, 0 };
                                GROUP_AFFINITY NewAff = { 0 /* == all active apparently */, (WORD)iProcessorGroup, 0, 0, 0 };
                                if (!g_pfnSetThreadGroupAffinity(ProcInfo.hThread, &NewAff, &OldAff))
                                    warnx(pCtx, "warning: Failed to set processor group to %d: %u\n",
                                          iProcessorGroup, GetLastError());
                            }

                            /*
                             * Now, we just need to resume the thread.
                             */
                            if (ResumeThread(ProcInfo.hThread))
                            {
                                CloseHandle(hWorkerPipe);
                                CloseHandle(ProcInfo.hThread);
                                pWorker->pid      = ProcInfo.dwProcessId;
                                pWorker->hProcess = ProcInfo.hProcess;
                                if (cVerbosity > 0)
                                    warnx(pCtx, "created %d bit worker %d\n", pWorker->cBits, pWorker->pid);
                                return 0;
                            }

                            /*
                             * Failed, bail out.
                             */
                            rc = errx(pCtx, -3, "ResumeThread failed: %u", GetLastError());
                        }
                        else
                            rc = errx(pCtx, -3, "%s", szErrMsg);
                        TerminateProcess(ProcInfo.hProcess, 1234);
                        CloseHandle(ProcInfo.hThread);
                        CloseHandle(ProcInfo.hProcess);
                    }
                    else
                        rc = errx(pCtx, -2, "CreateProcessW failed: %u (exe=%S cmdline=%S)",
                                  GetLastError(), wszExecutable, wszCommandLine);
                    CloseHandle(pWorker->OverlappedRead.hEvent);
                    pWorker->OverlappedRead.hEvent = INVALID_HANDLE_VALUE;
                }
                else
                    rc = errx(pCtx, -1, "CreateEventW failed: %u", GetLastError());
                CloseHandle(pWorker->hPipe);
                pWorker->hPipe = INVALID_HANDLE_VALUE;
            }
            else
                rc = errx(pCtx, -1, "Opening named pipe failed: %u", GetLastError());
            CloseHandle(hWorkerPipe);
        }
        else
            rc = errx(pCtx, -1, "CreateNamedPipeW failed: %u", GetLastError());

#else
        /*
         * Create a socket pair.
         */
        int aiPair[2] = { -1, -1 };
        if (socketpair(AF_LOCAL, SOCK_STREAM, 0, aiPair) == 0)
        {
            pWorker->fdSocket = aiPair[1];

            rc = -1;
        }
        else
            rc = err(pCtx, -1, "socketpair");
#endif
    }
    else
        rc = errx(pCtx, -1, "KBUILD_BIN_PATH is too long");
    return rc;
}


/**
 * Selects an idle worker or spawns a new one.
 *
 * @returns Pointer to the selected worker instance.  NULL on error.
 * @param   pCtx                The command execution context.
 * @param   pWorker             The idle worker instance to respawn.
 *                              On failure this will be freed!
 * @param   cBitsWorker         The worker bitness - 64 or 32.
 */
static int kSubmitRespawnWorker(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, int cVerbosity)
{
    /*
     * Clean up after the old worker.
     */
#ifdef KBUILD_OS_WINDOWS
    DWORD   rcWait;

    /* Close the pipe handle first, breaking the pipe in case it's not already
       busted up.  Close the event semaphore too before waiting for the process. */
    if (pWorker->hPipe != INVALID_HANDLE_VALUE)
    {
        if (!CloseHandle(pWorker->hPipe))
            warnx(pCtx, "CloseHandle(pWorker->hPipe): %u", GetLastError());
        pWorker->hPipe = INVALID_HANDLE_VALUE;
    }

    if (!CloseHandle(pWorker->OverlappedRead.hEvent))
        warnx(pCtx, "CloseHandle(pWorker->OverlappedRead.hEvent): %u", GetLastError());
    pWorker->OverlappedRead.hEvent = INVALID_HANDLE_VALUE;

    if (pWorker->pStdOut)
        MkWinChildcareWorkerDrainPipes(NULL, pWorker->pStdOut, pWorker->pStdErr);

    /* It's probably shutdown already, if not give it 10 milliseconds before
       we terminate it forcefully. */
    rcWait = WaitForSingleObject(pWorker->hProcess, 10);
    if (rcWait != WAIT_OBJECT_0)
    {
        BOOL fRc = TerminateProcess(pWorker->hProcess, 127);

        if (pWorker->pStdOut)
            MkWinChildcareWorkerDrainPipes(NULL, pWorker->pStdOut, pWorker->pStdErr);

        rcWait = WaitForSingleObject(pWorker->hProcess, 100);
        if (rcWait != WAIT_OBJECT_0)
            warnx(pCtx, "WaitForSingleObject returns %u (and TerminateProcess %d)", rcWait, fRc);
    }

    if (pWorker->pStdOut)
        MkWinChildcareWorkerDrainPipes(NULL, pWorker->pStdOut, pWorker->pStdErr);

    if (!CloseHandle(pWorker->hProcess))
        warnx(pCtx, "CloseHandle(pWorker->hProcess): %u", GetLastError());
    pWorker->hProcess = INVALID_HANDLE_VALUE;

#else
    pid_t   pidWait;
    int     rc;

    if (pWorker->fdSocket != -1)
    {
        if (close(pWorker->fdSocket) != 0)
            warn(pCtx, "close(pWorker->fdSocket)");
        pWorker->fdSocket = -1;
    }

    kill(pWorker->pid, SIGTERM);
    pidWait = waitpid(pWorker->pid, &rc, 0);
    if (pidWait != pWorker->pid)
        warn(pCtx, "waitpid(pWorker->pid,,0)");
#endif

    /*
     * Unlink it from the hash table.
     */
    kSubmitPidHashRemove(pWorker);

    /*
     * Respawn it.
     */
    if (kSubmitSpawnWorker(pCtx, pWorker, cVerbosity) == 0)
    {
        /*
         * Insert it into the process ID hash table and idle list.
         */
        size_t idxHash = KWORKER_PID_HASH(pWorker->pid);
        pWorker->pNextPidHash = g_apPidHash[idxHash];
        g_apPidHash[idxHash] = pWorker;
        return 0;
    }

    kSubmitListUnlink(&g_IdleList, pWorker);
    free(pWorker);
    return -1;
}


/**
 * Selects an idle worker or spawns a new one.
 *
 * @returns Pointer to the selected worker instance.  NULL on error.
 * @param   cBitsWorker         The worker bitness - 64 or 32.
 */
static PWORKERINSTANCE kSubmitSelectWorkSpawnNewIfNecessary(PKMKBUILTINCTX pCtx, unsigned cBitsWorker, int cVerbosity)
{
    /*
     * Lookup up an idle worker.
     */
    PWORKERINSTANCE pWorker = g_IdleList.pHead;
    while (pWorker)
    {
        if (pWorker->cBits == cBitsWorker)
            return pWorker;
        pWorker = pWorker->pNext;
    }

    /*
     * Create a new worker instance.
     */
    pWorker = (PWORKERINSTANCE)xcalloc(sizeof(*pWorker));
    pWorker->cBits = cBitsWorker;
#if defined(CONFIG_NEW_WIN_CHILDREN) && defined(KBUILD_OS_WINDOWS)
    if (output_sync != OUTPUT_SYNC_NONE)
    {
        pWorker->pStdOut = MkWinChildcareCreateWorkerPipe(1, g_uWorkerSeqNo << 1);
        pWorker->pStdErr = MkWinChildcareCreateWorkerPipe(2, g_uWorkerSeqNo << 1);
    }
    if (   output_sync == OUTPUT_SYNC_NONE
        || (   pWorker->pStdOut != NULL
            && pWorker->pStdErr != NULL))
#endif
    {
        if (kSubmitSpawnWorker(pCtx, pWorker, cVerbosity) == 0)
        {
            /*
             * Insert it into the process ID hash table and idle list.
             */
            size_t idxHash = KWORKER_PID_HASH(pWorker->pid);
            pWorker->pNextPidHash = g_apPidHash[idxHash];
            g_apPidHash[idxHash] = pWorker;

            kSubmitListAppend(&g_IdleList, pWorker);
            return pWorker;
        }
    }
#if defined(CONFIG_NEW_WIN_CHILDREN) && defined(KBUILD_OS_WINDOWS)
    if (pWorker->pStdErr)
        MkWinChildcareDeleteWorkerPipe(pWorker->pStdErr);
    if (pWorker->pStdOut)
        MkWinChildcareDeleteWorkerPipe(pWorker->pStdOut);
#endif

    free(pWorker);
    return NULL;
}


/**
 * Composes a JOB mesage for a worker.
 *
 * @returns Pointer to the message.
 * @param   pszExecutable       The executable to run.
 * @param   papszArgs           The argument vector.
 * @param   papszEnvVars        The environment vector.
 * @param   pszCwd              The current directory.
 * @param   fWatcomBrainDamage  The wcc/wcc386 workaround.
 * @param   fNoPchCaching       Whether to disable precompiled header caching.
 * @param   pszSpecialEnv       Environment variable (name=value) subject to
 *                              special expansion in kWorker.  NULL if none.
 * @param   papszPostCmdArgs    The post command and it's arguments.
 * @param   cPostCmdArgs        Number of post command argument, including the
 *                              command.  Zero if no post command scheduled.
 * @param   pcbMsg              Where to return the message length.
 */
static void *kSubmitComposeJobMessage(const char *pszExecutable, char **papszArgs, char **papszEnvVars,
                                      const char *pszCwd, int fWatcomBrainDamage, int fNoPchCaching, const char *pszSpecialEnv,
                                      char **papszPostCmdArgs, uint32_t cPostCmdArgs, uint32_t *pcbMsg)
{
    size_t   cbTmp;
    size_t   cbSpecialEnv;
    uint32_t i;
    uint32_t cbMsg;
    uint32_t cArgs;
    uint32_t cEnvVars;
    uint8_t *pbMsg;
    uint8_t *pbCursor;

    /*
     * Adjust input.
     */
    if (!pszExecutable)
        pszExecutable = papszArgs[0];

    /*
     * Calculate the message length first.
     */
    cbMsg  = sizeof(cbMsg);
    cbMsg += sizeof("JOB");
    cbMsg += strlen(pszExecutable) + 1;
    cbMsg += strlen(pszCwd) + 1;

    cbMsg += sizeof(cArgs);
    for (i = 0; papszArgs[i] != NULL; i++)
        cbMsg += 1 + strlen(papszArgs[i]) + 1;
    cArgs  = i;

    cbMsg += sizeof(cArgs);
    for (i = 0; papszEnvVars[i] != NULL; i++)
        cbMsg += strlen(papszEnvVars[i]) + 1;
    cEnvVars = i;

    cbMsg += 1; /* fWatcomBrainDamage */
    cbMsg += 1; /* fNoPchCaching */

    cbSpecialEnv = pszSpecialEnv ? strchr(pszSpecialEnv, '=') - pszSpecialEnv : 0;
    cbMsg += cbSpecialEnv + 1;

    cbMsg += sizeof(cPostCmdArgs);
    for (i = 0; i < cPostCmdArgs; i++)
        cbMsg += strlen(papszPostCmdArgs[i]) + 1;

    /*
     * Compose the message.
     */
    pbMsg = pbCursor = xmalloc(cbMsg);

    /* header */
    memcpy(pbCursor, &cbMsg, sizeof(cbMsg));
    pbCursor += sizeof(cbMsg);
    memcpy(pbCursor, "JOB", sizeof("JOB"));
    pbCursor += sizeof("JOB");

    /* executable. */
    cbTmp = strlen(pszExecutable) + 1;
    memcpy(pbCursor, pszExecutable, cbTmp);
    pbCursor += cbTmp;

    /* cwd */
    cbTmp = strlen(pszCwd) + 1;
    memcpy(pbCursor, pszCwd, cbTmp);
    pbCursor += cbTmp;

    /* arguments */
    memcpy(pbCursor, &cArgs, sizeof(cArgs));
    pbCursor += sizeof(cArgs);
    for (i = 0; papszArgs[i] != NULL; i++)
    {
        *pbCursor++ = 0; /* Argument expansion flags (MSC, EMX). */
        cbTmp = strlen(papszArgs[i]) + 1;
        memcpy(pbCursor, papszArgs[i], cbTmp);
        pbCursor += cbTmp;
    }
    assert(i == cArgs);

    /* environment */
    memcpy(pbCursor, &cEnvVars, sizeof(cEnvVars));
    pbCursor += sizeof(cEnvVars);
    for (i = 0; papszEnvVars[i] != NULL; i++)
    {
        cbTmp = strlen(papszEnvVars[i]) + 1;
        memcpy(pbCursor, papszEnvVars[i], cbTmp);
        pbCursor += cbTmp;
    }
    assert(i == cEnvVars);

    /* flags */
    *pbCursor++ = fWatcomBrainDamage != 0;
    *pbCursor++ = fNoPchCaching != 0;

    /* Special environment variable name. */
    memcpy(pbCursor, pszSpecialEnv, cbSpecialEnv);
    pbCursor += cbSpecialEnv;
    *pbCursor++ = '\0';

    /* post command */
    memcpy(pbCursor, &cPostCmdArgs, sizeof(cPostCmdArgs));
    pbCursor += sizeof(cPostCmdArgs);
    for (i = 0; i < cPostCmdArgs; i++)
    {
        cbTmp = strlen(papszPostCmdArgs[i]) + 1;
        memcpy(pbCursor, papszPostCmdArgs[i], cbTmp);
        pbCursor += cbTmp;
    }
    assert(i == cPostCmdArgs);

    assert(pbCursor - pbMsg == (size_t)cbMsg);

    /*
     * Done.
     */
    *pcbMsg = cbMsg;
    return pbMsg;
}


/**
 * Sends the job message to the given worker, respawning the worker if
 * necessary.
 *
 * @returns 0 on success, non-zero on failure.
 *
 * @param   pCtx                The command execution context.
 * @param   pWorker             The work to send the request to.  The worker is
 *                              on the idle list.
 * @param   pvMsg               The message to send.
 * @param   cbMsg               The size of the message.
 * @param   fNoRespawning       Set if
 * @param   cVerbosity          The verbosity level.
 */
static int kSubmitSendJobMessage(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, void const *pvMsg, uint32_t cbMsg,
                                 int fNoRespawning, int cVerbosity)
{
    int cRetries;

    /*
     * Respawn the worker if it stopped by itself and we closed the pipe already.
     */
#ifdef KBUILD_OS_WINDOWS
    if (pWorker->hPipe == INVALID_HANDLE_VALUE)
#else
    if (pWorker->fdSocket == -1)
#endif
    {
        if (!fNoRespawning)
        {
            if (cVerbosity > 0)
                warnx(pCtx, "Respawning worker (#1)...\n");
            if (kSubmitRespawnWorker(pCtx, pWorker, cVerbosity) != 0)
                return 2;
        }

    }

    /*
     * Restart-on-broken-pipe loop. Necessary?
     */
    for (cRetries = !fNoRespawning ? 1 : 0; ; cRetries--)
    {
        /*
         * Try write the message.
         */
        uint32_t        cbLeft = cbMsg;
        uint8_t const  *pbLeft = (uint8_t const  *)pvMsg;
#ifdef KBUILD_OS_WINDOWS
        DWORD           dwErr;
        DWORD           cbWritten;
        while (WriteFile(pWorker->hPipe, pbLeft, cbLeft, &cbWritten, NULL /*pOverlapped*/))
        {
            assert(cbWritten <= cbLeft);
            cbLeft -= cbWritten;
            if (!cbLeft)
                return 0;

            /* This scenario shouldn't really ever happen. But just in case... */
            pbLeft += cbWritten;
        }
        dwErr = GetLastError();
        if (   (   dwErr != ERROR_BROKEN_PIPE
                && dwErr != ERROR_NO_DATA)
            || cRetries <= 0)
            return errx(pCtx, 1, "Error writing to worker: %u", dwErr);
#else
        ssize_t cbWritten
        while ((cbWritten = write(pWorker->fdSocket, pbLeft, cbLeft)) >= 0)
        {
            assert(cbWritten <= cbLeft);
            cbLeft -= cbWritten;
            if (!cbLeft)
                return 0;

            pbLeft += cbWritten;
        }
        if (  (   errno != EPIPE
               && errno != ENOTCONN
               && errno != ECONNRESET))
            || cRetries <= 0)
            return err(pCtx, 1, "Error writing to worker");
# error "later"
#endif

        /*
         * Broken connection. Try respawn the worker.
         */
        if (cVerbosity > 0)
            warnx(pCtx, "Respawning worker (#2)...\n");
        if (kSubmitRespawnWorker(pCtx, pWorker, cVerbosity) != 0)
            return 2;
    }
}


/**
 * Closes the connection on a worker that said it is going to exit now.
 *
 * This is a way of dealing with imperfect resource management in the worker, it
 * will monitor it a little and trigger a respawn when it looks bad.
 *
 * This function just closes the pipe / socket connection to the worker.  The
 * kSubmitSendJobMessage function will see this a trigger a respawn the next
 * time the worker is engaged.  This will usually mean there's a little delay in
 * which the process can terminate without us having to actively wait for it.
 *
 * @param   pCtx                The command execution context.
 * @param   pWorker             The worker instance.
 */
static void kSubmitCloseConnectOnExitingWorker(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker)
{
#ifdef KBUILD_OS_WINDOWS
    if (!CloseHandle(pWorker->hPipe))
        warnx(pCtx, "CloseHandle(pWorker->hPipe): %u", GetLastError());
    pWorker->hPipe = INVALID_HANDLE_VALUE;
#else
    if (close(pWorker->fdSocket) != 0)
        warn(pCtx, "close(pWorker->fdSocket)");
    pWorker->fdSocket = -1;
#endif
}


#ifdef KBUILD_OS_WINDOWS

/**
 * Handles read failure.
 *
 * @returns Exit code.
 * @param   pCtx                The command execution context.
 * @param   pWorker             The worker instance.
 * @param   dwErr               The error code.
 * @param   pszWhere            Where it failed.
 */
static int kSubmitWinReadFailed(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, DWORD dwErr, const char *pszWhere)
{
    DWORD dwExitCode;

    if (pWorker->cbResultRead == 0)
        errx(pCtx, 1, "%s/ReadFile failed: %u", pszWhere, dwErr);
    else
        errx(pCtx, 1, "%s/ReadFile failed: %u (read %u bytes)", pszWhere, dwErr, pWorker->cbResultRead);
    assert(dwErr != 0);

    /* Complete the result. */
    pWorker->Result.s.rcExit         = 127;
    pWorker->Result.s.bWorkerExiting = 1;
    pWorker->cbResultRead            = sizeof(pWorker->Result);

    if (GetExitCodeProcess(pWorker->hProcess, &dwExitCode))
    {
        if (dwExitCode != 0)
            pWorker->Result.s.rcExit = dwExitCode;
    }

    return dwErr != 0 ? (int)(dwErr & 0x7fffffff) : 0x7fffffff;

}


/**
 * Used by
 * @returns 0 if we got the whole result, -1 if I/O is pending, and windows last
 *          error on ReadFile failure.
 * @param   pCtx                The command execution context.
 * @param   pWorker             The worker instance.
 */
static int kSubmitReadMoreResultWin(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, const char *pszWhere)
{
    /*
     * Set up the result read, telling the sub_proc.c unit about it.
     */
    while (pWorker->cbResultRead < sizeof(pWorker->Result))
    {
        DWORD cbRead = 0;

        BOOL fRc = ResetEvent(pWorker->OverlappedRead.hEvent);
        assert(fRc); (void)fRc;

        pWorker->OverlappedRead.Offset     = 0;
        pWorker->OverlappedRead.OffsetHigh = 0;

        if (!ReadFile(pWorker->hPipe, &pWorker->Result.ab[pWorker->cbResultRead],
                     sizeof(pWorker->Result) - pWorker->cbResultRead,
                     &cbRead,
                     &pWorker->OverlappedRead))
        {
            DWORD dwErr = GetLastError();
            if (dwErr == ERROR_IO_PENDING)
                return -1;
            return kSubmitWinReadFailed(pCtx, pWorker, dwErr, pszWhere);
        }

        pWorker->cbResultRead += cbRead;
        assert(pWorker->cbResultRead <= sizeof(pWorker->Result));
    }
    return 0;
}

#endif /* KBUILD_OS_WINDOWS */


/**
 * Adds the given message to the history.
 *
 * @returns Pointer to old message, or NULL if no old msg to free.
 * @param   pWorker             The worker instance.
 * @param   pvMsg               The message.
 * @param   cbMsg               The message size.
 */
static void *kSubmitUpdateHistory(PWORKERINSTANCE pWorker, void *pvMsg, size_t cbMsg)
{
    unsigned iHistory = pWorker->iHistory % K_ELEMENTS(pWorker->aHistory);
    void    *pvRet;
    pWorker->iHistory++;
    pvRet = pWorker->aHistory[iHistory].pvMsg;
    pWorker->aHistory[iHistory].pvMsg = pvMsg;
    pWorker->aHistory[iHistory].cbMsg = cbMsg;
    return pvRet;
}

typedef struct HISTORYDUMPBUF
{
    char           *pszBuf;
    size_t          cbBuf;
    size_t          off;
    PKMKBUILTINCTX  pCtx;
} HISTORYDUMPBUF;


static void kSubmitDumpHistoryWrite(HISTORYDUMPBUF *pBuf, const char *pch, size_t cch)
{
    if (pBuf->off + cch >= pBuf->cbBuf)
    {
        size_t cbNew = pBuf->cbBuf ? pBuf->cbBuf * 2 : 65536;
        while (pBuf->off + cch >= cbNew)
            cbNew *= 2;
        pBuf->pszBuf = (char *)xrealloc(pBuf->pszBuf, cbNew);
        pBuf->cbBuf = cbNew;
    }

    memcpy(&pBuf->pszBuf[pBuf->off], pch, cch);
    pBuf->off += cch;
    pBuf->pszBuf[pBuf->off] = '\0';
}

static void kSubmitDumpHistoryPrintf(HISTORYDUMPBUF *pBuf, const char *pszFormat, ...)
{
    char szTmp[32];
    va_list va;
    va_start(va, pszFormat);
    for (;;)
    {
        const char *pszPct = strchr(pszFormat, '%');
        if (!pszPct)
        {
            kSubmitDumpHistoryWrite(pBuf, pszFormat, strlen(pszFormat));
            return;
        }
        if (pszPct != pszFormat)
        {
            kSubmitDumpHistoryWrite(pBuf, pszFormat, pszPct - pszFormat);
            pszFormat = pszPct;
        }
        pszFormat++;
        switch (*pszFormat++)
        {
            case 's':
            {
                const char * const psz = va_arg(va, const char *);
                size_t const       cch = strlen(psz);
                if (cch == 0 || memchr(psz, '\'', cch))
                {
                    kSubmitDumpHistoryWrite(pBuf, TUPLE("\"")); /** @todo what if there are '"' in the string? */
                    kSubmitDumpHistoryWrite(pBuf, psz, cch);
                    kSubmitDumpHistoryWrite(pBuf, TUPLE("\""));
                }
                else if (   !memchr(psz, ' ', cch)
                         && !memchr(psz, '\\', cch)
                         && !memchr(psz, '\t', cch)
                         && !memchr(psz, '\n', cch)
                         && !memchr(psz, '\r', cch)
                         && !memchr(psz, '&', cch)
                         && !memchr(psz, ';', cch)
                         && !memchr(psz, '|', cch))
                    kSubmitDumpHistoryWrite(pBuf, psz, cch);
                else
                {
                    kSubmitDumpHistoryWrite(pBuf, TUPLE("'"));
                    kSubmitDumpHistoryWrite(pBuf, psz, strlen(psz));
                    kSubmitDumpHistoryWrite(pBuf, TUPLE("'"));
                }
                break;
            }

            case 'd':
            {
                int iValue = va_arg(va, int);
                kSubmitDumpHistoryWrite(pBuf, szTmp, snprintf(szTmp, sizeof(szTmp), "%d", iValue));
                break;
            }

            case 'u':
            {
                unsigned uValue = va_arg(va, unsigned);
                kSubmitDumpHistoryWrite(pBuf, szTmp, snprintf(szTmp, sizeof(szTmp), "%u", uValue));
                break;
            }

            case '%':
                kSubmitDumpHistoryWrite(pBuf, "%s", 1);
                break;

            default:
                assert(0);
        }
    }
    va_end(va);
}

static void kSubmitDumpHistoryFlush(HISTORYDUMPBUF *pBuf)
{
    if (pBuf->off > 0)
        output_write_text(pBuf->pCtx->pOut, 1, pBuf->pszBuf, pBuf->off);
    pBuf->off = 0;
}

/**
 * Dumps the history for this worker to stderr in the given context.
 *
 * @param   pCtx                The command execution context. (Typically not a
 *                              real context.)
 * @param   pWorker             The worker instance.
 */
static void kSubmitDumpHistory(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker)
{
    HISTORYDUMPBUF  Buf      = { NULL, 0, 0, pCtx };
    int             iHistory = pWorker->iHistory;
    unsigned        cDumped  = 0;

    while (cDumped < K_ELEMENTS(pWorker->aHistory) && iHistory > 0)
    {
        unsigned const  idx    = (unsigned)--iHistory % K_ELEMENTS(pWorker->aHistory);
        const char     *pszMsg = (const char *)pWorker->aHistory[idx].pvMsg;
        ssize_t         cbMsg  = pWorker->aHistory[idx].cbMsg;
        const char     *pszExe;
        const char     *pszCwd;
        uint32_t        i;
        uint32_t        cArgs;
        const char     *pszArgs;
        size_t          cbArgs;
        uint32_t        cEnvVars;
        const char     *pszEnvVars;
        size_t          cbEnvVars;
        const char     *pszSpecialEnv;
        char            fNoPchCaching;
        char            fWatcomBrainDamage;
        uint32_t        cPostArgs;
        const char     *pszPostArgs;
        size_t          cbPostArgs;

        cDumped++;
        if (!pszMsg || !cbMsg)
            break;

#define SKIP_BYTES(a_cbSkip)    do { pszMsg += (a_cbSkip); cbMsg -= (a_cbSkip); } while (0)
#define SKIP_STR()              do { size_t const cbToSkip = strlen(pszMsg) + 1; SKIP_BYTES(cbToSkip); } while (0)
#define SKIP_STRING_ARRAY(a_cStrings, a_cbPreable) do { \
        for (i = 0; i < (a_cStrings) && cbMsg > 0; i++) { \
            size_t const cbToSkip = (a_cbPreable) + strlen(pszMsg + (a_cbPreable)) + 1; \
            SKIP_BYTES(cbToSkip); \
        } } while (0)

        /* Decode it: */
        SKIP_BYTES(sizeof(uint32_t) + sizeof("JOB"));
        pszExe = pszMsg;
        SKIP_STR();
        pszCwd = pszMsg;
        SKIP_STR();

        cArgs  = *(uint32_t *)pszMsg;
        SKIP_BYTES(sizeof(uint32_t));
        pszArgs = pszMsg;
        SKIP_STRING_ARRAY(cArgs, 1 /*fbFlags*/);
        cbArgs = pszMsg - pszArgs;

        cEnvVars = *(uint32_t *)pszMsg;
        SKIP_BYTES(sizeof(uint32_t));
        pszEnvVars = pszMsg;
        SKIP_STRING_ARRAY(cEnvVars, 0);
        cbEnvVars = pszMsg - pszEnvVars;

        fWatcomBrainDamage = pszMsg[0] != '\0';
        fNoPchCaching = pszMsg[1] != '\0';
        SKIP_BYTES(2);

        pszSpecialEnv = pszMsg;
        SKIP_STR();

        cPostArgs = *(uint32_t *)pszMsg;
        SKIP_BYTES(sizeof(uint32_t));
        pszPostArgs = pszMsg;
        SKIP_STRING_ARRAY(cPostArgs, 0);
        cbPostArgs = pszMsg - pszPostArgs;

        /* Produce parseable output: */
        kSubmitDumpHistoryPrintf(&Buf, "kWorker %u/%u:\n\tkSubmit", (long)pWorker->pid, iHistory, pszExe);
        if (fNoPchCaching)
            kSubmitDumpHistoryWrite(&Buf, TUPLE(" --no-pch-caching"));
        if (fWatcomBrainDamage)
            kSubmitDumpHistoryWrite(&Buf, TUPLE(" --watcom-brain-damage"));
        if (pszSpecialEnv)
            kSubmitDumpHistoryPrintf(&Buf, " --special-env %s", pszSpecialEnv);
        kSubmitDumpHistoryPrintf(&Buf, " --chdir %s \\\n", pszCwd);

        pszMsg = pszEnvVars;
        cbMsg  = cbEnvVars;
        for (i = 0; i < cEnvVars && cbMsg > 0; i++)
        {
            kSubmitDumpHistoryPrintf(&Buf, "\t--putenv %s \\\n", pszMsg);
            SKIP_STR();
        }

        if (cPostArgs > 0)
        {
            kSubmitDumpHistoryWrite(&Buf, TUPLE("\t--post-cmd "));
            pszMsg = pszPostArgs;
            cbMsg  = cbPostArgs;
            for (i = 0; i < cPostArgs && cbMsg > 0; i++)
            {
                kSubmitDumpHistoryPrintf(&Buf, " %s", pszMsg);
                SKIP_STR();
            }
            kSubmitDumpHistoryWrite(&Buf, TUPLE(" \\\n"));
        }
        kSubmitDumpHistoryWrite(&Buf, TUPLE("\t-- \\\n"));

        pszMsg = pszArgs;
        cbMsg  = cbArgs;
        for (i = 0; i < cArgs && cbMsg > 0; i++)
        {
            SKIP_BYTES(1);
            kSubmitDumpHistoryPrintf(&Buf, i + 1 < cArgs ? "\t%s \\\n" : "\t%s\n", pszMsg);
            SKIP_STR();
        }

#undef SKIP_BYTES
#undef SKIP_STR
#undef SKIP_STRING_ARRAY
    }

    kSubmitDumpHistoryFlush(&Buf);
    free(Buf.pszBuf);
}


/**
 * Marks the worker active.
 *
 * On windows this involves setting up the async result read and telling
 * sub_proc.c about the process.
 *
 * @returns Exit code.
 * @param   pCtx                The command execution context.
 * @param   pWorker             The worker instance to mark as active.
 * @param   cVerbosity          The verbosity level.
 * @param   pChild              The kmk child to associate the job with.
 * @param   pPidSpawned         If @a *pPidSpawned is non-zero if the child is
 *                              running, otherwise the worker is already done
 *                              and we've returned the exit code of the job.
 */
static int kSubmitMarkActive(PKMKBUILTINCTX pCtx, PWORKERINSTANCE pWorker, int cVerbosity, struct child *pChild, pid_t *pPidSpawned)
{
#ifdef KBUILD_OS_WINDOWS
    int rc;
#endif

    pWorker->cbResultRead = 0;

#ifdef KBUILD_OS_WINDOWS
    /*
     * Setup the async result read on windows.  If we're slow and the worker
     * very fast, this may actually get the result immediately.
     */
l_again:
    rc = kSubmitReadMoreResultWin(pCtx, pWorker, "kSubmitMarkActive");
    if (rc == -1)
    {
# ifndef CONFIG_NEW_WIN_CHILDREN
        if (process_kmk_register_submit(pWorker->OverlappedRead.hEvent, (intptr_t)pWorker, pPidSpawned) == 0)
        { /* likely */ }
        else
        {
            /* We need to do the waiting here because sub_proc.c has too much to do. */
            warnx(pCtx, "Too many processes for sub_proc.c to handle!");
            WaitForSingleObject(pWorker->OverlappedRead.hEvent, INFINITE);
            goto l_again;
        }
# else
        if (MkWinChildCreateSubmit((intptr_t)pWorker->OverlappedRead.hEvent, pWorker,
                                   pWorker->pStdOut, pWorker->pStdErr, pChild, pPidSpawned) == 0)
        { /* likely */ }
        else
        {
            /* We need to do the waiting here because sub_proc.c has too much to do. */
            warnx(pCtx, "MkWinChildCreateSubmit failed!");
            WaitForSingleObject(pWorker->OverlappedRead.hEvent, INFINITE);
            goto l_again;
        }
# endif
    }
    else
    {
        assert(rc == 0 || pWorker->Result.s.rcExit != 0);
        if (pWorker->Result.s.bWorkerExiting)
            kSubmitCloseConnectOnExitingWorker(pCtx, pWorker);
        if (pWorker->Result.s.rcExit && 1)
            kSubmitDumpHistory(pCtx, pWorker);
        *pPidSpawned = 0;
        return pWorker->Result.s.rcExit;
    }
#endif

    /*
     * Mark it busy and move it to the active instance.
     */
    pWorker->pBusyWith = pChild;
#ifndef KBUILD_OS_WINDOWS
    *pPidSpawned = pWorker->pid;
#endif

    kSubmitListUnlink(&g_IdleList, pWorker);
    kSubmitListAppend(&g_BusyList, pWorker);
    return 0;
}


#ifdef KBUILD_OS_WINDOWS

/**
 * Retrieve the worker child result.
 *
 * If incomplete, we restart the ReadFile operation like kSubmitMarkActive does.
 *
 * @returns 0 on success, -1 if ReadFile was restarted.
 * @param   pvUser              The worker instance.
 * @param   fBlock              if we're to block waiting for the result or not.
 * @param   prcExit             Where to return the exit code.
 * @param   piSigNo             Where to return the signal number.
 */
int kSubmitSubProcGetResult(intptr_t pvUser, int fBlock, int *prcExit, int *piSigNo)
{
    PWORKERINSTANCE pWorker = (PWORKERINSTANCE)pvUser;
    KMKBUILTINCTX   FakeCtx = { "kSubmit/GetResult", NULL };
    PKMKBUILTINCTX  pCtx = &FakeCtx;

    /*
     * Get the overlapped result.  There should be one since we're here
     * because of a satisfied WaitForMultipleObject.
     */
    DWORD cbRead = 0;
    if (GetOverlappedResult(pWorker->hPipe, &pWorker->OverlappedRead, &cbRead, fBlock ? TRUE : FALSE))
    {
        pWorker->cbResultRead += cbRead;
        assert(pWorker->cbResultRead <= sizeof(pWorker->Result));

        /* More to be read? */
        while (pWorker->cbResultRead < sizeof(pWorker->Result))
        {
            int rc = kSubmitReadMoreResultWin(pCtx, pWorker, "kSubmitSubProcGetResult/more");
            if (rc == -1)
                return -1;
            assert(rc == 0 || pWorker->Result.s.rcExit != 0);
        }
        assert(pWorker->cbResultRead == sizeof(pWorker->Result));
    }
    else
    {
        DWORD dwErr = GetLastError();
        if (dwErr == ERROR_IO_INCOMPLETE && !fBlock)
            return -1;
        kSubmitWinReadFailed(pCtx, pWorker, dwErr, "kSubmitSubProcGetResult/result");
    }

    /*
     * Okay, we've got a result.
     */
    *prcExit = pWorker->Result.s.rcExit;
    switch (pWorker->Result.s.rcExit)
    {
        default:                                *piSigNo = 0; break;
        case CONTROL_C_EXIT:                    *piSigNo = SIGINT; break;
        case STATUS_INTEGER_DIVIDE_BY_ZERO:     *piSigNo = SIGFPE; break;
        case STATUS_ACCESS_VIOLATION:           *piSigNo = SIGSEGV; break;
        case STATUS_PRIVILEGED_INSTRUCTION:
        case STATUS_ILLEGAL_INSTRUCTION:        *piSigNo = SIGILL; break;
    }
    if (pWorker->Result.s.rcExit && pWorker->fDebugDumpHistoryOnFailure)
        kSubmitDumpHistory(pCtx, pWorker);
    if (pWorker->Result.s.bWorkerExiting)
        kSubmitCloseConnectOnExitingWorker(pCtx, pWorker);

    return 0;
}


int kSubmitSubProcKill(intptr_t pvUser, int iSignal)
{
    return -1;
}


/**
 * Called by process_cleanup when it's done with the worker.
 *
 * @param   pvUser              The worker instance.
 */
void kSubmitSubProcCleanup(intptr_t pvUser)
{
    PWORKERINSTANCE pWorker = (PWORKERINSTANCE)pvUser;
    kSubmitListUnlink(&g_BusyList, pWorker);
    kSubmitListAppend(&g_IdleList, pWorker);
}

#endif /* KBUILD_OS_WINDOWS */


/**
 * atexit callback that trigger worker termination.
 */
static void kSubmitAtExitCallback(void)
{
    PWORKERINSTANCE pWorker;
    DWORD           msStartTick;
    DWORD           cKillRaids = 0;
    KMKBUILTINCTX   FakeCtx = { "kSubmit/atexit", NULL };
    PKMKBUILTINCTX  pCtx = &FakeCtx;

    /*
     * Tell all the workers to exit by breaking the connection.
     */
    for (pWorker = g_IdleList.pHead; pWorker != NULL; pWorker = pWorker->pNext)
        kSubmitCloseConnectOnExitingWorker(pCtx, pWorker);
    for (pWorker = g_BusyList.pHead; pWorker != NULL; pWorker = pWorker->pNext)
        kSubmitCloseConnectOnExitingWorker(pCtx, pWorker);

    /*
     * Wait a little while for them to stop.
     */
    Sleep(0);
    msStartTick = GetTickCount();
    for (;;)
    {
        /*
         * Collect handles of running processes.
         */
        PWORKERINSTANCE apWorkers[MAXIMUM_WAIT_OBJECTS];
        HANDLE          ahHandles[MAXIMUM_WAIT_OBJECTS];
        DWORD           cHandles = 0;

        for (pWorker = g_IdleList.pHead; pWorker != NULL; pWorker = pWorker->pNext)
            if (pWorker->hProcess != INVALID_HANDLE_VALUE)
            {
                if (cHandles < MAXIMUM_WAIT_OBJECTS)
                {
                    apWorkers[cHandles] = pWorker;
                    ahHandles[cHandles] = pWorker->hProcess;
                }
                cHandles++;
            }
        for (pWorker = g_BusyList.pHead; pWorker != NULL; pWorker = pWorker->pNext)
            if (pWorker->hProcess != INVALID_HANDLE_VALUE)
            {
                if (cHandles < MAXIMUM_WAIT_OBJECTS)
                {
                    apWorkers[cHandles] = pWorker;
                    ahHandles[cHandles] = pWorker->hProcess;
                }
                cHandles++;
            }
        if (cHandles == 0)
            return;

        /*
         * Wait for the processes.
         */
        for (;;)
        {
            DWORD cMsElapsed = GetTickCount() - msStartTick;
            DWORD dwWait = WaitForMultipleObjects(cHandles <= MAXIMUM_WAIT_OBJECTS ? cHandles : MAXIMUM_WAIT_OBJECTS,
                                                  ahHandles, FALSE /*bWaitAll*/,
                                                  cMsElapsed < 5000 ? 5000 - cMsElapsed + 16 : 16);
            if (   dwWait >= WAIT_OBJECT_0
                && dwWait <= WAIT_OBJECT_0 + MAXIMUM_WAIT_OBJECTS)
            {
                size_t idx = dwWait - WAIT_OBJECT_0;
                CloseHandle(apWorkers[idx]->hProcess);
                apWorkers[idx]->hProcess = INVALID_HANDLE_VALUE;

                if (cHandles <= MAXIMUM_WAIT_OBJECTS)
                {
                    /* Restart the wait with the worker removed, or quit if it was the last worker. */
                    cHandles--;
                    if (!cHandles)
                        return;
                    if (idx != cHandles)
                    {
                        apWorkers[idx] = apWorkers[cHandles];
                        ahHandles[idx] = ahHandles[cHandles];
                    }
                    continue;
                }
                /* else: Reconstruct the wait array so we get maximum coverage. */
            }
            else if (dwWait == WAIT_TIMEOUT)
            {
                /* Terminate the whole bunch. */
                cKillRaids++;
                if (cKillRaids == 1 && getenv("KMK_KSUBMIT_NO_KILL") == NULL)
                {
                    warnx(pCtx, "Killing %u lingering worker processe(s)!\n", cHandles);
                    for (pWorker = g_IdleList.pHead; pWorker != NULL; pWorker = pWorker->pNext)
                        if (pWorker->hProcess != INVALID_HANDLE_VALUE)
                            TerminateProcess(pWorker->hProcess, WAIT_TIMEOUT);
                    for (pWorker = g_BusyList.pHead; pWorker != NULL; pWorker = pWorker->pNext)
                        if (pWorker->hProcess != INVALID_HANDLE_VALUE)
                            TerminateProcess(pWorker->hProcess, WAIT_TIMEOUT);
                }
                else
                {
                    warnx(pCtx, "Giving up on the last %u worker processe(s). :-(\n", cHandles);
                    return;
                }
            }
            else
            {
                /* Some kind of wait error.  Could be a bad handle, check each and remove
                   bad ones as well as completed ones. */
                size_t idx;
                warnx(pCtx, "WaitForMultipleObjects unexpectedly returned %#u (err=%u)\n",
                      dwWait, GetLastError());
                for (idx = 0; idx < cHandles; idx++)
                {
                    dwWait = WaitForSingleObject(ahHandles[idx], 0 /*ms*/);
                    if (dwWait != WAIT_TIMEOUT)
                    {
                        CloseHandle(apWorkers[idx]->hProcess);
                        apWorkers[idx]->hProcess = INVALID_HANDLE_VALUE;
                    }
                }
            }
            break;
        } /* wait loop */
    } /* outer wait loop */
}


static int kmk_builtin_kSubmit_usage(PKMKBUILTINCTX pCtx, int fIsErr)
{
    kmk_builtin_ctx_printf(pCtx, fIsErr,
                           "usage: %s [-Z|--zap-env] [-E|--set <var=val>] [-U|--unset <var=val>]\n"
                           "           [-A|--append <var=val>] [-D|--prepend <var=val>]\n"
                           "           [-s|--special-env <var=val>] [-C|--chdir <dir>]\n"
                           "           [--wcc-brain-damage] [--no-pch-caching]\n"
                           "           [-3|--32-bit] [-6|--64-bit] [-v] [--debug-dump-history]\n"
                           "           [-P|--post-cmd <cmd> [args]] -- <program> [args]\n"
                           "   or: %s --help\n"
                           "   or: %s --version\n"
                           "\n"
                           "Options:\n"
                           "  -Z, --zap-env, -i, --ignore-environment\n"
                           "    Zaps the environment. Position dependent.\n"
                           "  -E, --set <var>=[value]\n"
                           "    Sets an environment variable putenv fashion. Position dependent.\n"
                           "  -U, --unset <var>\n"
                           "    Removes an environment variable. Position dependent.\n"
                           "  -A, --append <var>=<value>\n"
                           "    Appends the given value to the environment variable.\n"
                           "  -D,--prepend <var>=<value>\n"
                           "    Prepends the given value to the environment variable.\n"
                           "  -s,--special-env <var>=<value>\n"
                           "    Same as --set, but flags the variable for further expansion\n"
                           "    within kWorker. Replacements:\n"
                           "      @@PROCESSOR_GROUP@@   - The processor group number.\n"
                           "      @@AUTHENTICATION_ID@@ - The authentication ID from the process token.\n"
                           "      @@PID@@               - The kWorker process ID.\n"
                           "      @@@@                  - Escaped \"@@\".\n"
                           "      @@DEBUG_COUNTER@@     - An ever increasing counter (starts at zero).\n"
                           "  -C, --chdir <dir>\n"
                           "    Specifies the current directory for the program.  Relative paths\n"
                           "    are relative to the previous -C option.  Default is getcwd value.\n"
                           "  -3, --32-bit\n"
                           "    Selects a 32-bit kWorker process. Default: kmk bit count\n"
                           "  -6, --64-bit\n"
                           "    Selects a 64-bit kWorker process. Default: kmk bit count\n"
                           "  --wcc-brain-damage\n"
                           "    Works around wcc and wcc386 (Open Watcom) not following normal\n"
                           "    quoting conventions on Windows, OS/2, and DOS.\n"
                           "  --no-pch-caching\n"
                           "    Do not cache precompiled header files because they're being created.\n"
                           "  -v,--verbose\n"
                           "    More verbose execution.\n"
                           "  --debug-dump-history\n"
                           "    Dump the history as of the submitted command.  Handy for debugging\n"
                           "    trouble caused by a previous job.\n"
                           "  --debug-dump-history-on-failure, --no-debug-dump-history-on-failure\n"
                           "    Dump the history on failure.  Can also be enabled by a non-empty\n"
                           "    KMK_KSUBMIT_DUMP_HISTORY_ON_FAILURE variable (first invocation only).\n"
                           "  -P|--post-cmd <cmd> ...\n"
                           "    For running a built-in command on the output, specifying the command\n"
                           "    and all it's parameters.  Currently supported commands:\n"
                           "        kDepObj\n"
                           "  -V,--version\n"
                           "    Show the version number.\n"
                           "  -h,--help\n"
                           "    Show this usage information.\n"
                           "\n"
                           ,
                           pCtx->pszProgName, pCtx->pszProgName, pCtx->pszProgName);
    return 2;
}


int kmk_builtin_kSubmit(int argc, char **argv, char **envp, PKMKBUILTINCTX pCtx, struct child *pChild, pid_t *pPidSpawned)
{
#ifdef KBUILD_OS_WINDOWS
    static int      s_fInitialized      = 0;
#endif
    int             rcExit = 0;
    int             iArg;
    unsigned        cAllocatedEnvVars;
    unsigned        cEnvVars;
    char          **papszEnvVars;
    const char     *pszExecutable       = NULL;
    const char     *pszSpecialEnv       = NULL;
    int             iPostCmd            = argc;
    int             cPostCmdArgs        = 0;
    unsigned        cBitsWorker         = g_cArchBits;
    int             fWatcomBrainDamage  = 0;
    int             fNoPchCaching       = 0;
    int             fDebugDumpHistory   = 0;
    static int      s_fDebugDumpHistoryOnFailure = -1;
    int             fDebugDumpHistoryOnFailure = s_fDebugDumpHistoryOnFailure;
    int             cVerbosity          = 0;
    size_t const    cbCwdBuf            = GET_PATH_MAX;
    PATH_VAR(szCwd);

#ifdef KBUILD_OS_WINDOWS
    /*
     * First time thru we must perform some initializations.
     */
    if (s_fInitialized)
    { }
    else
    {
        MkWinChildInitCpuGroupAllocator(&g_SubmitProcessorGroupAllocator);
# if K_ARCH_BITS == 64
        MkWinChildInitCpuGroupAllocator(&g_SubmitProcessorGroupAllocator32);
# endif
        *(FARPROC *)&g_pfnSetThreadGroupAffinity = GetProcAddress(GetModuleHandleW(L"KERNEL32.DLL"), "SetThreadGroupAffinity");
        s_fInitialized = 1;
    }
#endif
    if (fDebugDumpHistoryOnFailure != -1)
    { /* likely */ }
    else
    {
        struct variable *pVar = lookup_variable(TUPLE("KMK_KSUBMIT_DUMP_HISTORY_ON_FAILURE"));
        fDebugDumpHistoryOnFailure = pVar && *pVar->value != '\0';
        s_fDebugDumpHistoryOnFailure = fDebugDumpHistoryOnFailure;
    }

    /*
     * Create default program environment.
     *
     * Note! We only clean up the environment on successful return, assuming
     *       make will stop after that.
     */
    if (getcwd_fs(szCwd, cbCwdBuf) != NULL)
    { /* likely */ }
    else
        return err(pCtx, 1, "getcwd_fs failed\n");

    /* The environment starts out in read-only mode and will be duplicated if modified. */
    cAllocatedEnvVars = 0;
    papszEnvVars = envp;
    cEnvVars = 0;
    while (papszEnvVars[cEnvVars] != NULL)
        cEnvVars++;

    /*
     * Parse the command line.
     */
    for (iArg = 1; iArg < argc; iArg++)
    {
        const char *pszArg = argv[iArg];
        if (*pszArg == '-')
        {
            char chOpt = *++pszArg;
            pszArg++;
            if (chOpt != '-')
            {
                if (chOpt != '\0')
                { /* likely */ }
                else
                {
                    errx(pCtx, 1, "Incomplete option: '-'");
                    return kmk_builtin_kSubmit_usage(pCtx, 1);
                }
            }
            else
            {
                /* '--' indicates where the bits to execute start. */
                if (*pszArg == '\0')
                {
                    iArg++;
                    break;
                }

                if (   strcmp(pszArg, "wcc-brain-damage") == 0
                    || strcmp(pszArg, "watcom-brain-damage") == 0)
                {
                    fWatcomBrainDamage = 1;
                    continue;
                }

                if (strcmp(pszArg, "no-pch-caching") == 0)
                {
                    fNoPchCaching = 1;
                    continue;
                }

                if (strcmp(pszArg, "debug-dump-history") == 0)
                {
                    fDebugDumpHistory = 1;
                    continue;
                }

                if (strcmp(pszArg, "debug-dump-history-on-failure") == 0)
                {
                    fDebugDumpHistoryOnFailure = 1;
                    continue;
                }

                if (strcmp(pszArg, "no-debug-dump-history-on-failure") == 0)
                {
                    fDebugDumpHistoryOnFailure = 0;
                    continue;
                }

                /* convert to short. */
                if (strcmp(pszArg, "help") == 0)
                    chOpt = 'h';
                else if (strcmp(pszArg, "version") == 0)
                    chOpt = 'V';
                else if (strcmp(pszArg, "set") == 0)
                    chOpt = 'E';
                else if (strcmp(pszArg, "append") == 0)
                    chOpt = 'A';
                else if (strcmp(pszArg, "prepend") == 0)
                    chOpt = 'D';
                else if (strcmp(pszArg, "unset") == 0)
                    chOpt = 'U';
                else if (   strcmp(pszArg, "zap-env") == 0
                         || strcmp(pszArg, "ignore-environment") == 0 /* GNU env compatibility. */ )
                    chOpt = 'Z';
                else if (strcmp(pszArg, "chdir") == 0)
                    chOpt = 'C';
                else if (strcmp(pszArg, "set-special") == 0)
                    chOpt = 's';
                else if (strcmp(pszArg, "post-cmd") == 0)
                    chOpt = 'P';
                else if (strcmp(pszArg, "32-bit") == 0)
                    chOpt = '3';
                else if (strcmp(pszArg, "64-bit") == 0)
                    chOpt = '6';
                else if (strcmp(pszArg, "verbose") == 0)
                    chOpt = 'v';
                else if (strcmp(pszArg, "executable") == 0)
                    chOpt = 'e';
                else
                {
                    errx(pCtx, 2, "Unknown option: '%s'", pszArg - 2);
                    return kmk_builtin_kSubmit_usage(pCtx, 1);
                }
                pszArg = "";
            }

            do
            {
                /* Get option value first, if the option takes one. */
                const char *pszValue = NULL;
                switch (chOpt)
                {
                    case 'A':
                    case 'C':
                    case 'E':
                    case 'U':
                    case 'D':
                    case 'e':
                    case 's':
                        if (*pszArg != '\0')
                            pszValue = pszArg + (*pszArg == ':' || *pszArg == '=');
                        else if (++iArg < argc)
                            pszValue = argv[iArg];
                        else
                        {
                            errx(pCtx, 1, "Option -%c requires a value!", chOpt);
                            return kmk_builtin_kSubmit_usage(pCtx, 1);
                        }
                        break;
                }

                switch (chOpt)
                {
                    case 'Z':
                    case 'i': /* GNU env compatibility. */
                        rcExit = kBuiltinOptEnvZap(pCtx, &papszEnvVars, &cEnvVars, &cAllocatedEnvVars, cVerbosity);
                        if (rcExit == 0)
                            break;
                        return rcExit;

                    case 'E':
                        rcExit = kBuiltinOptEnvSet(pCtx, &papszEnvVars, &cEnvVars, &cAllocatedEnvVars, cVerbosity, pszValue);
                        if (rcExit == 0)
                            break;
                        return rcExit;

                    case 'A':
                        rcExit = kBuiltinOptEnvAppend(pCtx, &papszEnvVars, &cEnvVars, &cAllocatedEnvVars, cVerbosity, pszValue);
                        if (rcExit == 0)
                            break;
                        return rcExit;

                    case 'D':
                        rcExit = kBuiltinOptEnvPrepend(pCtx, &papszEnvVars, &cEnvVars, &cAllocatedEnvVars, cVerbosity, pszValue);
                        if (rcExit == 0)
                            break;
                        return rcExit;

                    case 'U':
                        rcExit = kBuiltinOptEnvUnset(pCtx, &papszEnvVars, &cEnvVars, &cAllocatedEnvVars, cVerbosity, pszValue);
                        if (rcExit == 0)
                            break;
                        return rcExit;

                    case 'C':
                        rcExit = kBuiltinOptChDir(pCtx, szCwd, cbCwdBuf, pszValue);
                        if (rcExit == 0)
                            break;
                        return rcExit;

                    case 's':
                        if (pszSpecialEnv)
                            return errx(pCtx, 1, "The -s option can only be used once!");
                        pszSpecialEnv = pszValue;
                        rcExit = kBuiltinOptEnvSet(pCtx, &papszEnvVars, &cEnvVars, &cAllocatedEnvVars, cVerbosity, pszValue);
                        if (rcExit == 0)
                            break;
                        return rcExit;

                    case 'P':
                        if (cPostCmdArgs > 0)
                            return errx(pCtx, 1, "The -P option can only be used once!");
                        if (*pszArg != '\0')
                            return errx(pCtx, 1, "The cmd part of the -P needs to be a separate argument!");
                        iPostCmd = ++iArg;
                        if (iArg >= argc)
                            return errx(pCtx, 1, "The -P option requires a command following it!");
                        while (iArg < argc && strcmp(argv[iArg], "--") != 0)
                            iArg++;
                        cPostCmdArgs = iArg - iPostCmd;
                        iArg--;
                        break;

                    case '3':
                        cBitsWorker = 32;
                        break;

                    case '6':
                        cBitsWorker = 64;
                        break;

                    case 'e':
                        pszExecutable = pszValue;
                        break;

                    case 'v':
                        cVerbosity++;
                        break;

                    case 'h':
                        kmk_builtin_kSubmit_usage(pCtx, 0);
                        kBuiltinOptEnvCleanup(&papszEnvVars, cEnvVars, &cAllocatedEnvVars);
                        return 0;

                    case 'V':
                        kBuiltinOptEnvCleanup(&papszEnvVars, cEnvVars, &cAllocatedEnvVars);
                        return kbuild_version(argv[0]);
                }
            } while ((chOpt = *pszArg++) != '\0');
        }
        else
        {
            errx(pCtx, 1, "Unknown argument: '%s'", pszArg);
            return kmk_builtin_kSubmit_usage(pCtx, 1);
        }
    }

    /*
     * Check that we've got something to execute.
     */
    if (iArg < argc)
    {
        uint32_t        cbMsg;
        void           *pvMsg   = kSubmitComposeJobMessage(pszExecutable, &argv[iArg], papszEnvVars, szCwd,
                                                           fWatcomBrainDamage, fNoPchCaching, pszSpecialEnv,
                                                           &argv[iPostCmd], cPostCmdArgs, &cbMsg);
        PWORKERINSTANCE pWorker = kSubmitSelectWorkSpawnNewIfNecessary(pCtx, cBitsWorker, cVerbosity);
        if (pWorker)
        {
            /* Before we send off the job, we should dump pending output, since
               the kWorker process currently does not coordinate its output with
               the output.c mechanics. */
#ifdef CONFIG_NEW_WIN_CHILDREN
            if (pCtx->pOut && !pWorker->pStdOut)
#else
            if (pCtx->pOut)
#endif
                output_dump(pCtx->pOut);
            pWorker->fDebugDumpHistoryOnFailure = fDebugDumpHistoryOnFailure;
            rcExit = kSubmitSendJobMessage(pCtx, pWorker, pvMsg, cbMsg, 0 /*fNoRespawning*/, cVerbosity);
            if (rcExit == 0)
            {
                pvMsg = kSubmitUpdateHistory(pWorker, pvMsg, cbMsg);
                if (fDebugDumpHistory)
                    kSubmitDumpHistory(pCtx, pWorker);
                rcExit = kSubmitMarkActive(pCtx, pWorker, cVerbosity, pChild, pPidSpawned);
            }

            if (!g_fAtExitRegistered)
                if (atexit(kSubmitAtExitCallback) == 0)
                    g_fAtExitRegistered = 1;
        }
        else
            rcExit = 1;
        free(pvMsg);
    }
    else
    {
        errx(pCtx, 1, "Nothing to executed!");
        rcExit = kmk_builtin_kSubmit_usage(pCtx, 1);
    }

    kBuiltinOptEnvCleanup(&papszEnvVars, cEnvVars, &cAllocatedEnvVars);
    return rcExit;
}