Adding upstream version 1:0.1.9998svn3589+dfsg.upstream/1%0.1.9998svn3589+dfsg

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 2250 insertions, 0 deletions

diff --git a/src/kmk/incdep.c b/src/kmk/incdep.c
new file mode 100644
index 0000000..09de6ff
--- /dev/null
+++ b/src/kmk/incdep.c
@@ -0,0 +1,2250 @@
+#ifdef CONFIG_WITH_INCLUDEDEP
+/* $Id: incdep.c 3565 2022-05-24 20:40:24Z bird $ */
+/** @file
+ * incdep - Simple dependency files.
+ */
+
+/*
+ * Copyright (c) 2007-2010 knut st. osmundsen <bird-kBuild-spamx@anduin.net>
+ *
+ * This file is part of kBuild.
+ *
+ * kBuild is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * kBuild is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with kBuild.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+/*******************************************************************************
+*   Header Files                                                               *
+*******************************************************************************/
+#ifdef __OS2__
+# define INCL_BASE
+# define INCL_ERRORS
+#endif
+#ifdef KBUILD_OS_WINDOWS
+# ifdef KMK
+#  define INCDEP_USE_KFSCACHE
+# endif
+#endif
+
+#include "makeint.h"
+
+#if !defined(WINDOWS32) && !defined(__OS2__)
+# define HAVE_PTHREAD
+#endif
+
+#include <assert.h>
+
+#include <glob.h>
+
+#include "filedef.h"
+#include "dep.h"
+#include "job.h"
+#include "commands.h"
+#include "variable.h"
+#include "rule.h"
+#include "debug.h"
+#include "strcache2.h"
+
+#ifdef HAVE_FCNTL_H
+# include <fcntl.h>
+#else
+# include <sys/file.h>
+#endif
+
+#ifdef WINDOWS32
+# include <io.h>
+# include <process.h>
+# include <Windows.h>
+# define PARSE_IN_WORKER
+#endif
+
+#ifdef INCDEP_USE_KFSCACHE
+# include "nt/kFsCache.h"
+extern PKFSCACHE g_pFsCache; /* dir-nt-bird.c for now */
+#endif
+
+#ifdef __OS2__
+# include <os2.h>
+# include <sys/fmutex.h>
+#endif
+
+#ifdef HAVE_PTHREAD
+# include <pthread.h>
+#endif
+
+#ifdef __APPLE__
+# include <malloc/malloc.h>
+# define PARSE_IN_WORKER
+#endif
+
+#if defined(__gnu_linux__) || defined(__linux__)
+# define PARSE_IN_WORKER
+#endif
+
+
+/*******************************************************************************
+*   Structures and Typedefs                                                    *
+*******************************************************************************/
+struct incdep_variable_in_set
+{
+    struct incdep_variable_in_set *next;
+    /* the parameters */
+    struct strcache2_entry *name_entry;     /* dep strcache - WRONG */
+    const char *value;                      /* xmalloc'ed */
+    unsigned int value_length;
+    int duplicate_value;                    /* 0 */
+    enum variable_origin origin;
+    int recursive;
+    struct variable_set *set;
+    const floc *flocp;                      /* NILF */
+};
+
+struct incdep_variable_def
+{
+    struct incdep_variable_def *next;
+    /* the parameters */
+    const floc *flocp;                      /* NILF */
+    struct strcache2_entry *name_entry;     /* dep strcache - WRONG */
+    char *value;                            /* xmalloc'ed, free it */
+    unsigned int value_length;
+    enum variable_origin origin;
+    enum variable_flavor flavor;
+    int target_var;
+};
+
+struct incdep_recorded_file
+{
+    struct incdep_recorded_file *next;
+
+    /* the parameters */
+    struct strcache2_entry *filename_entry; /* dep strcache; converted to a nameseq record. */
+    struct dep *deps;                       /* All the names are dep strcache entries. */
+    const floc *flocp;                     /* NILF */
+};
+
+
+/* per dep file structure. */
+struct incdep
+{
+  struct incdep *next;
+  char *file_base;
+  char *file_end;
+
+  int worker_tid;
+#ifdef PARSE_IN_WORKER
+  unsigned int err_line_no;
+  const char *err_msg;
+
+  struct incdep_variable_in_set *recorded_variables_in_set_head;
+  struct incdep_variable_in_set *recorded_variables_in_set_tail;
+
+  struct incdep_variable_def *recorded_variable_defs_head;
+  struct incdep_variable_def *recorded_variable_defs_tail;
+
+  struct incdep_recorded_file *recorded_file_head;
+  struct incdep_recorded_file *recorded_file_tail;
+#endif
+#ifdef INCDEP_USE_KFSCACHE
+  /** Pointer to the fs cache object for this file (it exists and is a file). */
+  PKFSOBJ pFileObj;
+#else
+  char name[1];
+#endif
+};
+
+
+/*******************************************************************************
+*   Global Variables                                                           *
+*******************************************************************************/
+
+/* mutex protecting the globals and an associated condition/event. */
+#ifdef HAVE_PTHREAD
+static pthread_mutex_t incdep_mtx;
+static pthread_cond_t  incdep_cond_todo;
+static pthread_cond_t  incdep_cond_done;
+
+#elif defined (WINDOWS32)
+static CRITICAL_SECTION incdep_mtx;
+static HANDLE incdep_hev_todo;
+static HANDLE incdep_hev_done;
+static int volatile incdep_hev_todo_waiters;
+static int volatile incdep_hev_done_waiters;
+
+#elif defined (__OS2__)
+static _fmutex incdep_mtx;
+static HEV incdep_hev_todo;
+static HEV incdep_hev_done;
+static int volatile incdep_hev_todo_waiters;
+static int volatile incdep_hev_done_waiters;
+#endif
+
+/* flag indicating whether the threads, lock and event/condvars has
+   been initialized or not. */
+static int incdep_initialized;
+
+/* the list of files that needs reading. */
+static struct incdep * volatile incdep_head_todo;
+static struct incdep * volatile incdep_tail_todo;
+
+/* the number of files that are currently being read. */
+static int volatile incdep_num_reading;
+
+/* the list of files that have been read. */
+static struct incdep * volatile incdep_head_done;
+static struct incdep * volatile incdep_tail_done;
+
+
+/* The handles to the worker threads. */
+#ifdef HAVE_PTHREAD
+# define INCDEP_MAX_THREADS 1
+static pthread_t incdep_threads[INCDEP_MAX_THREADS];
+
+#elif defined (WINDOWS32)
+# define INCDEP_MAX_THREADS 2
+static HANDLE incdep_threads[INCDEP_MAX_THREADS];
+
+#elif defined (__OS2__)
+# define INCDEP_MAX_THREADS 2
+static TID incdep_threads[INCDEP_MAX_THREADS];
+#endif
+
+static struct alloccache incdep_rec_caches[INCDEP_MAX_THREADS];
+static struct alloccache incdep_dep_caches[INCDEP_MAX_THREADS];
+static struct strcache2 incdep_dep_strcaches[INCDEP_MAX_THREADS];
+static struct strcache2 incdep_var_strcaches[INCDEP_MAX_THREADS];
+static unsigned incdep_num_threads;
+
+/* flag indicating whether the worker threads should terminate or not. */
+static int volatile incdep_terminate;
+
+#ifdef __APPLE__
+/* malloc zone for the incdep threads. */
+static malloc_zone_t *incdep_zone;
+#endif
+
+
+/*******************************************************************************
+*   Internal Functions                                                         *
+*******************************************************************************/
+static void incdep_flush_it (floc *);
+static void eval_include_dep_file (struct incdep *, floc *);
+static void incdep_commit_recorded_file (const char *filename, struct dep *deps,
+                                         const floc *flocp);
+
+
+/* xmalloc wrapper.
+   For working around multithreaded performance problems found on Darwin,
+   Linux (glibc), and possibly other systems. */
+static void *
+incdep_xmalloc (struct incdep *cur, size_t size)
+{
+  void *ptr;
+
+#ifdef __APPLE__
+  if (cur && cur->worker_tid != -1)
+    {
+      ptr = malloc_zone_malloc (incdep_zone, size);
+      if (!ptr)
+        O (fatal, NILF, _("virtual memory exhausted"));
+    }
+  else
+    ptr = xmalloc (size);
+#else
+  ptr = xmalloc (size);
+#endif
+
+  (void)cur;
+  return ptr;
+}
+
+#if 0
+/* cmalloc wrapper */
+static void *
+incdep_xcalloc (struct incdep *cur, size_t size)
+{
+  void *ptr;
+
+#ifdef __APPLE__
+  if (cur && cur->worker_tid != -1)
+    ptr = malloc_zone_calloc (incdep_zone, size, 1);
+  else
+    ptr = calloc (size, 1);
+#else
+  ptr = calloc (size, 1);
+#endif
+  if (!ptr)
+    fatal (NILF, _("virtual memory exhausted"));
+
+  (void)cur;
+  return ptr;
+}
+#endif /* unused */
+
+/* free wrapper */
+static void
+incdep_xfree (struct incdep *cur, void *ptr)
+{
+  /* free() *must* work for the allocation hacks above because
+     of free_dep_chain. */
+  free (ptr);
+  (void)cur;
+}
+
+/* alloc a dep structure. These are allocated in bunches to save time. */
+struct dep *
+incdep_alloc_dep (struct incdep *cur)
+{
+  struct alloccache *cache;
+  if (cur->worker_tid != -1)
+    cache = &incdep_dep_caches[cur->worker_tid];
+  else
+    cache = &dep_cache;
+  return alloccache_calloc (cache);
+}
+
+/* duplicates the dependency list pointed to by srcdep. */
+static struct dep *
+incdep_dup_dep_list (struct incdep *cur, struct dep const *srcdep)
+{
+  struct alloccache *cache;
+  struct dep *retdep;
+  struct dep *dstdep;
+
+  if (cur->worker_tid != -1)
+    cache = &incdep_dep_caches[cur->worker_tid];
+  else
+    cache = &dep_cache;
+
+  if (srcdep)
+    {
+      retdep = dstdep = alloccache_alloc (cache);
+      for (;;)
+        {
+          dstdep->name = srcdep->name; /* string cached */
+          dstdep->includedep = srcdep->includedep;
+          srcdep = srcdep->next;
+          if (!srcdep)
+            {
+              dstdep->next = NULL;
+              break;
+            }
+          dstdep->next = alloccache_alloc (cache);
+          dstdep = dstdep->next;
+        }
+    }
+  else
+    retdep = NULL;
+  return retdep;
+}
+
+
+/* allocate a record. */
+static void *
+incdep_alloc_rec (struct incdep *cur)
+{
+  return alloccache_alloc (&incdep_rec_caches[cur->worker_tid]);
+}
+
+/* free a record. */
+static void
+incdep_free_rec (struct incdep *cur, void *rec)
+{
+  /*alloccache_free (&incdep_rec_caches[cur->worker_tid], rec); - doesn't work of course. */
+}
+
+
+/* grow a cache. */
+static void *
+incdep_cache_allocator (void *thrd, unsigned int size)
+{
+  (void)thrd;
+#ifdef __APPLE__
+  return malloc_zone_malloc (incdep_zone, size);
+#else
+  return xmalloc (size);
+#endif
+}
+
+/* term a cache. */
+static void
+incdep_cache_deallocator (void *thrd, void *ptr, unsigned int size)
+{
+  (void)thrd;
+  (void)size;
+  free (ptr);
+}
+
+/* acquires the lock */
+void
+incdep_lock(void)
+{
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  pthread_mutex_lock (&incdep_mtx);
+#elif defined (WINDOWS32)
+  EnterCriticalSection (&incdep_mtx);
+#elif defined (__OS2__)
+  _fmutex_request (&incdep_mtx, 0);
+#endif
+}
+
+/* releases the lock */
+void
+incdep_unlock(void)
+{
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  pthread_mutex_unlock (&incdep_mtx);
+#elif defined(WINDOWS32)
+  LeaveCriticalSection (&incdep_mtx);
+#elif defined(__OS2__)
+  _fmutex_release (&incdep_mtx);
+#endif
+}
+
+/* signals the main thread that there is stuff todo. caller owns the lock. */
+static void
+incdep_signal_done (void)
+{
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  pthread_cond_broadcast (&incdep_cond_done);
+#elif defined (WINDOWS32)
+  if (incdep_hev_done_waiters)
+    SetEvent (incdep_hev_done);
+#elif defined (__OS2__)
+  if (incdep_hev_done_waiters)
+    DosPostEventSem (incdep_hev_done);
+#endif
+}
+
+/* waits for a reader to finish reading. caller owns the lock. */
+static void
+incdep_wait_done (void)
+{
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  pthread_cond_wait (&incdep_cond_done, &incdep_mtx);
+
+#elif defined (WINDOWS32)
+  ResetEvent (incdep_hev_done);
+  incdep_hev_done_waiters++;
+  incdep_unlock ();
+  WaitForSingleObject (incdep_hev_done, INFINITE);
+  incdep_lock ();
+  incdep_hev_done_waiters--;
+
+#elif defined (__OS2__)
+  ULONG ulIgnore;
+  DosResetEventSem (incdep_hev_done, &ulIgnore);
+  incdep_hev_done_waiters++;
+  incdep_unlock ();
+  DosWaitEventSem (incdep_hev_done, SEM_INDEFINITE_WAIT);
+  incdep_lock ();
+  incdep_hev_done_waiters--;
+#endif
+}
+
+/* signals the worker threads. caller owns the lock. */
+static void
+incdep_signal_todo (void)
+{
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  pthread_cond_broadcast (&incdep_cond_todo);
+#elif defined(WINDOWS32)
+  if (incdep_hev_todo_waiters)
+    SetEvent (incdep_hev_todo);
+#elif defined(__OS2__)
+  if (incdep_hev_todo_waiters)
+    DosPostEventSem (incdep_hev_todo);
+#endif
+}
+
+/* waits for stuff to arrive in the todo list. caller owns the lock. */
+static void
+incdep_wait_todo (void)
+{
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  pthread_cond_wait (&incdep_cond_todo, &incdep_mtx);
+
+#elif defined (WINDOWS32)
+  ResetEvent (incdep_hev_todo);
+  incdep_hev_todo_waiters++;
+  incdep_unlock ();
+  WaitForSingleObject (incdep_hev_todo, INFINITE);
+  incdep_lock ();
+  incdep_hev_todo_waiters--;
+
+#elif defined (__OS2__)
+  ULONG ulIgnore;
+  DosResetEventSem (incdep_hev_todo, &ulIgnore);
+  incdep_hev_todo_waiters++;
+  incdep_unlock ();
+  DosWaitEventSem (incdep_hev_todo, SEM_INDEFINITE_WAIT);
+  incdep_lock ();
+  incdep_hev_todo_waiters--;
+#endif
+}
+
+/* Reads a dep file into memory. */
+static int
+incdep_read_file (struct incdep *cur, floc *f)
+{
+#ifdef INCDEP_USE_KFSCACHE
+  size_t const cbFile = (size_t)cur->pFileObj->Stats.st_size;
+
+  assert(cur->pFileObj->fHaveStats);
+  cur->file_base = incdep_xmalloc (cur, cbFile + 1);
+  if (cur->file_base)
+    {
+      if (kFsCacheFileSimpleOpenReadClose (g_pFsCache, cur->pFileObj, 0, cur->file_base, cbFile))
+        {
+          cur->file_end = cur->file_base + cbFile;
+          cur->file_base[cbFile] = '\0';
+          return 0;
+        }
+      incdep_xfree (cur, cur->file_base);
+    }
+  OSS (error, f, "%s/%s: error reading file", cur->pFileObj->pParent->Obj.pszName, cur->pFileObj->pszName);
+
+#else /* !INCDEP_USE_KFSCACHE */
+  int fd;
+  struct stat st;
+
+  errno = 0;
+# ifdef O_BINARY
+  fd = open (cur->name, O_RDONLY | O_BINARY, 0);
+# else
+  fd = open (cur->name, O_RDONLY, 0);
+# endif
+  if (fd < 0)
+    {
+      /* ignore non-existing dependency files. */
+      int err = errno;
+      if (err == ENOENT || stat (cur->name, &st) != 0)
+        return 1;
+      OSS (error, f, "%s: %s", cur->name, strerror (err));
+      return -1;
+    }
+# ifdef KBUILD_OS_WINDOWS /* fewer kernel calls */
+  if (!birdStatOnFdJustSize (fd, &st.st_size))
+# else
+  if (!fstat (fd, &st))
+# endif
+    {
+      cur->file_base = incdep_xmalloc (cur, st.st_size + 1);
+      if (read (fd, cur->file_base, st.st_size) == st.st_size)
+        {
+          close (fd);
+          cur->file_end = cur->file_base + st.st_size;
+          cur->file_base[st.st_size] = '\0';
+          return 0;
+        }
+
+      /* bail out */
+
+      OSS (error, f, "%s: read: %s", cur->name, strerror (errno));
+      incdep_xfree (cur, cur->file_base);
+    }
+  else
+    OSS (error, f, "%s: fstat: %s", cur->name, strerror (errno));
+
+  close (fd);
+#endif /* !INCDEP_USE_KFSCACHE */
+  cur->file_base = cur->file_end = NULL;
+  return -1;
+}
+
+/* Free the incdep structure. */
+static void
+incdep_freeit (struct incdep *cur)
+{
+#ifdef PARSE_IN_WORKER
+  assert (!cur->recorded_variables_in_set_head);
+  assert (!cur->recorded_variable_defs_head);
+  assert (!cur->recorded_file_head);
+#endif
+
+  incdep_xfree (cur, cur->file_base);
+#ifdef INCDEP_USE_KFSCACHE
+  /** @todo release object ref some day... */
+#endif
+  cur->next = NULL;
+  free (cur);
+}
+
+/* A worker thread. */
+void
+incdep_worker (int thrd)
+{
+  incdep_lock ();
+
+  while (!incdep_terminate)
+   {
+      /* get job from the todo list. */
+
+      struct incdep *cur = incdep_head_todo;
+      if (!cur)
+        {
+          incdep_wait_todo ();
+          continue;
+        }
+      if (cur->next)
+        incdep_head_todo = cur->next;
+      else
+        incdep_head_todo = incdep_tail_todo = NULL;
+      incdep_num_reading++;
+
+      /* read the file. */
+
+      incdep_unlock ();
+      cur->worker_tid = thrd;
+
+      incdep_read_file (cur, NILF);
+#ifdef PARSE_IN_WORKER
+      eval_include_dep_file (cur, NILF);
+#endif
+
+      cur->worker_tid = -1;
+      incdep_lock ();
+
+      /* insert finished job into the done list. */
+
+      incdep_num_reading--;
+      cur->next = NULL;
+      if (incdep_tail_done)
+        incdep_tail_done->next = cur;
+      else
+        incdep_head_done = cur;
+      incdep_tail_done = cur;
+
+      incdep_signal_done ();
+   }
+
+  incdep_unlock ();
+}
+
+/* Thread library specific thread functions wrapping incdep_wroker. */
+#ifdef HAVE_PTHREAD
+static void *
+incdep_worker_pthread (void *thrd)
+{
+  incdep_worker ((size_t)thrd);
+  return NULL;
+}
+
+#elif defined (WINDOWS32)
+static unsigned __stdcall
+incdep_worker_windows (void *thrd)
+{
+  incdep_worker ((size_t)thrd);
+  return 0;
+}
+
+#elif defined (__OS2__)
+static void
+incdep_worker_os2 (void *thrd)
+{
+  incdep_worker ((size_t)thrd);
+}
+#endif
+
+/* Checks if threads are enabled or not.
+
+   This is a special hack so that is possible to disable the threads when in a
+   debian fakeroot environment.  Thus, in addition to the KMK_THREADS_DISABLED
+   and KMK_THREADS_ENABLED environment variable check we also check for signs
+   of fakeroot.  */
+static int
+incdep_are_threads_enabled (void)
+{
+#if defined (CONFIG_WITHOUT_THREADS)
+  return 0;
+#endif
+
+  /* Generic overrides. */
+  if (getenv ("KMK_THREADS_DISABLED"))
+    {
+      O (message, 1, "Threads disabled (environment)");
+      return 0;
+    }
+  if (getenv ("KMK_THREADS_ENABLED"))
+    return 1;
+
+#if defined (__gnu_linux__) || defined (__linux__) || defined(__GLIBC__)
+  /* Try detect fakeroot. */
+  if (getenv ("FAKEROOTKEY")
+   || getenv ("FAKEROOTUID")
+   || getenv ("FAKEROOTGID")
+   || getenv ("FAKEROOTEUID")
+   || getenv ("FAKEROOTEGID")
+   || getenv ("FAKEROOTSUID")
+   || getenv ("FAKEROOTSGID")
+   || getenv ("FAKEROOTFUID")
+   || getenv ("FAKEROOTFGID")
+   || getenv ("FAKEROOTDONTTRYCHOWN")
+   || getenv ("FAKEROOT_FD_BASE")
+   || getenv ("FAKEROOT_DB_SEARCH_PATHS"))
+    {
+      O (message, 1, "Threads disabled (fakeroot)");
+      return 0;
+    }
+
+  /* LD_PRELOAD could indicate undetected debian fakeroot or some
+     other ingenius library which cannot deal correctly with threads. */
+  if (getenv ("LD_PRELOAD"))
+    {
+      O (message, 1, "Threads disabled (LD_PRELOAD)");
+      return 0;
+    }
+
+#elif defined(__APPLE__) \
+   || defined(__sun__) || defined(__SunOS__) || defined(__sun) || defined(__SunOS) \
+   || defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__) \
+   || defined(__HAIKU__)
+  /* No broken preload libraries known to be in common use on these platforms... */
+
+#elif defined(_MSC_VER) || defined(_WIN32) || defined(__OS2__)
+  /* No preload mess to care about. */
+
+#else
+# error "Add your self to the appropriate case above and send a patch to bird."
+#endif
+  return 1;
+}
+
+/* Creates the the worker threads. */
+static void
+incdep_init (floc *f)
+{
+  unsigned i;
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  int rc;
+  pthread_attr_t attr;
+
+#elif defined (WINDOWS32)
+  unsigned tid;
+  uintptr_t hThread;
+
+#elif defined (__OS2__)
+  int rc;
+  int tid;
+#endif
+  (void)f;
+
+  /* heap hacks */
+
+#ifdef __APPLE__
+  incdep_zone = malloc_create_zone (0, 0);
+  if (!incdep_zone)
+    incdep_zone = malloc_default_zone ();
+#endif
+
+
+  /* create the mutex and two condition variables / event objects. */
+
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+  rc = pthread_mutex_init (&incdep_mtx, NULL);
+  if (rc)
+    ON (fatal, f, _("pthread_mutex_init failed: err=%d"), rc);
+  rc = pthread_cond_init (&incdep_cond_todo, NULL);
+  if (rc)
+    ON (fatal, f, _("pthread_cond_init failed: err=%d"), rc);
+  rc = pthread_cond_init (&incdep_cond_done, NULL);
+  if (rc)
+    ON (fatal, f, _("pthread_cond_init failed: err=%d"), rc);
+
+#elif defined (WINDOWS32)
+  InitializeCriticalSection (&incdep_mtx);
+  incdep_hev_todo = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
+  if (!incdep_hev_todo)
+    ON (fatal, f, _("CreateEvent failed: err=%d"), GetLastError());
+  incdep_hev_done = CreateEvent (NULL, TRUE /*bManualReset*/, FALSE /*bInitialState*/, NULL);
+  if (!incdep_hev_done)
+    ON (fatal, f, _("CreateEvent failed: err=%d"), GetLastError());
+  incdep_hev_todo_waiters = 0;
+  incdep_hev_done_waiters = 0;
+
+#elif defined (__OS2__)
+  _fmutex_create (&incdep_mtx, 0);
+  rc = DosCreateEventSem (NULL, &incdep_hev_todo, 0, FALSE);
+  if (rc)
+    ON (fatal, f, _("DosCreateEventSem failed: rc=%d"), rc);
+  rc = DosCreateEventSem (NULL, &incdep_hev_done, 0, FALSE);
+  if (rc)
+    ON (fatal, f, _("DosCreateEventSem failed: rc=%d"), rc);
+  incdep_hev_todo_waiters = 0;
+  incdep_hev_done_waiters = 0;
+#endif
+
+  /* create the worker threads and associated per thread data. */
+
+  incdep_terminate = 0;
+  if (incdep_are_threads_enabled())
+    {
+      incdep_num_threads = sizeof (incdep_threads) / sizeof (incdep_threads[0]);
+      if (incdep_num_threads + 1 > job_slots)
+        incdep_num_threads = job_slots <= 1 ? 1 : job_slots - 1;
+      for (i = 0; i < incdep_num_threads; i++)
+        {
+          /* init caches */
+          unsigned rec_size = sizeof (struct incdep_variable_in_set);
+          if (rec_size < sizeof (struct incdep_variable_def))
+            rec_size = sizeof (struct incdep_variable_def);
+          if (rec_size < sizeof (struct incdep_recorded_file))
+            rec_size = sizeof (struct incdep_recorded_file);
+          alloccache_init (&incdep_rec_caches[i], rec_size, "incdep rec",
+                           incdep_cache_allocator, (void *)(size_t)i);
+          alloccache_init (&incdep_dep_caches[i], sizeof(struct dep), "incdep dep",
+                           incdep_cache_allocator, (void *)(size_t)i);
+          strcache2_init (&incdep_dep_strcaches[i],
+                          "incdep dep", /* name */
+                          65536,        /* hash size */
+                          0,            /* default segment size*/
+#ifdef HAVE_CASE_INSENSITIVE_FS
+                          1,            /* case insensitive */
+#else
+                          0,            /* case insensitive */
+#endif
+                          0);           /* thread safe */
+
+          strcache2_init (&incdep_var_strcaches[i],
+                          "incdep var", /* name */
+                          32768,        /* hash size */
+                          0,            /* default segment size*/
+                          0,            /* case insensitive */
+                          0);           /* thread safe */
+
+          /* create the thread. */
+#if defined (HAVE_PTHREAD) && !defined (CONFIG_WITHOUT_THREADS)
+          rc = pthread_attr_init (&attr);
+          if (rc)
+            ON (fatal, f, _("pthread_attr_init failed: err=%d"), rc);
+          /*rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); */
+          rc = pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
+          if (rc)
+            ON (fatal, f, _("pthread_attr_setdetachstate failed: err=%d"), rc);
+          rc = pthread_create (&incdep_threads[i], &attr,
+                               incdep_worker_pthread, (void *)(size_t)i);
+          if (rc)
+            ON (fatal, f, _("pthread_mutex_init failed: err=%d"), rc);
+          pthread_attr_destroy (&attr);
+
+#elif defined (WINDOWS32)
+          tid = 0;
+          hThread = _beginthreadex (NULL, 128*1024, incdep_worker_windows,
+                                    (void *)i, 0, &tid);
+          if (hThread == 0 || hThread == ~(uintptr_t)0)
+            ON (fatal, f, _("_beginthreadex failed: err=%d"), errno);
+          incdep_threads[i] = (HANDLE)hThread;
+
+#elif defined (__OS2__)
+          tid = _beginthread (incdep_worker_os2, NULL, 128*1024, (void *)i);
+          if (tid <= 0)
+            ON (fatal, f, _("_beginthread failed: err=%d"), errno);
+          incdep_threads[i] = tid;
+#endif
+        }
+    }
+  else
+    incdep_num_threads = 0;
+
+  incdep_initialized = 1;
+}
+
+/* Flushes outstanding work and terminates the worker threads.
+   This is called from snap_deps(). */
+void
+incdep_flush_and_term (void)
+{
+  unsigned i;
+
+  if (!incdep_initialized)
+    return;
+
+  /* flush any out standing work */
+
+  incdep_flush_it (NILF);
+
+  /* tell the threads to terminate */
+
+  incdep_lock ();
+  incdep_terminate = 1;
+  incdep_signal_todo ();
+  incdep_unlock ();
+
+  /* wait for the threads to quit */
+
+  for (i = 0; i < incdep_num_threads; i++)
+    {
+      /* more later? */
+
+      /* terminate or join up the allocation caches. */
+      alloccache_term (&incdep_rec_caches[i], incdep_cache_deallocator, (void *)(size_t)i);
+      alloccache_join (&dep_cache, &incdep_dep_caches[i]);
+      strcache2_term (&incdep_dep_strcaches[i]);
+      strcache2_term (&incdep_var_strcaches[i]);
+    }
+  incdep_num_threads = 0;
+
+  /* destroy the lock and condition variables / event objects. */
+
+  /* later */
+
+  incdep_initialized = 0;
+}
+
+#ifdef PARSE_IN_WORKER
+/* Flushes a strcache entry returning the actual string cache entry.
+   The input is freed! */
+static const char *
+incdep_flush_strcache_entry (struct strcache2_entry *entry)
+{
+  if (!entry->user)
+    entry->user = (void *) strcache2_add_hashed_file (&file_strcache,
+                                                      (const char *)(entry + 1),
+                                                      entry->length, entry->hash);
+  return (const char *)entry->user;
+}
+
+/* Flushes the recorded instructions. */
+static void
+incdep_flush_recorded_instructions (struct incdep *cur)
+{
+  struct incdep_variable_in_set *rec_vis;
+  struct incdep_variable_def *rec_vd;
+  struct incdep_recorded_file *rec_f;
+
+  /* Display saved error. */
+
+  if (cur->err_msg)
+#ifdef INCDEP_USE_KFSCACHE
+    OSSNS (error, NILF, "%s/%s(%d): %s", cur->pFileObj->pParent->Obj.pszName, cur->pFileObj->pszName,
+           cur->err_line_no, cur->err_msg);
+#else
+    OSNS (error,NILF, "%s(%d): %s", cur->name, cur->err_line_no, cur->err_msg);
+#endif
+
+
+  /* define_variable_in_set */
+
+  rec_vis = cur->recorded_variables_in_set_head;
+  cur->recorded_variables_in_set_head = cur->recorded_variables_in_set_tail = NULL;
+  if (rec_vis)
+    do
+      {
+        void *free_me = rec_vis;
+        unsigned int name_length = rec_vis->name_entry->length;
+        define_variable_in_set (incdep_flush_strcache_entry (rec_vis->name_entry),
+                                name_length,
+                                rec_vis->value,
+                                rec_vis->value_length,
+                                rec_vis->duplicate_value,
+                                rec_vis->origin,
+                                rec_vis->recursive,
+                                rec_vis->set,
+                                rec_vis->flocp);
+        rec_vis = rec_vis->next;
+        incdep_free_rec (cur, free_me);
+      }
+    while (rec_vis);
+
+  /* do_variable_definition */
+
+  rec_vd = cur->recorded_variable_defs_head;
+  cur->recorded_variable_defs_head = cur->recorded_variable_defs_tail = NULL;
+  if (rec_vd)
+    do
+      {
+        void *free_me = rec_vd;
+        do_variable_definition_2 (rec_vd->flocp,
+                                  incdep_flush_strcache_entry (rec_vd->name_entry),
+                                  rec_vd->value,
+                                  rec_vd->value_length,
+                                  0,
+                                  rec_vd->value,
+                                  rec_vd->origin,
+                                  rec_vd->flavor,
+                                  rec_vd->target_var);
+        rec_vd = rec_vd->next;
+        incdep_free_rec (cur, free_me);
+      }
+    while (rec_vd);
+
+  /* record_files */
+
+  rec_f = cur->recorded_file_head;
+  cur->recorded_file_head = cur->recorded_file_tail = NULL;
+  if (rec_f)
+    do
+      {
+        void *free_me = rec_f;
+        struct dep *dep;
+
+        for (dep = rec_f->deps; dep; dep = dep->next)
+          dep->name = incdep_flush_strcache_entry ((struct strcache2_entry *)dep->name);
+
+        incdep_commit_recorded_file (incdep_flush_strcache_entry (rec_f->filename_entry),
+                                     rec_f->deps,
+                                     rec_f->flocp);
+
+        rec_f = rec_f->next;
+        incdep_free_rec (cur, free_me);
+      }
+    while (rec_f);
+}
+#endif /* PARSE_IN_WORKER */
+
+/* Record / issue a warning about a misformed dep file. */
+static void
+incdep_warn (struct incdep *cur, unsigned int line_no, const char *msg)
+{
+  if (cur->worker_tid == -1)
+#ifdef INCDEP_USE_KFSCACHE
+    OSSNS (error,NILF, "%s/%s(%d): %s", cur->pFileObj->pParent->Obj.pszName, cur->pFileObj->pszName, line_no, msg);
+#else
+    OSNS (error, NILF, "%s(%d): %s", cur->name, line_no, msg);
+#endif
+#ifdef PARSE_IN_WORKER
+  else
+    {
+      cur->err_line_no = line_no;
+      cur->err_msg = msg;
+    }
+#endif
+}
+
+/* Dependency or file strcache allocation / recording. */
+static const char *
+incdep_dep_strcache (struct incdep *cur, const char *str, int len)
+{
+  const char *ret;
+  if (cur->worker_tid == -1)
+    {
+      /* Make sure the string is terminated before we hand it to
+         strcache_add_len so it does have to make a temporary copy
+         of it on the stack. */
+      char ch = str[len];
+      ((char *)str)[len] = '\0';
+      ret = strcache_add_len (str, len);
+      ((char *)str)[len] = ch;
+    }
+  else
+    {
+      /* Add it out the strcache of the thread. */
+      ret = strcache2_add (&incdep_dep_strcaches[cur->worker_tid], str, len);
+      ret = (const char *)strcache2_get_entry(&incdep_dep_strcaches[cur->worker_tid], ret);
+    }
+  return ret;
+}
+
+/* Variable name allocation / recording. */
+static const char *
+incdep_var_strcache (struct incdep *cur, const char *str, int len)
+{
+  const char *ret;
+  if (cur->worker_tid == -1)
+    {
+      /* XXX: we're leaking this memory now! This will be fixed later. */
+      ret = xmalloc (len + 1);
+      memcpy ((char *)ret, str, len);
+      ((char *)ret)[len] = '\0';
+    }
+  else
+    {
+      /* Add it out the strcache of the thread. */
+      ret = strcache2_add (&incdep_var_strcaches[cur->worker_tid], str, len);
+      ret = (const char *)strcache2_get_entry(&incdep_var_strcaches[cur->worker_tid], ret);
+    }
+  return ret;
+}
+
+/* Record / perform a variable definition in a set.
+   The NAME is in the string cache.
+   The VALUE is on the heap.
+   The DUPLICATE_VALUE is always 0. */
+static void
+incdep_record_variable_in_set (struct incdep *cur,
+                               const char *name, unsigned int name_length,
+                               const char *value,
+                               unsigned int value_length,
+                               int duplicate_value,
+                               enum variable_origin origin,
+                               int recursive,
+                               struct variable_set *set,
+                               const floc *flocp)
+{
+  assert (!duplicate_value);
+  if (cur->worker_tid == -1)
+    define_variable_in_set (name, name_length, value, value_length,
+                            duplicate_value, origin, recursive, set, flocp);
+#ifdef PARSE_IN_WORKER
+  else
+    {
+      struct incdep_variable_in_set *rec =
+        (struct incdep_variable_in_set *)incdep_alloc_rec (cur);
+      rec->name_entry = (struct strcache2_entry *)name;
+      rec->value = value;
+      rec->value_length = value_length;
+      rec->duplicate_value = duplicate_value;
+      rec->origin = origin;
+      rec->recursive = recursive;
+      rec->set = set;
+      rec->flocp = flocp;
+
+      rec->next = NULL;
+      if (cur->recorded_variables_in_set_tail)
+        cur->recorded_variables_in_set_tail->next = rec;
+      else
+        cur->recorded_variables_in_set_head = rec;
+      cur->recorded_variables_in_set_tail = rec;
+    }
+#endif
+}
+
+/* Record / perform a variable definition. The VALUE should be disposed of. */
+static void
+incdep_record_variable_def (struct incdep *cur,
+                            const floc *flocp,
+                            const char *name,
+                            unsigned int name_length,
+                            char *value,
+                            unsigned int value_length,
+                            enum variable_origin origin,
+                            enum variable_flavor flavor,
+                            int target_var)
+{
+  if (cur->worker_tid == -1)
+    do_variable_definition_2 (flocp, name, value, value_length, 0, value,
+                              origin, flavor, target_var);
+#ifdef PARSE_IN_WORKER
+  else
+    {
+      struct incdep_variable_def *rec =
+        (struct incdep_variable_def *)incdep_alloc_rec (cur);
+      rec->flocp = flocp;
+      rec->name_entry = (struct strcache2_entry *)name;
+      rec->value = value;
+      rec->value_length = value_length;
+      rec->origin = origin;
+      rec->flavor = flavor;
+      rec->target_var = target_var;
+
+      rec->next = NULL;
+      if (cur->recorded_variable_defs_tail)
+        cur->recorded_variable_defs_tail->next = rec;
+      else
+        cur->recorded_variable_defs_head = rec;
+      cur->recorded_variable_defs_tail = rec;
+    }
+#else
+  (void)name_length;
+#endif
+}
+
+/* Similar to record_files in read.c, only much much simpler. */
+static void
+incdep_commit_recorded_file (const char *filename, struct dep *deps,
+                             const floc *flocp)
+{
+  struct file *f;
+
+  /* Perform some validations. */
+  if (filename[0] == '.'
+      && (   streq(filename, ".POSIX")
+          || streq(filename, ".EXPORT_ALL_VARIABLES")
+          || streq(filename, ".INTERMEDIATE")
+          || streq(filename, ".LOW_RESOLUTION_TIME")
+          || streq(filename, ".NOTPARALLEL")
+          || streq(filename, ".ONESHELL")
+          || streq(filename, ".PHONY")
+          || streq(filename, ".PRECIOUS")
+          || streq(filename, ".SECONDARY")
+          || streq(filename, ".SECONDTARGETEXPANSION")
+          || streq(filename, ".SILENT")
+          || streq(filename, ".SHELLFLAGS")
+          || streq(filename, ".SUFFIXES")
+         )
+     )
+    {
+      OS (error, flocp, _("reserved filename '%s' used in dependency file, ignored"), filename);
+      return;
+    }
+
+  /* Lookup or create an entry in the database. */
+  f = enter_file (filename);
+  if (f->double_colon)
+    {
+      OS (error, flocp, _("dependency file '%s' has a double colon entry already, ignoring"), filename);
+      return;
+    }
+  f->is_target = 1;
+
+  /* Append dependencies. */
+  deps = enter_prereqs (deps, NULL);
+  if (deps)
+    {
+      struct dep *last = f->deps;
+      if (!last)
+        f->deps = deps;
+      else
+        {
+          while (last->next)
+            last = last->next;
+          last->next = deps;
+        }
+    }
+}
+
+/* Record a file.*/
+static void
+incdep_record_file (struct incdep *cur,
+                    const char *filename,
+                    struct dep *deps,
+                    const floc *flocp)
+{
+  if (cur->worker_tid == -1)
+    incdep_commit_recorded_file (filename, deps, flocp);
+#ifdef PARSE_IN_WORKER
+  else
+    {
+      struct incdep_recorded_file *rec =
+        (struct incdep_recorded_file *) incdep_alloc_rec (cur);
+
+      rec->filename_entry = (struct strcache2_entry *)filename;
+      rec->deps = deps;
+      rec->flocp = flocp;
+
+      rec->next = NULL;
+      if (cur->recorded_file_tail)
+        cur->recorded_file_tail->next = rec;
+      else
+        cur->recorded_file_head = rec;
+      cur->recorded_file_tail = rec;
+    }
+#endif
+}
+
+/* Counts slashes backwards from SLASH, stopping at START. */
+static size_t incdep_count_slashes_backwards(const char *slash, const char *start)
+{
+  size_t slashes = 1;
+  assert (*slash == '\\');
+  while ((uintptr_t)slash > (uintptr_t)start && slash[0 - slashes] == '\\')
+    slashes++;
+  return slashes;
+}
+
+/* Whitespace cannot be escaped at the end of a line, there has to be
+   some stuff following it other than a line continuation slash.
+
+   So, we look ahead and makes sure that there is something non-whitespaced
+   following this allegedly escaped whitespace.
+
+   This code ASSUMES the file content is zero terminated! */
+static int incdep_verify_escaped_whitespace(const char *ws)
+{
+  char ch;
+
+  assert(ws[-1] == '\\');
+  assert(ISBLANK((unsigned int)ws[0]));
+
+  /* If the character following the '\ ' sequence is not a whitespace,
+     another escape character or null terminator, we're good. */
+  ws += 2;
+  ch = *ws;
+  if (ch != '\\' && !ISSPACE((unsigned int)ch) && ch != '\0')
+    return 1;
+
+  /* Otherwise we'll have to parse forward till we hit the end of the
+     line/file or something. */
+  while ((ch = *ws++) != '\0')
+    {
+      if (ch == '\\')
+        {
+           /* escaped newline? */
+           ch = *ws;
+           if (ch == '\n')
+             ws++;
+           else  if (ch == '\r' && ws[1] == '\n')
+             ws += 2;
+           else
+             return 1;
+        }
+      else if (ISBLANK((unsigned int)ch))
+      { /* contine */ }
+      else if (!ISSPACE((unsigned int)ch))
+        return 1;
+      else
+        return 0; /* newline; all trailing whitespace will be ignored. */
+    }
+
+  return 0;
+}
+
+/* Unescapes the next filename and returns cached copy.
+
+   Modifies the input string that START points to.
+
+   When NEXTP is not NULL, ASSUME target filename and that END isn't entirely
+   accurate in case the filename ends with a trailing backslash.  There can be
+   more than one filename in a this case.  NEXTP will be set to the first
+   character after then filename.
+
+   When NEXTP is NULL, ASSUME exactly one dependency filename and that END is
+   accurately deliminating the string.
+   */
+static const char *
+incdep_unescape_and_cache_filename(struct incdep *curdep, char *start, const char *end,
+                                   int const is_dep, const char **nextp, unsigned int *linenop)
+{
+  unsigned const esc_mask = MAP_BLANK        /*  ' ' + '\t' */
+                          | MAP_COLON        /* ':' */
+                          | MAP_COMMENT      /* '#' */
+                          | MAP_EQUALS       /* '=' */
+                          | MAP_SEMI         /* ';' */
+                          | (  is_dep
+                             ? MAP_PIPE      /* '|' */
+                             : MAP_PERCENT); /* '%' */
+  unsigned const all_esc_mask = esc_mask | MAP_BLANK | MAP_NEWLINE;
+  unsigned const stop_mask = nextp ? MAP_BLANK | MAP_NEWLINE | (!is_dep ? MAP_COLON : 0) : 0;
+  char volatile *src;
+  char volatile *dst;
+
+  /*
+   * Skip forward to the first escaped character so we can avoid unnecessary shifting.
+   */
+#if 1
+  src = start;
+  dst = start;
+#elif 1
+  static const char s_szStop[] = "\n\r\t ";
+
+  src = memchr(start, '$', end - start);
+  dst = memchr(start, '\\', end - start);
+  if (src && ((uintptr_t)src < (uintptr_t)dst || dst == NULL))
+    dst = src;
+  else if (dst && ((uintptr_t)dst < (uintptr_t)src || src == NULL))
+    src = dst;
+  else
+    {
+      assert(src == NULL && dst == NULL);
+      if (nextp)
+        {
+           int i = sizeof(s_szStop);
+           while (i-- > 0)
+             {
+               char *stop = memchr(start, s_szStop[i], end - start);
+               if (stop)
+                 end = stop;
+             }
+            *nextp = end;
+        }
+      return incdep_dep_strcache (curdep, start, end - start);
+    }
+  if (nextp)
+    {
+      char *stop = src;
+      int i = sizeof(s_szStop);
+      while (i-- > 0)
+        {
+          char *stop2 = memchr(start, s_szStop[i], stop - start);
+          if (stop2)
+            stop = stop2;
+        }
+      if (stop != src)
+        {
+          *nextp = stop;
+          return incdep_dep_strcache (curdep, start, stop - start);
+        }
+    }
+#endif
+
+  /*
+   * Copy char-by-char, undoing escaping as we go along.
+   */
+  while ((uintptr_t)src < (uintptr_t)end)
+    {
+      const char ch = *src++;
+      if (ch != '\\' && ch != '$')
+        {
+          if (!STOP_SET (ch, stop_mask))
+            *dst++ = ch;
+          else
+            {
+              src--;
+              break;
+            }
+        }
+      else
+        {
+          char ch2 = *src++;  /* No bounds checking to handle "/dir/file\ : ..."  when end points at " :". */
+          if (ch == '$')
+            {
+              if (ch2 != '$') /* $$ -> $ - Ignores secondary expansion! */
+                  src--;
+              *dst++ = ch;
+            }
+          else
+            {
+              unsigned int ch2_map;
+
+              /* Eat all the slashes and see what's at the end of them as that's all
+                 that's relevant.  If there is an escapable char, we'll emit half of
+                 the slashes. */
+              size_t const max_slashes = src - start - 1;
+              size_t slashes = 1;
+              while (ch2 == '\\')
+                {
+                  slashes++;
+                  ch2 = *src++;
+                }
+
+              /* Is it escapable? */
+              ch2_map = stopchar_map[(unsigned char)ch2];
+              if (ch2_map & all_esc_mask)
+                {
+                  /* Non-whitespace is simple: Slash slashes, output or stop. */
+                  if (!(ch2_map & (MAP_BLANK | MAP_NEWLINE)))
+                    {
+                      assert(ch2_map & esc_mask);
+                      while (slashes >= 2)
+                        {
+                          *dst++ = '\\';
+                          slashes -= 2;
+                        }
+                      if (slashes || !(stop_mask & ch2_map))
+                        *dst++ = ch2;
+                      else
+                        {
+                          src--;
+                          break;
+                        }
+                    }
+                  /* Escaped blanks or newlines.
+
+                     We have to pretent that we've already replaced any escaped newlines
+                     and associated whitespace with a single space here.  We also have to
+                     pretend trailing whitespace doesn't exist when IS_DEP is non-zero.
+                     This makes for pretty interesting times... */
+                  else
+                    {
+                      char ch3;
+
+                      /* An Escaped blank is interesting because it is striped unconditionally
+                         at the end of a line, regardless of how many escaped newlines may
+                         following it.  We join the escaped newline handling if we fine one
+                         following us. */
+                      if (ch2_map & MAP_BLANK)
+                        {
+                          /* skip whitespace and check for escaped newline. */
+                          volatile char * const src_saved = src;
+                          while ((ch3 = *src) != '\0' && ISBLANK(ch3))
+                            src++;
+                          if (ch3 == '\\' && src[1] == '\n')
+                            src += 2; /* Escaped blank & newline joins into single space. */
+                          else if (ch3 == '\\' && src[1] == '\r' && src[2] == '\n')
+                            src += 3; /* -> Join the escaped newline code below on the next line. */
+                          else if (STOP_SET(ch3, stop_mask & MAP_NEWLINE))
+                            { /* last thing on the line, no blanks to escape. */
+                              while (slashes-- > 0)
+                                *dst++ = '\\';
+                              break;
+                            }
+                          else
+                            {
+                              src = src_saved;
+                              while (slashes >= 2)
+                                {
+                                  *dst++ = '\\';
+                                  slashes -= 2;
+                                }
+                              if (slashes)
+                                {
+                                  *dst++ = ch2;
+                                  continue;
+                                }
+                              assert (nextp || (uintptr_t)src >= (uintptr_t)end);
+                              break;
+                            }
+                        }
+                      /* Escaped newlines get special treatment as they an any adjacent whitespace
+                         gets reduced to a single space, including subsequent escaped newlines.
+                         In addition, if this is the final dependency/file and there is no
+                         significant new characters following this escaped newline, the replacement
+                         space will also be stripped and we won't have anything to escape, meaning
+                         that the slashes will remain as is.  Finally, none of this space stuff can
+                         be stop characters, unless of course a newline isn't escaped. */
+                      else
+                        {
+                          assert (ch2_map & MAP_NEWLINE);
+                          if (ch2 == '\r' && *src == '\n')
+                            src++;
+                        }
+
+                      /* common space/newline code */
+                      for (;;)
+                        {
+                          if (linenop)
+                            *linenop += 1;
+                          while ((uintptr_t)src < (uintptr_t)end && ISBLANK(*src))
+                            src++;
+                          if ((uintptr_t)src >= (uintptr_t)end)
+                            {
+                              ch3 = '\0';
+                              break;
+                            }
+                          ch3 = *src;
+                          if (ch3 != '\\')
+                            break;
+                          ch3 = src[1];
+                          if (ch3 == '\n')
+                            src += 2;
+                          else if (ch3 == '\r' && src[2] == '\n')
+                            src += 3;
+                          else
+                              break;
+                        }
+
+                      if (is_dep && STOP_SET(ch3, stop_mask | MAP_NUL))
+                        { /* last thing on the line, no blanks to escape. */
+                          while (slashes-- > 0)
+                            *dst++ = '\\';
+                          break;
+                        }
+                      while (slashes >= 2)
+                        {
+                          *dst++ = '\\';
+                          slashes -= 2;
+                        }
+                      if (slashes)
+                        *dst++ = ' ';
+                      else
+                        {
+                          assert (nextp || (uintptr_t)src >= (uintptr_t)end);
+                          break;
+                        }
+                    }
+                }
+              /* Just output the slash if non-escapable character: */
+              else
+                {
+                  while (slashes-- > 0)
+                    *dst++ = '\\';
+                  src--;
+                }
+            }
+        }
+    }
+
+  if (nextp)
+    *nextp = (const char *)src;
+  return incdep_dep_strcache(curdep, start, dst - start);
+}
+
+/* no nonsense dependency file including.
+
+   Because nobody wants bogus dependency files to break their incremental
+   builds with hard to comprehend error messages, this function does not
+   use the normal eval routine but does all the parsing itself. This isn't,
+   as much work as it sounds, because the necessary feature set is very
+   limited.
+
+   eval_include_dep_file groks:
+
+   define var
+   endef
+
+   var [|:|?|>]= value [\]
+
+   [\]
+   file: [deps] [\]
+
+   */
+static void
+eval_include_dep_file (struct incdep *curdep, floc *f)
+{
+  unsigned line_no = 1;
+  const char *file_end = curdep->file_end;
+  const char *cur = curdep->file_base;
+  const char *endp;
+
+  /* if no file data, just return immediately. */
+  if (!cur)
+    return;
+
+  /* now parse the file. */
+  while ((uintptr_t)cur < (uintptr_t)file_end)
+    {
+      /* skip empty lines */
+      while ((uintptr_t)cur < (uintptr_t)file_end && ISSPACE (*cur) && *cur != '\n')
+        ++cur;
+      if ((uintptr_t)cur >= (uintptr_t)file_end)
+        break;
+      if (*cur == '#')
+        {
+          cur = memchr (cur, '\n', file_end - cur);
+          if (!cur)
+            break;
+        }
+      if (*cur == '\\')
+        {
+          unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
+                           : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
+                           : (file_end - cur == 1) ? 1 : 0;
+           if (eol_len)
+             {
+               cur += eol_len;
+               line_no++;
+               continue;
+             }
+        }
+      if (*cur == '\n')
+        {
+          cur++;
+          line_no++;
+          continue;
+        }
+
+      /* define var
+         ...
+         endef */
+      if (strneq (cur, "define ", 7))
+        {
+          const char *var;
+          unsigned var_len;
+          const char *value_start;
+          const char *value_end;
+          char *value;
+          unsigned value_len;
+          int found_endef = 0;
+
+          /* extract the variable name. */
+          cur += 7;
+          while (ISBLANK (*cur))
+            ++cur;
+          value_start = endp = memchr (cur, '\n', file_end - cur);
+          if (!endp)
+              endp = cur;
+          while (endp > cur && ISSPACE (endp[-1]))
+            --endp;
+          var_len = endp - cur;
+          if (!var_len)
+          {
+              incdep_warn (curdep, line_no, "bogus define statement.");
+              break;
+          }
+          var = incdep_var_strcache (curdep, cur, var_len);
+
+          /* find the end of the variable. */
+          cur = value_end = value_start = value_start + 1;
+          ++line_no;
+          while ((uintptr_t)cur < (uintptr_t)file_end)
+            {
+              /* check for endef, don't bother with skipping leading spaces. */
+              if (   file_end - cur >= 5
+                  && strneq (cur, "endef", 5))
+                {
+                  endp = cur + 5;
+                  while ((uintptr_t)endp < (uintptr_t)file_end && ISSPACE (*endp) && *endp != '\n')
+                    endp++;
+                  if ((uintptr_t)endp >= (uintptr_t)file_end || *endp == '\n')
+                    {
+                      found_endef = 1;
+                      cur = (uintptr_t)endp >= (uintptr_t)file_end ? file_end : endp + 1;
+                      break;
+                    }
+                }
+
+              /* skip a line ahead. */
+              cur = value_end = memchr (cur, '\n', file_end - cur);
+              if (cur != NULL)
+                ++cur;
+              else
+                cur = value_end = file_end;
+              ++line_no;
+            }
+
+          if (!found_endef)
+            {
+              incdep_warn (curdep, line_no, "missing endef, dropping the rest of the file.");
+              break;
+            }
+          value_len = value_end - value_start;
+          if (memchr (value_start, '\0', value_len))
+            {
+              incdep_warn (curdep, line_no, "'\\0' in define, dropping the rest of the file.");
+              break;
+            }
+
+          /* make a copy of the value, converting \r\n to \n, and define it. */
+          value = incdep_xmalloc (curdep, value_len + 1);
+          endp = memchr (value_start, '\r', value_len);
+          if (endp)
+            {
+              const char *src = value_start;
+              char *dst = value;
+              for (;;)
+                {
+                  size_t len = endp - src;
+                  memcpy (dst, src, len);
+                  dst += len;
+                  src = endp;
+                  if ((uintptr_t)src + 1 < (uintptr_t)file_end && src[1] == '\n')
+                      src++; /* skip the '\r' */
+                  if (src >= value_end)
+                    break;
+                  endp = memchr (endp + 1, '\r', src - value_end);
+                  if (!endp)
+                    endp = value_end;
+                }
+              value_len = dst - value;
+            }
+          else
+            memcpy (value, value_start, value_len);
+          value [value_len] = '\0';
+
+          incdep_record_variable_in_set (curdep,
+                                         var, var_len, value, value_len,
+                                         0 /* don't duplicate */, o_file,
+                                         0 /* defines are recursive but this is faster */,
+                                         NULL /* global set */, f);
+        }
+
+      /* file: deps
+         OR
+         variable [:]= value */
+      else
+        {
+          const char *equalp;
+          const char *eol;
+
+          /* Look for a colon or an equal sign.  In the assignment case, we
+             require it to be on the same line as the variable name to simplify
+             the code.  Because of clang, we cannot make the same assumptions
+             with file dependencies.  So, start with the equal. */
+
+          assert (*cur != '\n');
+          eol = memchr (cur, '\n', file_end - cur);
+          if (!eol)
+            eol = file_end;
+          equalp = memchr (cur, '=', eol - cur);
+          if (equalp && equalp != cur && (ISSPACE(equalp[-1]) || equalp[-1] != '\\'))
+            {
+              /* An assignment of some sort. */
+              const char *var;
+              unsigned var_len;
+              const char *value_start;
+              const char *value_end;
+              char *value;
+              unsigned value_len;
+              unsigned multi_line = 0;
+              enum variable_flavor flavor;
+
+              /* figure the flavor first. */
+              flavor = f_recursive;
+              if (equalp > cur)
+                {
+                  if (equalp[-1] == ':')
+                    flavor = f_simple;
+                  else if (equalp[-1] == '?')
+                    flavor = f_conditional;
+                  else if (equalp[-1] == '+')
+                    flavor = f_append;
+                  else if (equalp[-1] == '>')
+                    flavor = f_prepend;
+                }
+
+              /* extract the variable name. */
+              endp = flavor == f_recursive ? equalp : equalp - 1;
+              while (endp > cur && ISBLANK (endp[-1]))
+                --endp;
+              var_len = endp - cur;
+              if (!var_len)
+                {
+                  incdep_warn (curdep, line_no, "empty variable. (includedep)");
+                  break;
+                }
+              if (   memchr (cur, '$', var_len)
+                  || memchr (cur, ' ', var_len)
+                  || memchr (cur, '\t', var_len))
+                {
+                  incdep_warn (curdep, line_no, "fancy variable name. (includedep)");
+                  break;
+                }
+              var = incdep_var_strcache (curdep, cur, var_len);
+
+              /* find the start of the value. */
+              cur = equalp + 1;
+              while ((uintptr_t)cur < (uintptr_t)file_end && ISBLANK (*cur))
+                cur++;
+              value_start = cur;
+
+              /* find the end of the value / line (this isn't 101% correct). */
+              value_end = cur;
+              while ((uintptr_t)cur < (uintptr_t)file_end)
+                {
+                  endp = value_end = memchr (cur, '\n', file_end - cur);
+                  if (!value_end)
+                    value_end = file_end;
+                  if (value_end - 1 >= cur && value_end[-1] == '\r')
+                    --value_end;
+                  if (value_end - 1 < cur || value_end[-1] != '\\')
+                    {
+                      cur = endp ? endp + 1 : file_end;
+                      break;
+                    }
+                  --value_end;
+                  if (value_end - 1 >= cur && value_end[-1] == '\\')
+                    {
+                      incdep_warn (curdep, line_no, "fancy escaping! (includedep)");
+                      cur = NULL;
+                      break;
+                    }
+                  if (!endp)
+                    {
+                      cur = file_end;
+                      break;
+                    }
+
+                  cur = endp + 1;
+                  ++multi_line;
+                  ++line_no;
+                }
+              if (!cur)
+                break;
+              ++line_no;
+
+              /* make a copy of the value, converting \r\n to \n, and define it. */
+              value_len = value_end - value_start;
+              value = incdep_xmalloc (curdep, value_len + 1);
+              if (!multi_line)
+                  memcpy (value, value_start, value_len);
+              else
+                {
+                  /* unescape it */
+                  const char *src = value_start;
+                  char *dst = value;
+                  while (src < value_end)
+                    {
+                      const char *nextp;
+
+                      endp = memchr (src, '\n', value_end - src);
+                      if (!endp)
+                        nextp = endp = value_end;
+                      else
+                        nextp = endp + 1;
+                      if (endp > src && endp[-1] == '\r')
+                        --endp;
+                      if (endp > src && endp[-1] == '\\')
+                        --endp;
+
+                      if (src != value_start)
+                        *dst++ = ' ';
+                      memcpy (dst, src, endp - src);
+                      dst += endp - src;
+                      src = nextp;
+                    }
+                  value_len = dst - value;
+                }
+              value [value_len] = '\0';
+
+              /* do the definition */
+              if (flavor == f_recursive
+               || (   flavor == f_simple
+                   && !memchr (value, '$', value_len)))
+                incdep_record_variable_in_set (curdep,
+                                               var, var_len, value, value_len,
+                                               0 /* don't duplicate */, o_file,
+                                               flavor == f_recursive /* recursive */,
+                                               NULL /* global set */, f);
+              else
+                incdep_record_variable_def (curdep,
+                                            f, var, var_len, value, value_len,
+                                            o_file, flavor, 0 /* not target var */);
+            }
+          else
+            {
+              /* Expecting: file: dependencies */
+
+              int unescape_filename = 0;
+              const char *filename;
+              const char *fnnext;
+              const char *fnend;
+              const char *colonp;
+              struct dep *deps = 0;
+              struct dep **nextdep = &deps;
+              struct dep *dep;
+
+
+              /* Locate the next file colon.  If it's not within the bounds of
+                 the current line, check that all new line chars are escaped. */
+
+              colonp = memchr (cur, ':', file_end - cur);
+              while (   colonp
+                     && (   (   colonp != cur
+                             && colonp[-1] == '\\'
+                             && incdep_count_slashes_backwards (&colonp[-1], cur) & 1)
+#ifdef HAVE_DOS_PATHS
+                         || (   colonp + 1 < file_end
+                             && (colonp[1] == '/' || colonp[1] == '\\')
+                             && colonp > cur
+                             && isalpha ((unsigned char)colonp[-1])
+                             && (   colonp == cur + 1
+                                 || ISBLANK ((unsigned char)colonp[-2])))
+#endif
+                        )
+                    )
+                  colonp = memchr (colonp + 1, ':', file_end - (colonp + 1));
+              if (!colonp)
+                {
+                  incdep_warn (curdep, line_no, "no colon.");
+                  break;
+                }
+
+              if ((uintptr_t)colonp < (uintptr_t)eol)
+                  unescape_filename = memchr (cur, '\\', colonp - cur) != NULL
+                                   || memchr (cur, '$', colonp - cur) != NULL;
+              else if (memchr (eol, '=', colonp - eol))
+                {
+                  incdep_warn (curdep, line_no, "multi line assignment / dependency confusion.");
+                  break;
+                }
+              else
+                {
+                  const char *sol = cur;
+                  do
+                    {
+                      char *eol2 = (char *)eol - 1;
+                      if ((uintptr_t)eol2 >= (uintptr_t)sol && *eol2 == '\r') /* DOS line endings. */
+                        eol2--;
+                      if ((uintptr_t)eol2 < (uintptr_t)sol || *eol2 != '\\')
+                          incdep_warn (curdep, line_no, "no colon.");
+                      else if (eol2 != sol && eol2[-1] == '\\')
+                          incdep_warn (curdep, line_no, "fancy EOL escape. (includedep)");
+                      else
+                        {
+                          line_no++;
+                          sol = eol + 1;
+                          eol = memchr (sol, '\n', colonp - sol);
+                          continue;
+                        }
+                      sol = NULL;
+                      break;
+                    }
+                  while (eol != NULL);
+                  if (!sol)
+                    break;
+                  unescape_filename = 1;
+                }
+
+              /* Extract the first filename after trimming and basic checks. */
+              fnend = colonp;
+              while ((uintptr_t)fnend > (uintptr_t)cur && ISBLANK (fnend[-1]))
+                --fnend;
+              if (cur == fnend)
+                {
+                  incdep_warn (curdep, line_no, "empty filename.");
+                  break;
+                }
+              fnnext = cur;
+              if (!unescape_filename)
+                {
+                  while (fnnext != fnend && !ISBLANK (*fnnext))
+                    fnnext++;
+                  filename = incdep_dep_strcache (curdep, cur, fnnext - cur);
+                }
+              else
+                filename = incdep_unescape_and_cache_filename (curdep, (char *)fnnext, fnend, 0, &fnnext, NULL);
+
+              /* parse any dependencies. */
+              cur = colonp + 1;
+              while ((uintptr_t)cur < (uintptr_t)file_end)
+                {
+                  const char *dep_file;
+
+                  /* skip blanks and count lines. */
+                  char ch = 0;
+                  while ((uintptr_t)cur < (uintptr_t)file_end && ISSPACE ((ch = *cur)) && ch != '\n')
+                    ++cur;
+                  if ((uintptr_t)cur >= (uintptr_t)file_end)
+                    break;
+                  if (ch == '\n')
+                    {
+                      cur++;
+                      line_no++;
+                      break;
+                    }
+
+                  /* continuation + eol? */
+                  if (ch == '\\')
+                    {
+                      unsigned eol_len = (file_end - cur > 1 && cur[1] == '\n') ? 2
+                                       : (file_end - cur > 2 && cur[1] == '\r' && cur[2] == '\n') ? 3
+                                       : (file_end - cur == 1) ? 1 : 0;
+                      if (eol_len)
+                        {
+                          cur += eol_len;
+                          line_no++;
+                          continue;
+                        }
+                    }
+
+                  /* find the end of the filename and cache it */
+                  dep_file = NULL;
+                  endp = cur;
+                  for (;;)
+                    if ((uintptr_t)endp < (uintptr_t)file_end)
+                      {
+                         ch = *endp;
+                         if (ch != '\\' && ch != '$' )
+                           {
+                             if (!ISSPACE (ch))
+                               endp++;
+                             else
+                               {
+                                 dep_file = incdep_dep_strcache(curdep, cur, endp - cur);
+                                 break;
+                               }
+                           }
+                         else
+                           {
+                             /* potential escape sequence, let the unescaper do the rest. */
+                             dep_file = incdep_unescape_and_cache_filename (curdep, (char *)cur, file_end, 1, &endp, &line_no);
+                             break;
+                           }
+                      }
+                    else
+                      {
+                        dep_file = incdep_dep_strcache(curdep, cur, endp - cur);
+                        break;
+                      }
+
+                  /* add it to the list. */
+                  *nextdep = dep = incdep_alloc_dep (curdep);
+                  dep->includedep = 1;
+                  dep->name = dep_file;
+                  nextdep = &dep->next;
+
+                  cur = endp;
+                }
+
+              /* enter the file with its dependencies. */
+              incdep_record_file (curdep, filename, deps, f);
+
+              /* More files? Record them with the same dependency list. */
+              if ((uintptr_t)fnnext < (uintptr_t)fnend)
+                for (;;)
+                  {
+                    const char *filename_prev = filename;
+                    while (fnnext != fnend && ISBLANK (*fnnext))
+                      fnnext++;
+                    if (fnnext == fnend)
+                      break;
+                    if (*fnnext == '\\')
+                      {
+                        if (fnnext[1] == '\n')
+                          {
+                            line_no++;
+                            fnnext += 2;
+                            continue;
+                          }
+                        if (fnnext[1] == '\r' && fnnext[2] == '\n')
+                          {
+                            line_no++;
+                            fnnext += 3;
+                            continue;
+                          }
+                      }
+
+                    if (!unescape_filename)
+                      {
+                        const char *fnstart = fnnext;
+                        while (fnnext != fnend && !ISBLANK (*fnnext))
+                          fnnext++;
+                        filename = incdep_dep_strcache (curdep, fnstart, fnnext - fnstart);
+                      }
+                    else
+                      filename = incdep_unescape_and_cache_filename (curdep, (char *)fnnext, fnend, 0, &fnnext, NULL);
+                    if (filename != filename_prev) /* clang optimization. */
+                      incdep_record_file (curdep, filename, incdep_dup_dep_list (curdep, deps), f);
+                  }
+            }
+        }
+    }
+
+  /* free the file data */
+  incdep_xfree (curdep, curdep->file_base);
+  curdep->file_base = curdep->file_end = NULL;
+}
+
+/* Flushes the incdep todo and done lists. */
+static void
+incdep_flush_it (floc *f)
+{
+  incdep_lock ();
+  for (;;)
+    {
+      struct incdep *cur = incdep_head_done;
+
+      /* if the done list is empty, grab a todo list entry. */
+      if (!cur && incdep_head_todo)
+        {
+          cur = incdep_head_todo;
+          if (cur->next)
+            incdep_head_todo = cur->next;
+          else
+            incdep_head_todo = incdep_tail_todo = NULL;
+          incdep_unlock ();
+
+          incdep_read_file (cur, f);
+          eval_include_dep_file (cur, f);
+          incdep_freeit (cur);
+
+          incdep_lock ();
+          continue;
+        }
+
+      /* if the todo list and done list are empty we're either done
+         or will have to wait for the thread(s) to finish. */
+      if (!cur && !incdep_num_reading)
+          break; /* done */
+      if (!cur)
+        {
+          while (!incdep_head_done)
+            incdep_wait_done ();
+          cur = incdep_head_done;
+        }
+
+      /* we grab the entire done list and work thru it. */
+      incdep_head_done = incdep_tail_done = NULL;
+      incdep_unlock ();
+
+      while (cur)
+        {
+          struct incdep *next = cur->next;
+#ifdef PARSE_IN_WORKER
+          incdep_flush_recorded_instructions (cur);
+#else
+          eval_include_dep_file (cur, f);
+#endif
+          incdep_freeit (cur);
+          cur = next;
+        }
+
+      incdep_lock ();
+    } /* outer loop */
+  incdep_unlock ();
+}
+
+
+/* splits up a list of file names and feeds it to eval_include_dep_file,
+   employing threads to try speed up the file reading. */
+void
+eval_include_dep (const char *names, floc *f, enum incdep_op op)
+{
+  struct incdep *head = 0;
+  struct incdep *tail = 0;
+  struct incdep *cur;
+  const char *names_iterator = names;
+  const char *name;
+  unsigned int name_len;
+
+  /* loop through NAMES, creating a todo list out of them. */
+
+  while ((name = find_next_token (&names_iterator, &name_len)) != 0)
+    {
+#ifdef INCDEP_USE_KFSCACHE
+       KFSLOOKUPERROR enmError;
+       PKFSOBJ pFileObj = kFsCacheLookupWithLengthA (g_pFsCache, name, name_len, &enmError);
+       if (!pFileObj)
+         continue;
+       if (pFileObj->bObjType != KFSOBJ_TYPE_FILE)
+         {
+           kFsCacheObjRelease (g_pFsCache, pFileObj);
+           continue;
+         }
+
+       cur = xmalloc (sizeof (*cur));            /* not incdep_xmalloc here */
+       cur->pFileObj = pFileObj;
+#else
+       cur = xmalloc (sizeof (*cur) + name_len); /* not incdep_xmalloc here */
+       memcpy (cur->name, name, name_len);
+       cur->name[name_len] = '\0';
+#endif
+
+       cur->file_base = cur->file_end = NULL;
+       cur->worker_tid = -1;
+#ifdef PARSE_IN_WORKER
+       cur->err_line_no = 0;
+       cur->err_msg = NULL;
+       cur->recorded_variables_in_set_head = NULL;
+       cur->recorded_variables_in_set_tail = NULL;
+       cur->recorded_variable_defs_head = NULL;
+       cur->recorded_variable_defs_tail = NULL;
+       cur->recorded_file_head = NULL;
+       cur->recorded_file_tail = NULL;
+#endif
+
+       cur->next = NULL;
+       if (tail)
+         tail->next = cur;
+       else
+         head = cur;
+       tail = cur;
+    }
+
+#ifdef ELECTRIC_HEAP
+  if (1)
+#else
+  if (op == incdep_read_it)
+#endif
+    {
+      /* work our way thru the files directly */
+
+      cur = head;
+      while (cur)
+        {
+          struct incdep *next = cur->next;
+          incdep_read_file (cur, f);
+          eval_include_dep_file (cur, f);
+          incdep_freeit (cur);
+          cur = next;
+        }
+    }
+  else
+    {
+      /* initialize the worker threads and related stuff the first time around. */
+
+      if (!incdep_initialized)
+        incdep_init (f);
+
+      /* queue the files and notify the worker threads. */
+
+      incdep_lock ();
+
+      if (incdep_tail_todo)
+        incdep_tail_todo->next = head;
+      else
+        incdep_head_todo = head;
+      incdep_tail_todo = tail;
+
+      incdep_signal_todo ();
+      incdep_unlock ();
+
+      /* flush the todo queue if we're requested to do so. */
+
+      if (op == incdep_flush)
+        incdep_flush_it (f);
+    }
+}
+
+#endif /* CONFIG_WITH_INCLUDEDEP */
+