summaryrefslogtreecommitdiffstats
path: root/kBuild/footer-pass2-fetches.kmk
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:21:29 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-11 08:21:29 +0000
commit29cd838eab01ed7110f3ccb2e8c6a35c8a31dbcc (patch)
tree63ef546b10a81d461e5cf5ed9e98a68cd7dee1aa /kBuild/footer-pass2-fetches.kmk
parentInitial commit. (diff)
downloadkbuild-29cd838eab01ed7110f3ccb2e8c6a35c8a31dbcc.tar.xz
kbuild-29cd838eab01ed7110f3ccb2e8c6a35c8a31dbcc.zip
Adding upstream version 1:0.1.9998svn3589+dfsg.upstream/1%0.1.9998svn3589+dfsg
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'kBuild/footer-pass2-fetches.kmk')
-rw-r--r--kBuild/footer-pass2-fetches.kmk434
1 files changed, 434 insertions, 0 deletions
diff --git a/kBuild/footer-pass2-fetches.kmk b/kBuild/footer-pass2-fetches.kmk
new file mode 100644
index 0000000..ffa75b7
--- /dev/null
+++ b/kBuild/footer-pass2-fetches.kmk
@@ -0,0 +1,434 @@
+# $Id: footer-pass2-fetches.kmk 3121 2017-10-31 10:58:59Z bird $
+## @file
+# kBuild - Footer - Target lists - Pass 2 - Fetches.
+#
+
+#
+# Copyright (c) 2004-2017 knut st. osmundsen <bird-kBuild-spam-xviiv@anduin.net>
+#
+# This file is part of kBuild.
+#
+# kBuild is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version source of the License, or
+# (at your option) any later version.
+#
+# kBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with kBuild; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+#
+#
+# As a special exception you are granted permission to include this file, via
+# the kmk include directive, as you wish without this in itself causing the
+# resulting makefile, program or whatever to be covered by the GPL license.
+# This exception does not however invalidate any other reasons why the makefile,
+# program, whatever should not be covered the GPL.
+#
+#
+
+
+## @page pg_fetches Fetching Tools, Sources and Similar.
+#
+# The targets listed in the the FETCHES target list have the following attributes:
+# SOURCES
+# INST
+# FETCHTOOL
+# FETCHFLAGS
+# FETCHDIR
+# UNPACKTOOL
+# UNPACKFLAGS
+#
+# As usual the target name is an alias for 'creating' the target. Other
+# aliases are:
+# pass_fetches
+# fetch
+# unfetch
+# download
+# unpack
+#
+# @remark
+#
+# This is a little bit complex because we must guarantee that if a source file
+# changes only sligtly we must refetch it and to a proper unpacking of it. It
+# is also a desire that fetched archives and unpacked files can be deleted to
+# save space.
+#
+# Thus, we must be able to cleanup what we've unpacked should any of the
+# sources be removed. We do this by maintaining a file listing the files
+# and directories that was unpacked. This operation is named 'unfetch'.
+#
+# We make use of the SIZE and MD5 attributes for each of the sources to
+# create a digest that is stored in the primary target file. Subsequent
+# runswill compare their digest with it to decide if a refetch is required.
+# When a refetch is found necessary, an 'unfetch' is performed first to
+# clean out old files and directores. Note even changes in source order
+# will cause a refetch due to the way the digest is constructed and
+# evaluated.
+#
+# By not depending directly on the archives (nor on any unpacked files)
+# but on a goal made up from the archive name, size and md5, we allow
+# the user to delete the archives. Naturally, this means we'll have to
+# check and fetch missing archives before attempting to unpack them.
+#
+# @remark
+#
+# This feature will *NOT* work correctly with vanilla GNU make becuase
+# it makes use of includedep to avoid too many unnecessary files.
+#
+# @todo
+# 0. Move the fetches out into a unit.
+# 1. Download corruption / continuation.
+# 2. It's quite possible that there is one too many indirect dependency now...
+#
+
+## generates the fetch rule
+define def_fetch_src_fetch_rule
+# Indirect goal for downloading something.
+.PRECIOUS: $(out)
+$(out) + $($(target)_$(srcname)_FETCH_2_OUTPUT) +| $($(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE) : \
+ | $($(target)_$(srcname)_FETCH_2_DEPORD)
+ %$$(call MSG_FETCH_DL,$(target),$(source),$(out))
+ @## @todo do fancy stuff like download continuation.
+ $$(QUIET)$$(RM) -f -- $(out)
+ $(cmds)
+ $$(QUIET)$(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out))
+
+
+
+# Intermediate goal for making sure the md5 and size matches. it will (re) fetch the archive if necessary.
+$(out).checked_$(md5)_$(size): $($(target)_$(srcname)_FETCH_2_DEPEND) | $($(target)_$(srcname)_FETCH_2_DEPORD)
+ %$$(call MSG_FETCH_CHK,$(target),$(source),$(out))
+ $$(QUIET)$$(RM) -f -- $$@
+ @# (re)fetch the file if it doesn't exist or if it doesn't matches the md5.
+ @## @todo do fancy stuff like download continuation.
+ $$(QUIET)( test -f $(out) && $(if $(md5),$$(MD5SUM_EXT) -b -C $(md5) $(out), true) ) \
+ || ( $$(RM_EXT) -f $(out) \
+ && $$(MAKE) $(out) -f $(MAKEFILE) --no-print-directory )
+ $$(QUIET2)$$(APPEND) $$@
+
+_TARGET_$(target)_FETCHED += $(out) $(out).checked_$(md5)_$(size)
+
+# Just a little precaution.
+.NOTPARALLEL: $(out) $(out).checked_$(md5)_$(size)
+
+endef # def_fetch_src_fetch_rule
+$(eval-opt-var def_fetch_src_fetch_rule)
+
+## generates the unpack rule
+define def_fetch_src_unpack_rule
+# This is the unpack rule. it has an order-only dependency on the download check.
+$(out) + $($(target)_$(srcname)_UNPACK_2_OUTPUT) +| $($(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE) : \
+ $($(target)_$(srcname)_UNPACK_2_DEPEND) \
+ | $($(target)_$(srcname)_UNPACK_2_DEPORD) \
+ $(archive).checked_$(md5)_$(size) \
+ $(dir $(out))
+ %$$(call MSG_FETCH_UP,$(target),$(archive),$(inst))
+ $$(QUIET)$$(RM) -f -- $(out)
+ $$(QUIET)$$(MKDIR) -p -- $(dir $(out))
+ @# if the source archive doesn't exist fetch it (may have been deleted to save space).
+ $$(QUIET)test -f $(archive) \
+ || ( $$(RM_EXT) -f $(archive).checked_$(md5)_$(size) \
+ && $$(MAKE) $(archive).checked_$(md5)_$(size) -f $(MAKEFILE) --no-print-directory )
+ $(cmds)
+ $$(QUIET2)$$(APPEND) $(out) $(notdir $(archive).checked_$(md5)_$(size))
+ $$(QUIET2)$$(APPEND) $(out) $(notdir $(out))
+
+$(eval _TARGET_$(target)_UNPACKED += $(out))
+_TARGET_$(target)_DIGEST := $(_TARGET_$(target)_DIGEST)-$(srcname)_$(md5)_$(size)
+
+.NOTPARALLEL: $(out)
+
+endef # def_fetch_src_unpack_rule
+$(eval-opt-var def_fetch_src_unpack_rule)
+
+## Processes a fetch source
+#
+define def_fetch_src
+#$ (warning dbg: def_fetch_src: source='$(source)' target='$(target)')
+
+# common
+local srcname := $(notdir $(source))
+local inst := $(firstword \
+ $($(target)_$(source)_INST)\
+ $($(target)_$(srcname)_INST)\
+ $($(source)_INST)\
+ $($(srcname)_INST)\
+ $($(target)_INST)\
+)
+ifneq ($(patsubst %/,ok,$(inst)),ok)
+$(error kBuild: Bad or missing INST property for source '$(source)' in target '$(target)': $(inst))
+endif
+## @todo Install-revamp: FIXME
+INSTARGET_$(target)_$(srcname) := $(inst)
+local fetchdir := $(firstword \
+ $($(target)_$(source)_FETCHDIR)\
+ $($(target)_$(srcname)_FETCHDIR)\
+ $($(source)_FETCHDIR)\
+ $($(srcname)_FETCHDIR)\
+ $($(target)_FETCHDIR)\
+ $(FETCHDIR)\
+ $(PATH_TARGET)\
+)
+local deps := \
+ $($(target)_$(source)_DEPS)\
+ $($(target)_$(srcname)_DEPS)\
+ $($(source)_DEPS)\
+ $($(srcname)_DEPS)\
+ $($(target)_DEPS)
+local orderdeps := \
+ $($(target)_$(source)_ORDERDEPS)\
+ $($(target)_$(srcname)_ORDERDEPS)\
+ $($(source)_ORDERDEPS)\
+ $($(srcname)_ORDERDEPS)\
+ $($(target)_ORDERDEPS)
+local md5 := $(firstword \
+ $($(target)_$(source)_MD5)\
+ $($(target)_$(srcname)_MD5)\
+ $($(source)_MD5)\
+ $($(srcname)_MD5)\
+ $($(target)_MD5)\
+)
+local size := $(firstword \
+ $($(target)_$(source)_SIZE)\
+ $($(target)_$(srcname)_SIZE)\
+ $($(source)_SIZE)\
+ $($(srcname)_SIZE)\
+ $($(target)_SIZE)\
+)
+clean_files += \
+ $($(target)_$(source)_CLEAN)\
+ $($(target)_$(srcname)_CLEAN)\
+ $($(source)_CLEAN)\
+ $($(srcname)_CLEAN)
+local dep := # not legal for fetch and unpack tools
+
+
+#
+# The fetching.
+#
+local out := $(fetchdir)/$(srcname)
+local archive := $(out)
+$(target)_$(srcname)_1_TARGET = $(TARGET_$(target)_$(srcname))
+$(call KB_FN_ASSIGN_DEPRECATED,TARGET_$(target)_$(srcname),$(TARGET_$(target)_$(srcname)),TARGET_$(target)_$(srcname))
+
+local dirdep := $(call DIRDEP,$(fetchdir))
+local tool := $(firstword \
+ $($(target)_$(source)_FETCHTOOL)\
+ $($(target)_$(srcname)_FETCHTOOL)\
+ $($(target)_$(source)_TOOL)\
+ $($(target)_$(srcname)_TOOL)\
+ $($(source)_FETCHTOOL)\
+ $($(srcname)_FETCHTOOL)\
+ $($(source)_TOOL)\
+ $($(srcname)_TOOL)\
+ $($(target)_FETCHTOOL)\
+ $($(target)_TOOL)\
+ )
+local flags :=\
+ $(TOOL_$(tool)_FETCHFLAGS)\
+ $(FETCHFLAGS)\
+ $($(target)_FETCHFLAGS)\
+ $($(srcname)_FETCHFLAGS)\
+ $($(source)_FETCHFLAGS)\
+ $($(target)_$(srcname)_FETCHFLAGS)\
+ $($(target)_$(source)_FETCHFLAGS)
+
+#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) fetchdir=$(fetchdir) md5=$(md5) size=$(size))
+
+ifndef TOOL_$(tool)_FETCH_CMDS
+$(warning kBuild: tools: \
+ 1 $($(target)_$(source)_FETCHTOOL)\
+ 2 $($(target)_$(srcname)_FETCHTOOL)\
+ 3 $($(target)_$(source)_TOOL)\
+ 4 $($(target)_$(srcname)_TOOL)\
+ 5 $($(source)_FETCHTOOL)\
+ 6 $($(srcname)_FETCHTOOL)\
+ 7 $($(source)_TOOL)\
+ 8 $($(srcname)_TOOL)\
+ 9 $($(target)_FETCHTOOL)\
+ 10 $($(target)_TOOL) )
+$(error kBuild: TOOL_$(tool)_FETCH_CMDS is not defined. source=$(source) target=$(target) )
+endif
+
+# call the tool
+local cmds := $(TOOL_$(tool)_FETCH_CMDS)
+$(target)_$(srcname)_FETCH_2_OUTPUT := $(TOOL_$(tool)_FETCH_OUTPUT)
+$(target)_$(srcname)_FETCH_2_OUTPUT_MAYBE := $(TOOL_$(tool)_FETCH_OUTPUT_MAYBE)
+$(target)_$(srcname)_FETCH_2_DEPEND := $(TOOL_$(tool)_FETCH_DEPEND) $(deps)
+$(target)_$(srcname)_FETCH_2_DEPORD := $(TOOL_$(tool)_FETCH_DEPORD) $(dirdep) $(orderdeps)
+
+# generate the fetch rule.
+$(eval $(def_fetch_src_fetch_rule))
+
+
+#
+# The unpacking / installing.
+#
+local out := $(inst)_kBuild_$(target)_$(srcname)_unpacked.lst
+local dirdep := $(call DIRDEP,$(inst))
+local tool := $(firstword \
+ $($(target)_$(source)_UNPACKTOOL)\
+ $($(target)_$(srcname)_UNPACKTOOL)\
+ $($(target)_$(source)_TOOL)\
+ $($(target)_$(srcname)_TOOL)\
+ $($(source)_UNPACKTOOL)\
+ $($(srcname)_UNPACKTOOL)\
+ $($(source)_TOOL)\
+ $($(srcname)_TOOL)\
+ $($(target)_UNPACKTOOL)\
+ $($(target)_TOOL) \
+ )
+ifeq ($(tool),)
+local tool := $(toupper $(subst .,,$(suffix $(subst tar.,TAR,$(srcname)))))
+$(evalval def_tools_include)
+endif
+local flags :=\
+ $(TOOL_$(tool)_UNPACKFLAGS)\
+ $(UNPACKFLAGS)\
+ $($(target)_UNPACKFLAGS)\
+ $($(srcname)_UNPACKFLAGS)\
+ $($(source)_UNPACKFLAGS)\
+ $($(target)_$(srcname)_UNPACKFLAGS)\
+ $($(target)_$(source)_UNPACKFLAGS)
+
+#$ (warning dbg: target=$(target) source=$(source) $(srcname)=$(srcname) tool=$(tool) out=$(out) flags=$(flags) dirdep=$(dirdep) inst=$(inst) md5=$(md5) size=$(size))
+ifndef TOOL_$(tool)_UNPACK_CMDS
+$(warning kBuild: tools: \
+ 1 $($(target)_$(source)_UNPACKTOOL)\
+ 2 $($(target)_$(srcname)_UNPACKTOOL)\
+ 3 $($(target)_$(source)_TOOL)\
+ 4 $($(target)_$(srcname)_TOOL)\
+ 5 $($(source)_UNPACKTOOL)\
+ 6 $($(srcname)_UNPACKTOOL)\
+ 7 $($(source)_TOOL)\
+ 8 $($(srcname)_TOOL)\
+ 9 $($(target)_UNPACKTOOL)\
+ 10 $($(target)_TOOL) \
+ 11 $(toupper $(subst tar.,TAR,$(ext $(srcname)))) \
+ )
+$(error kBuild: TOOL_$(tool)_UNPACK_CMDS is not defined. source=$(source) target=$(target) )
+endif
+
+# call the tool
+local cmds := $(TOOL_$(tool)_UNPACK_CMDS)
+$(target)_$(srcname)_UNPACK_2_OUTPUT := $(TOOL_$(tool)_UNPACK_OUTPUT)
+$(target)_$(srcname)_UNPACK_2_OUTPUT_MAYBE := $(TOOL_$(tool)_UNPACK_OUTPUT_MAYBE)
+$(target)_$(srcname)_UNPACK_2_DEPEND := $(TOOL_$(tool)_UNPACK_DEPEND) $(deps)
+$(target)_$(srcname)_UNPACK_2_DEPORD := $(TOOL_$(tool)_UNPACK_DEPORD) $(dirdep) $(orderdeps)
+
+# generate the fetch rule.
+$(eval $(def_fetch_src_unpack_rule))
+
+_DIRS += $(inst) $(fetchdir)
+
+endef # def_fetch_src
+$(eval-opt-var def_fetch_src)
+
+
+##
+# Define the target level rules for a fetch.
+# @param target
+# @param out
+# @param inst
+# @param _TARGET_$(target)_UNPACKED
+# @param _TARGET_$(target)_DIGEST
+# @param bld_trg
+# @param bld_trg_arch
+define def_fetch_rules
+
+$(out).lst: $(_TARGET_$(target)_UNPACKED) | $(call DIRDEP,$(inst))
+ %$$(call MSG_FETCH_OK,$(target))
+ $$(QUIET)$$(RM) -f -- $$@ $$@.tmp
+ $$(QUIET2)$$(APPEND) $$@.tmp '$(notdir $(out))'
+ $$(QUIET)$(if $(_TARGET_$(target)_UNPACKED),$$(CAT_EXT) $(_TARGET_$(target)_UNPACKED) >> $$@.tmp)
+ $$(QUIET)$$(MV) -f -- $$@.tmp $$@
+
+$(out)_unfetched:
+ %$$(call MSG_UNFETCH,$(target))
+ $$(QUIET)$$(RM) -f -- $$(addprefix $(inst),$$(shell $$(CAT_EXT) $(out).lst 2> /dev/null | $$(SED) -e '/\/$$$$/d'))
+ $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@) \
+ $$(addprefix $(inst),$$(sort $$(dir $$(shell $$(CAT_EXT) $(out).lst 2> /dev/null))))
+ $$(QUIET)$$(RM) -f -- $(out).lst $(out)
+ $$(QUIET)$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@)
+
+$(out): $(comp-vars _TARGET_$(target)_DIGEST_PREV,_TARGET_$(target)_DIGEST,FORCE) | $(call DIRDEP,$(inst))
+ $$(QUIET)$$(RM) -f -- $$@
+ %$$(if $$(_TARGET_$(target)_DIGEST),$$(if $$(eq $$(file-size $(out).lst),-1)\
+ ,$$(call MSG_REFETCH,$(target)),$$(call MSG_FETCH,$(target))),$$(call MSG_UNFETCH,$(target)))
+ $$(QUIET)$(TEST_EXT) -f $(out).lst -- $$(MAKE) -f $(MAKEFILE) --no-print-directory $(out)_unfetched
+if $(KBUILD_KMK_REVISION) > 2911
+ $$(QUIET)kmk_builtin_dircache deleted "$(dir $(out))"
+endif
+ $$(QUIET)$$(if $$(_TARGET_$(target)_DIGEST),$$(MAKE) -f $(MAKEFILE) --no-print-directory $(out).lst,$$(RMDIR) -p --ignore-fail-on-non-empty --ignore-fail-on-not-exist -- $$(dir $$@))
+ $$(QUIET2)$$(if $$(_TARGET_$(target)_DIGEST),$$(APPEND) $$@ "_TARGET_$(target)_DIGEST_PREV := $(_TARGET_$(target)_DIGEST)")
+
+.NOTPARALLEL: $(out).lst $(out)_unfetched $(out)
+
+endef
+
+
+##
+# Deal with one fetch target.
+# @param target
+# @param bld_trg
+# @param bld_trg_arch
+define def_fetch
+# common
+## @todo Install-revamp: FIXME
+INSTARGET_$(target) := $($(target)_INST)
+ifneq ($(patsubst %/,ok,$(INSTARGET_$(target))),ok)
+$(error kBuild: Bad or missing INST property for target '$(target)'. \
+ $(target)_INST='$($(target)_INST)' ($(origin $(target)_INST)))
+endif
+_TARGET_$(target)_FETCHED :=
+_TARGET_$(target)_UNPACKED :=
+_TARGET_$(target)_DIGEST :=
+local clean_files := $($(target)_CLEAN) $($(target)_CLEAN.$(bld_trg)) $($(target)_CLEAN.$(bld_trg).$(bld_trg_arch)) $($(target)_CLEAN.$(bld_trg_arch)) $($(target)_CLEAN.$(KBUILD_TYPE))
+
+# The 'sources'.
+#$ (warning dbg fetch: target=$(target) sources=$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(KBUILD_TARGET)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(KBUILD_TARGET).$(bld_trg_arch)))
+$(foreach source,$($(target)_SOURCES) $($(target)_SOURCES.$(KBUILD_TYPE)) $($(target)_SOURCES.$(bld_trg)) $($(target)_SOURCES.$(bld_trg_arch)) $($(target)_SOURCES.$(bld_trg).$(bld_trg_arch)),\
+ $(evalval def_fetch_src))
+
+# The target.
+local inst := $(INSTARGET_$(target))
+local out := $(inst)_kBuild_fetch_$(target)
+
+$(eval includedep $(out))
+
+$(eval $(def_fetch_rules))
+
+# Define the aliases here (doesn't work if defined in def_fetch_rules, just like includedep).
+$(target): $(out)
+$(target)_unfetch: $(out)_unfetched
+
+_FETCHES += $(out)
+_DOWNLOADS += $(_TARGET_$(target)_FETCHED)
+_UNPACKS += $(_TARGET_$(target)_UNPACKED)
+_UNFETCHES += $(out)_unfetched
+_DIRS += $(inst)
+_CLEAN_FILES += $(clean_files)
+
+endef
+$(eval-opt-var def_fetch)
+
+# Walk the FETCH target lists.
+bld_trg := $(KBUILD_TARGET)
+bld_trg_arch := $(KBUILD_TARGET_ARCH)
+$(foreach target, $(_ALL_FETCHES), \
+ $(evalvalctx def_fetch))
+
+# some aliases.
+download: $(_DOWNLOADS)
+unpack: $(_UNPACKS)
+fetch: $(_FETCHES)
+unfetch: $(_UNFETCHES)
+
+ifdef KBUILD_PROFILE_SELF
+ $(evalcall def_profile_self, done fetching targets)
+endif