buildroot/package/pkg-utils.mk

216 lines
6.7 KiB
Makefile
Raw Normal View History

################################################################################
#
# This file contains various utility functions used by the package
# infrastructure, or by the packages themselves.
#
################################################################################
#
# Manipulation of .config files based on the Kconfig
# infrastructure. Used by the BusyBox package, the Linux kernel
# package, and more.
#
define KCONFIG_ENABLE_OPT # (option, file)
$(SED) "/\\<$(1)\\>/d" $(2)
echo '$(1)=y' >> $(2)
endef
define KCONFIG_SET_OPT # (option, value, file)
$(SED) "/\\<$(1)\\>/d" $(3)
echo '$(1)=$(2)' >> $(3)
endef
define KCONFIG_DISABLE_OPT # (option, file)
$(SED) "/\\<$(1)\\>/d" $(2)
echo '# $(1) is not set' >> $(2)
endef
# Helper functions to determine the name of a package and its
# directory from its makefile directory, using the $(MAKEFILE_LIST)
# variable provided by make. This is used by the *-package macros to
# automagically find where the package is located.
.mk files: bulk aligment and whitespace cleanup of assignments The Buildroot coding style defines one space around make assignments and does not align the assignment symbols. This patch does a bulk fix of offending packages. The package infrastructures (or more in general assignments to calculated variable names, like $(2)_FOO) are not touched. Alignment of line continuation characters (\) is kept as-is. The sed command used to do this replacement is: find * -name "*.mk" | xargs sed -i \ -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\s*$#\1 \2#' -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\s*\([^\\]\+\)$#\1 \2 \3#' -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\s*\([^\\ \t]\+\s*\\\)\s*$#\1 \2 \3#' -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\(\s*\\\)#\1 \2\3#' Brief explanation of this command: ^\([A-Z0-9a-z_]\+\) a regular variable at the beginning of the line \([?:+]\?=\) any assignment character =, :=, ?=, += \([^\\]\+\) any string not containing a line continuation \([^\\ \t]\+\s*\\\) string, optional whitespace, followed by a line continuation character \(\s*\\\) optional whitespace, followed by a line continuation character Hence, the first subexpression handles empty assignments, the second handles regular assignments, the third handles regular assignments with line continuation, and the fourth empty assignments with line continuation. This expression was tested on following test text: (initial tab not included) FOO = spaces before FOO = spaces before and after FOO = tab before FOO = tab and spaces before FOO = tab after FOO = tab and spaces after FOO = spaces and tab after FOO = \ FOO = bar \ FOO = bar space \ FOO = \ GENIMAGE_DEPENDENCIES = host-pkgconf libconfuse FOO += spaces before FOO ?= spaces before and after FOO := FOO = FOO = FOO = FOO = $(MAKE1) CROSS_COMPILE=$(TARGET_CROSS) -C AT91BOOTSTRAP3_DEFCONFIG = \ AXEL_DISABLE_I18N=--i18n=0 After this bulk change, following manual fixups were done: - fix line continuation alignment in cegui06 and spice (the sed expression leaves the number of whitespace between the value and line continuation character intact, but the whitespace before that could have changed, causing misalignment. - qt5base was reverted, as this package uses extensive alignment which actually makes the code more readable. Finally, the end result was manually reviewed. Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com> Cc: Yann E. Morin <yann.morin.1998@free.fr> Cc: Thomas Petazzoni <thomas.petazzoni@free-electrons.com> Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2014-10-07 01:06:03 -06:00
pkgdir = $(dir $(lastword $(MAKEFILE_LIST)))
pkgname = $(lastword $(subst /, ,$(pkgdir)))
# Define extractors for different archive suffixes
INFLATE.bz2 = $(BZCAT)
INFLATE.gz = $(ZCAT)
INFLATE.lz = $(LZCAT)
INFLATE.lzma = $(XZCAT)
INFLATE.tbz = $(BZCAT)
INFLATE.tbz2 = $(BZCAT)
INFLATE.tgz = $(ZCAT)
INFLATE.xz = $(XZCAT)
INFLATE.tar = cat
# suitable-extractor(filename): returns extractor based on suffix
suitable-extractor = $(INFLATE$(suffix $(1)))
EXTRACTOR_PKG_DEPENDENCY.lzma = $(BR2_XZCAT_HOST_DEPENDENCY)
EXTRACTOR_PKG_DEPENDENCY.xz = $(BR2_XZCAT_HOST_DEPENDENCY)
EXTRACTOR_PKG_DEPENDENCY.lz = $(BR2_LZIP_HOST_DEPENDENCY)
# extractor-pkg-dependency(filename): returns a Buildroot package
# dependency needed to extract file based on suffix
extractor-pkg-dependency = $(EXTRACTOR_PKG_DEPENDENCY$(suffix $(1)))
# extractor-system-dependency(filename): returns the name of the tool
# needed to extract 'filename', and is meant to be used with
# DL_TOOLS_DEPENDENCIES, in order to check that the necesary tool is
# provided by the system Buildroot runs on.
#
# $(firstword) is used here because the extractor can have arguments,
# like ZCAT="gzip -d -c", and to check for the dependency we only want
# 'gzip'.
extractor-system-dependency = $(if $(EXTRACTOR_PKG_DEPENDENCY$(suffix $(1))),,\
$(firstword $(INFLATE$(suffix $(1)))))
# check-deprecated-variable -- throw an error on deprecated variables
# example:
# $(eval $(call check-deprecated-variable,FOO_MAKE_OPT,FOO_MAKE_OPTS))
define check-deprecated-variable # (deprecated var, new var)
ifneq ($$(origin $(1)),undefined)
$$(error Package error: use $(2) instead of $(1). Please fix your .mk file)
endif
endef
# $(1): YES or NO
define yesno-to-bool
$(subst NO,false,$(subst YES,true,$(1)))
endef
core: introduce new global show-info Users are increasingly trying to extract information about packages. For example, they might need to get the list of URIs, or the dependencies of a package. Although we do have a bunch of rules to generate some of that, this is done in ad-hoc way, with most of the output formats just ad-hoc, raw, unformatted blurbs, mostly internal data dumped as-is. Introduce a new rule, show-info, that provides a properly formatted output of all the meta-information about packages: name, type, version, licenses, dependencies... We choose to use JSON as the output format, because it is pretty versatile, has parsers in virtually all languages, has tools to parse from the shell (jq). It also closely matches Python data structure, which makes it easy to use with our own internal tools as well. Finally, JSON being a key-value store, allows for easy expanding the output without requiring existing consumers to be updated; new, unknown keys are simply ignored by those (as long as they are true JSON parsers). The complex part of this change was the conditional output of parts of the data: virtual packages have no source, version, license or downloads, unlike non-virtual packages. Same goes for filesystems. We use a wrapper macro, show-info, that de-multiplexes unto either the package-related- or filesystem-related macros, and for packages, we also use a detailed macro for non-virtual packages. It is non-trivial to properly output correct JSON blurbs, especially when trying to output an array of objects, like so, where the last item shall not be followed by a comma: [ { ... }, { ... } ] So, we use a trick (as sugegsted by Arnout), to $(subst) any pair of ",}" or ", }" or ",]" or ", ]" with only the respective closing symbol, "}" or "]". The whole stuff is $(strip)ed to make it a somewhat-minified JSON blurb that fits on a single line with all spaces squashed (but still with spaces, as it is not possible to differentiate spaces between JSON elements from spaces inside JSON strings). Reported-by: Thomas De Schampheleire <patrickdepinguin@gmail.com> Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr> Cc: Thomas De Schampheleire <patrickdepinguin@gmail.com> Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com> Cc: Arnout Vandecappelle <arnout@mind.be> Signed-off-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>
2019-04-15 13:47:31 -06:00
# json-info -- return package or filesystem metadata formatted as an entry
# of a JSON dictionnary
# $(1): upper-case package or filesystem name
define json-info
"$($(1)_NAME)": {
"type": "$($(1)_TYPE)",
$(if $(filter rootfs,$($(1)_TYPE)), \
$(call _json-info-fs,$(1)), \
$(call _json-info-pkg,$(1)), \
)
}
endef
# _json-info-pkg, _json-info-pkg-details, _json-info-fs: private helpers
# for json-info, above
define _json-info-pkg
$(if $($(1)_IS_VIRTUAL), \
"virtual": true$(comma),
"virtual": false$(comma)
$(call _json-info-pkg-details,$(1)) \
)
"dependencies": [
$(call make-comma-list,$(sort $($(1)_FINAL_ALL_DEPENDENCIES)))
],
"reverse_dependencies": [
$(call make-comma-list,$(sort $($(1)_RDEPENDENCIES)))
]
endef
define _json-info-pkg-details
"version": "$($(1)_DL_VERSION)",
"licenses": "$($(1)_LICENSE)",
"dl_dir": "$($(1)_DL_SUBDIR)",
"install_target": $(call yesno-to-bool,$($(1)_INSTALL_TARGET)),
"install_staging": $(call yesno-to-bool,$($(1)_INSTALL_STAGING)),
"install_images": $(call yesno-to-bool,$($(1)_INSTALL_IMAGES)),
core: introduce new global show-info Users are increasingly trying to extract information about packages. For example, they might need to get the list of URIs, or the dependencies of a package. Although we do have a bunch of rules to generate some of that, this is done in ad-hoc way, with most of the output formats just ad-hoc, raw, unformatted blurbs, mostly internal data dumped as-is. Introduce a new rule, show-info, that provides a properly formatted output of all the meta-information about packages: name, type, version, licenses, dependencies... We choose to use JSON as the output format, because it is pretty versatile, has parsers in virtually all languages, has tools to parse from the shell (jq). It also closely matches Python data structure, which makes it easy to use with our own internal tools as well. Finally, JSON being a key-value store, allows for easy expanding the output without requiring existing consumers to be updated; new, unknown keys are simply ignored by those (as long as they are true JSON parsers). The complex part of this change was the conditional output of parts of the data: virtual packages have no source, version, license or downloads, unlike non-virtual packages. Same goes for filesystems. We use a wrapper macro, show-info, that de-multiplexes unto either the package-related- or filesystem-related macros, and for packages, we also use a detailed macro for non-virtual packages. It is non-trivial to properly output correct JSON blurbs, especially when trying to output an array of objects, like so, where the last item shall not be followed by a comma: [ { ... }, { ... } ] So, we use a trick (as sugegsted by Arnout), to $(subst) any pair of ",}" or ", }" or ",]" or ", ]" with only the respective closing symbol, "}" or "]". The whole stuff is $(strip)ed to make it a somewhat-minified JSON blurb that fits on a single line with all spaces squashed (but still with spaces, as it is not possible to differentiate spaces between JSON elements from spaces inside JSON strings). Reported-by: Thomas De Schampheleire <patrickdepinguin@gmail.com> Signed-off-by: "Yann E. MORIN" <yann.morin.1998@free.fr> Cc: Thomas De Schampheleire <patrickdepinguin@gmail.com> Cc: Thomas Petazzoni <thomas.petazzoni@bootlin.com> Cc: Arnout Vandecappelle <arnout@mind.be> Signed-off-by: Arnout Vandecappelle (Essensium/Mind) <arnout@mind.be>
2019-04-15 13:47:31 -06:00
"downloads": [
$(foreach dl,$(sort $($(1)_ALL_DOWNLOADS)),
{
"source": "$(notdir $(dl))",
"uris": [
$(call make-comma-list,
$(subst \|,|,
$(call DOWNLOAD_URIS,$(dl),$(1))
)
)
]
},
)
],
endef
define _json-info-fs
"dependencies": [
$(call make-comma-list,$(sort $($(1)_DEPENDENCIES)))
]
endef
# clean-json -- cleanup pseudo-json into clean json:
# - remove commas before closing ] and }
# - minify with $(strip)
clean-json = $(strip \
$(subst $(comma)},}, $(subst $(comma)$(space)},$(space)}, \
$(subst $(comma)],], $(subst $(comma)$(space)],$(space)], \
$(strip $(1)) \
)))) \
)
core: implement per-package SDK and target This commit implements the core of the move to per-package SDK and target directories. The main idea is that instead of having a global output/host and output/target in which all packages install files, we switch to per-package host and target directories, that only contain their explicit dependencies. There are two main benefits: - Packages will now see only the dependencies they explicitly list in their <pkg>_DEPENDENCIES variable, and the recursive dependencies thereof. - We can support top-level parallel build properly, because a package only "sees" its own host directory and target directory, isolated from the build of other packages that can happen in parallel. It works as follows: - A new output/per-package/ directory is created, which will contain one sub-directory per package, and inside it, a "host" directory and a "target" directory: output/per-package/busybox/target output/per-package/busybox/host output/per-package/host-fakeroot/target output/per-package/host-fakeroot/host This output/per-package/ directory is PER_PACKAGE_DIR. - The global TARGET_DIR and HOST_DIR variable now automatically point to the per-package directory when PKG is defined. So whenever a package references $(HOST_DIR) or $(TARGET_DIR) in its build process, it effectively references the per-package host/target directories. Note that STAGING_DIR is a sub-dir of HOST_DIR, so it is handled as well. - Of course, packages have dependencies, so those dependencies must be installed in the per-package host and target directories. To do so, we simply rsync (using hard links to save space and time) the host and target directories of the direct dependencies of the package to the current package host and target directories. We only need to take care of direct dependencies (and not recursively all dependencies), because we accumulate into those per-package host and target directories the files installed by the dependencies. Note that this only works because we make the assumption that one package does *not* overwrite files installed by another package. This is done for "extract dependencies" at the beginning of the extract step, and for "normal dependencies" at the beginning of the configure step. This is basically enough to make per-package SDK and target work. The only gotcha is that at the end of the build, output/target and output/host are empty, which means that: - The filesystem image creation code cannot work. - We don't have a SDK to build code outside of Buildroot. In order to fix this, this commit extends the target-finalize step so that it starts by populating output/target and output/host by rsync-ing into them the target and host directories of all packages listed in the $(PACKAGES) variable. It is necessary to do this sequentially in the target-finalize step and not in each package. Doing it in package installation means that it can be done in parallel. In that case, there is a chance that two rsyncs are creating the same hardlink or directory at the same time, which makes one of them fail. This change to per-package directories has an impact on the RPATH built into the host binaries, as those RPATH now point to various per-package host directories, and no longer to the global host directory. We do not try to rewrite such RPATHs during the build as having such RPATHs is perfectly fine, but we still need to handle two fallouts from this change: - The check-host-rpath script, which verifies at the end of each package installation that it has the appropriate RPATH, is modified to understand that a RPATH to $(PER_PACKAGE_DIR)/<pkg>/host/lib is a correct RPAT. - The fix-rpath script, which mungles the RPATH mainly for the SDK preparation, is modified to rewrite the RPATH to not point to per-package directories. Indeed the patchelf --make-rpath-relative call only works if the RPATH points to the ROOTDIR passed as argument, and this ROOTDIR is the global host directory. Rewriting the RPATH to not point to per-package host directories prior to this is an easy solution to this issue. Signed-off-by: Thomas Petazzoni <thomas.petazzoni@bootlin.com> Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2019-11-05 09:46:40 -07:00
ifeq ($(BR2_PER_PACKAGE_DIRECTORIES),y)
# rsync the contents of per-package directories
# $1: space-separated list of packages to rsync from
# $2: 'host' or 'target'
# $3: destination directory
define per-package-rsync
mkdir -p $(3)
$(foreach pkg,$(1),\
rsync -a --link-dest=$(PER_PACKAGE_DIR)/$(pkg)/$(2)/ \
$(PER_PACKAGE_DIR)/$(pkg)/$(2)/ \
$(3)$(sep))
endef
# prepares the per-package HOST_DIR and TARGET_DIR of the current
# package, by rsync the host and target directories of the
# dependencies of this package. The list of dependencies is passed as
# argument, so that this function can be used to prepare with
# different set of dependencies (download, extract, configure, etc.)
#
# $1: space-separated list of packages to rsync from
define prepare-per-package-directory
$(call per-package-rsync,$(1),host,$(HOST_DIR))
$(call per-package-rsync,$(1),target,$(TARGET_DIR))
endef
endif
#
# legal-info helper functions
#
.mk files: bulk aligment and whitespace cleanup of assignments The Buildroot coding style defines one space around make assignments and does not align the assignment symbols. This patch does a bulk fix of offending packages. The package infrastructures (or more in general assignments to calculated variable names, like $(2)_FOO) are not touched. Alignment of line continuation characters (\) is kept as-is. The sed command used to do this replacement is: find * -name "*.mk" | xargs sed -i \ -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\s*$#\1 \2#' -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\s*\([^\\]\+\)$#\1 \2 \3#' -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\s*\([^\\ \t]\+\s*\\\)\s*$#\1 \2 \3#' -e 's#^\([A-Z0-9a-z_]\+\)\s*\([?:+]\?=\)\(\s*\\\)#\1 \2\3#' Brief explanation of this command: ^\([A-Z0-9a-z_]\+\) a regular variable at the beginning of the line \([?:+]\?=\) any assignment character =, :=, ?=, += \([^\\]\+\) any string not containing a line continuation \([^\\ \t]\+\s*\\\) string, optional whitespace, followed by a line continuation character \(\s*\\\) optional whitespace, followed by a line continuation character Hence, the first subexpression handles empty assignments, the second handles regular assignments, the third handles regular assignments with line continuation, and the fourth empty assignments with line continuation. This expression was tested on following test text: (initial tab not included) FOO = spaces before FOO = spaces before and after FOO = tab before FOO = tab and spaces before FOO = tab after FOO = tab and spaces after FOO = spaces and tab after FOO = \ FOO = bar \ FOO = bar space \ FOO = \ GENIMAGE_DEPENDENCIES = host-pkgconf libconfuse FOO += spaces before FOO ?= spaces before and after FOO := FOO = FOO = FOO = FOO = $(MAKE1) CROSS_COMPILE=$(TARGET_CROSS) -C AT91BOOTSTRAP3_DEFCONFIG = \ AXEL_DISABLE_I18N=--i18n=0 After this bulk change, following manual fixups were done: - fix line continuation alignment in cegui06 and spice (the sed expression leaves the number of whitespace between the value and line continuation character intact, but the whitespace before that could have changed, causing misalignment. - qt5base was reverted, as this package uses extensive alignment which actually makes the code more readable. Finally, the end result was manually reviewed. Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com> Cc: Yann E. Morin <yann.morin.1998@free.fr> Cc: Thomas Petazzoni <thomas.petazzoni@free-electrons.com> Signed-off-by: Peter Korsgaard <peter@korsgaard.com>
2014-10-07 01:06:03 -06:00
LEGAL_INFO_SEPARATOR = "::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::"
define legal-warning # text
echo "WARNING: $(1)" >>$(LEGAL_WARNINGS)
endef
define legal-warning-pkg # pkg, text
echo "WARNING: $(1): $(2)" >>$(LEGAL_WARNINGS)
endef
define legal-warning-nosource # pkg, {local|override}
$(call legal-warning-pkg,$(1),sources not saved ($(2) packages not handled))
endef
define legal-manifest # {HOST|TARGET}, pkg, version, license, license-files, source, url, dependencies
echo '"$(2)","$(3)","$(4)","$(5)","$(6)","$(7)","$(8)"' >>$(LEGAL_MANIFEST_CSV_$(1))
endef
define legal-license-file # pkgname, pkgname-pkgver, pkg-hashfile, filename, file-fullpath, {HOST|TARGET}
mkdir -p $(LICENSE_FILES_DIR_$(6))/$(2)/$(dir $(4)) && \
{ \
support/download/check-hash $(3) $(5) $(4); \
case $${?} in (0|3) ;; (*) exit 1;; esac; \
} && \
cp $(5) $(LICENSE_FILES_DIR_$(6))/$(2)/$(4)
endef
non-virtual-deps = $(foreach p,$(1),$(if $($(call UPPERCASE,$(p))_IS_VIRTUAL),,$(p)))
# Returns the list of recursive dependencies and their licensing terms
# for the package specified in parameter (in lowercase). If that
# package is a target package, remove host packages from the list.
legal-deps = \
$(foreach p,\
$(filter-out $(if $(1:host-%=),host-%),\
$(call non-virtual-deps,\
$($(call UPPERCASE,$(1))_FINAL_RECURSIVE_DEPENDENCIES))),$(p) [$($(call UPPERCASE,$(p))_LICENSE)])