From e70bf8f4f487b56276db09390949cd865bfca43e Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Mon, 13 Nov 2023 21:00:03 +0800 Subject: hostreqs: Bump Coreutils minimal version to 8.1 and check for nproc We'll use nproc for setting MAKEFLAGS and TESTSUITEFLAGS. And if nproc is not available, we'll end up with "MAKEFLAGS=-j" which is very dangerous because it allows make to spawn infinite number of jobs. Check it early. The nproc program is added in Coreutils 8.1 so we need to bump the minimal version. --- chapter02/hostreqs.xml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/chapter02/hostreqs.xml b/chapter02/hostreqs.xml index 69533b9c7..63dcd943f 100644 --- a/chapter02/hostreqs.xml +++ b/chapter02/hostreqs.xml @@ -54,8 +54,9 @@ should be a link to bison or a small script that executes bison) + - Coreutils-7.0 + Coreutils-8.1 @@ -218,8 +219,8 @@ ver_kernel() fi } -# Coreutils first because-sort needs Coreutils >= 7.0 -ver_check Coreutils sort 7.0 || bail "--version-sort unsupported" +# Coreutils first because --version-sort needs Coreutils >= 7.0 +ver_check Coreutils sort 8.1 || bail "Coreutils too old, stop" ver_check Bash bash 3.2 ver_check Binutils ld 2.13.1 ver_check Bison bison 2.7 @@ -259,7 +260,13 @@ echo "Compiler check:" if printf "int main(){}" | g++ -x c++ - then echo "OK: g++ works"; else echo "ERROR: g++ does NOT work"; fi -rm -f a.out +rm -f a.out + +if [ "$(nproc)" = "" ]; then + echo "ERROR: nproc is not available or it produces empty output" +else + echo "OK: nproc reports $(nproc) logical cores are available" +fi EOF bash version-check.sh -- cgit v1.2.3-54-g00ecf From a0a803c0b053dd9b4867690d8bc25fc0b97fa486 Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Mon, 11 Sep 2023 14:42:06 +0800 Subject: settingenviron: Set MAKEFLAGS for parallelism in ~lfs/.bashrc --- chapter04/aboutsbus.xml | 12 ----------- chapter04/settingenviron.xml | 49 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/chapter04/aboutsbus.xml b/chapter04/aboutsbus.xml index f88db24f6..42a523fa1 100644 --- a/chapter04/aboutsbus.xml +++ b/chapter04/aboutsbus.xml @@ -37,18 +37,6 @@ numbers can vary by as much as dozens of minutes in some cases. - For many modern systems with multiple processors (or cores) the - compilation time for a package can be reduced by performing a "parallel - make" by either setting an environment variable or telling the - make program how many processors are available. For - instance, an Intel i5-6500 CPU can support four simultaneous processes with: - - export MAKEFLAGS='-j4' - - or by building with: - - make -j4 - When multiple processors are used in this way, the SBU units in the book will vary even more than they normally would. In some cases, the make step will simply fail. Analyzing the output of the build process will also diff --git a/chapter04/settingenviron.xml b/chapter04/settingenviron.xml index b255a4b45..2d8d6045f 100644 --- a/chapter04/settingenviron.xml +++ b/chapter04/settingenviron.xml @@ -195,6 +195,55 @@ EOF completed LFS system. + + For many modern systems with multiple processors (or cores) the + compilation time for a package can be reduced by performing a "parallel + make" by telling the make program how many processors are available via + a command line option or an environment variable. For instance, an Intel + Core i9-13900K processor has 8 P (performance) cores and + 16 E (efficiency) cores, and a P core can simultaneously run two threads + so each P core are modeled as two logical cores by the Linux kernel. + As the result there are 32 logical cores in total. One obvious way to + use all these logical cores is allowing make to spawn + up to 32 build jobs. This can be done by passing the + -j32 option to make: + + + make -j32 + + + Or set the MAKEFLAGS environment variable and its + content will be automatically used by make as + command line options: + + + export MAKEFLAGS=-j32 + + + + Never pass a -j option without a number to + make or set such an option in + MAKEFLAGS. Doing so will allow make + to spawn infinite build jobs and cause system stability issue. + + + + + To use all logical cores available for building packages in + and + , set MAKEFLAGS + now in .bashrc: + + +cat >> ~/.bashrc << "EOF" +export MAKEFLAGS=-j$(nproc) +EOF + + + Replace $(nproc) with the number of logical + cores you want to use if you don't want to use all the logical cores. + + Finally, to ensure the environment is fully prepared for building the temporary tools, force the bash shell to read the new user profile: -- cgit v1.2.3-54-g00ecf From 95ebbb42b7eda5114e7ea751030916f9c9eaa5fb Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Mon, 11 Sep 2023 14:42:45 +0800 Subject: chroot: Set MAKEFLAGS and TESTSUITEFLAGS for parallelism --- chapter07/chroot.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/chapter07/chroot.xml b/chapter07/chroot.xml index b8de03dc2..a8cd5f878 100644 --- a/chapter07/chroot.xml +++ b/chapter07/chroot.xml @@ -23,8 +23,21 @@ TERM="$TERM" \ PS1='(lfs chroot) \u:\w\$ ' \ PATH=/usr/bin:/usr/sbin \ + MAKEFLAGS="-j$(nproc)" \ + TESTSUITEFLAGS="-j$(nproc)" \ /bin/bash --login + + Again, replace $(nproc) with the number + of logical cores you want to use for building packages in this chapter + and the following chapters if you don't want to use all available + logical cores. The test suites of some packages (notably Autoconf, + Libtool, and Tar) in &ch-final; are not affected by + MAKEFLAGS, they use a TESTSUITEFLAGS + environment variable instead. So we set it here as well for running + these test suites with multiple cores. + + The -i option given to the env command will clear all the variables in the chroot environment. After that, only the HOME, TERM, PS1, and -- cgit v1.2.3-54-g00ecf From c17a77452a184025e6577e8cdf1315905acaa6bf Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Mon, 11 Sep 2023 16:07:26 +0800 Subject: automake: Do not use only 4 cores for testing if we have more --- chapter08/automake.xml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/chapter08/automake.xml b/chapter08/automake.xml index 4fc1bd277..5d7541049 100644 --- a/chapter08/automake.xml +++ b/chapter08/automake.xml @@ -53,11 +53,14 @@ make - Using the -j4 make option speeds up the tests, even on systems with - only one processor, due to internal delays in individual tests. To test + Using four parallel jobs speeds up the tests, even on systems with + less logical cores, due to internal delays in individual tests. To test the results, issue: -make -j4 check +make -j$(($(nproc)>4?$(nproc):4)) check + + Replace $((...)) with the number of + logical cores you want to use if you don't want to use all. The test t/subobj.sh is known to fail. -- cgit v1.2.3-54-g00ecf From 3a8d1825cc7dcb9cc8a7c01f5e98d672723bbe6a Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Wed, 13 Sep 2023 19:41:56 +0800 Subject: openssl: Add HARNESS_JOBS=$(nproc) --- chapter08/openssl.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chapter08/openssl.xml b/chapter08/openssl.xml index 6a4441552..25295f056 100644 --- a/chapter08/openssl.xml +++ b/chapter08/openssl.xml @@ -63,7 +63,7 @@ To test the results, issue: -make test +HARNESS_JOBS=$(nproc) make test One test, 30-test_afalg.t, is known to fail if the host kernel does not have enabled, -- cgit v1.2.3-54-g00ecf From d490056b64d0b55df01096871647a76574a142fc Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Wed, 13 Sep 2023 19:48:55 +0800 Subject: perl: Use TEST_JOBS=$(nproc) make test_harness for test suite --- chapter08/perl.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chapter08/perl.xml b/chapter08/perl.xml index d3159595c..2bb849645 100644 --- a/chapter08/perl.xml +++ b/chapter08/perl.xml @@ -111,7 +111,7 @@ export BUILD_BZIP2=0 To test the results (approximately 11 SBU), issue: -make test +TEST_JOBS=$(nproc) make test_harness Install the package and clean up: -- cgit v1.2.3-54-g00ecf From e07c41946983e891b283482c294ed748494c2578 Mon Sep 17 00:00:00 2001 From: Xi Ruoyao Date: Mon, 13 Nov 2023 21:32:40 +0800 Subject: Package updates - Update to elfutils-0.190 (#5373) - Update to vim-9.0.2103 - Update to linux-6.6.1 (#5369) - Update to xz-5.4.5 (#5371) - Update to iana-etc-20231107 - Update to gawk-5.3.0 (#5372) - Update to bash-5.2.21 (#5375) - Update to iproute2-6.6.0 (#5374) --- chapter01/changelog.xml | 38 ++++++++++++++++++++++++++++++++++++ packages.ent | 52 ++++++++++++++++++++++++------------------------- 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/chapter01/changelog.xml b/chapter01/changelog.xml index 82a3d85be..8ad5949fb 100644 --- a/chapter01/changelog.xml +++ b/chapter01/changelog.xml @@ -39,6 +39,44 @@ or as appropriate for the entry or if needed the entire day's listitem. --> + + 2023-11-13 + + + [xry111] - Update to elfutils-0.190. Fixes + #5373. + + + [xry111] - Update to vim-9.0.2103. Addresses + #4500. + + + [xry111] - Update to linux-6.6.1. Fixes + #5369. + + + [xry111] - Update to xz-5.4.5. Fixes + #5371. + + + [xry111] - Update to iana-etc-20231107. Addresses + #5006. + + + [xry111] - Update to gawk-5.3.0. Fixes + #5372. + + + [xry111] - Update to bash-5.2.21. Fixes + #5375. + + + [xry111] - Update to iproute2-6.6.0. Fixes + #5374. + + + + 2023-11-01 diff --git a/packages.ent b/packages.ent index 9e0b0db2d..fbb39bc31 100644 --- a/packages.ent +++ b/packages.ent @@ -47,10 +47,10 @@ - - + + - + @@ -148,10 +148,10 @@ - - + + - + @@ -208,10 +208,10 @@ - - + + - + @@ -317,10 +317,10 @@ - - + + - + @@ -341,10 +341,10 @@ - - + + - + @@ -430,13 +430,13 @@ - - + + - + - + - + - - + + @@ -762,10 +762,10 @@ - - + + - + -- cgit v1.2.3-54-g00ecf