aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Bergin <peter@berginkonsult.se>2018-10-16 22:30:32 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-10-18 11:03:13 +0100
commita937cff2746073d1dea37d85e7305d8d6705ff28 (patch)
tree9e1ebd0923815407ad583d0c96af3e79203e2b2d
parent715ec20c433cb4ed5fde938c33a42b2a296e4e56 (diff)
downloadopenembedded-core-contrib-a937cff2746073d1dea37d85e7305d8d6705ff28.tar.gz
openembedded-core-contrib-a937cff2746073d1dea37d85e7305d8d6705ff28.tar.bz2
openembedded-core-contrib-a937cff2746073d1dea37d85e7305d8d6705ff28.zip
rpm: handle virtual memory usage when limit is set
Fix the situation where the task do_package_write_rpm ends up in "liblzma: memory allocation failed". This happens if the host environment has set a limit on virtual_memory for the user with 'ulimit -v' for packages with a lot of binary packages, e.g. glibc-locale. Signed-off-by: Peter Bergin <peter@berginkonsult.se> Signed-off-by: Ross Burton <ross.burton@intel.com>
-rw-r--r--meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch65
-rw-r--r--meta/recipes-devtools/rpm/rpm_4.14.2.bb1
2 files changed, 66 insertions, 0 deletions
diff --git a/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
new file mode 100644
index 0000000000..6454785254
--- /dev/null
+++ b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
@@ -0,0 +1,65 @@
+From 0066b862bb3a09f39295abd5d972a53ac8dc1555 Mon Sep 17 00:00:00 2001
+From: Peter Bergin <peter@berginkonsult.se>
+Date: Wed, 19 Sep 2018 15:12:31 +0200
+Subject: [PATCH] rpm/rpmio.c: restrict virtual memory usage if limit set
+
+A solution to avoid OOM situation when the virtual memory is restricted
+for a user (ulimit -v). As the lzopen_internal function is run in parallel
+one instance per CPU thread the available virtual memory is limited per
+CPU thread.
+
+Upstream-Status: Pending [merge of multithreading patches to upstream]
+
+Signed-off-by: Peter Bergin <peter@berginkonsult.se>
+---
+ rpmio/rpmio.c | 34 ++++++++++++++++++++++++++++++++++
+ 1 file changed, 34 insertions(+)
+
+diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c
+index e051c98..b3c56b6 100644
+--- a/rpmio/rpmio.c
++++ b/rpmio/rpmio.c
+@@ -845,6 +845,40 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
+ }
+ #endif
+
++ struct rlimit virtual_memory;
++ getrlimit(RLIMIT_AS, &virtual_memory);
++ if (virtual_memory.rlim_cur != RLIM_INFINITY) {
++ const uint64_t virtual_memlimit = virtual_memory.rlim_cur;
++ const uint64_t virtual_memlimit_per_cpu_thread =
++ virtual_memlimit / lzma_cputhreads();
++ uint64_t memory_usage_virt;
++ rpmlog(RPMLOG_NOTICE, "XZ: virtual memory restricted to %lu and "
++ "per CPU thread %lu\n", virtual_memlimit, virtual_memlimit_per_cpu_thread);
++ /* keep reducing the number of compression threads until memory
++ usage falls below the limit per CPU thread*/
++ while ((memory_usage_virt = lzma_stream_encoder_mt_memusage(&mt_options)) >
++ virtual_memlimit_per_cpu_thread) {
++ /* If number of threads goes down to zero lzma_stream_encoder will
++ * will return UINT64_MAX. We must check here to avoid an infinite loop.
++ * If we get into situation that one thread requires more virtual memory
++ * than available we set one thread, print error message and try anyway. */
++ if (--mt_options.threads == 0) {
++ mt_options.threads = 1;
++ rpmlog(RPMLOG_WARNING,
++ "XZ: Could not adjust number of threads to get below "
++ "virtual memory limit %lu. usage %lu\n",
++ virtual_memlimit_per_cpu_thread, memory_usage_virt);
++ break;
++ }
++ }
++ if (threads != (int)mt_options.threads)
++ rpmlog(RPMLOG_NOTICE,
++ "XZ: Adjusted the number of threads from %d to %d to not "
++ "exceed the memory usage limit of %lu bytes\n",
++ threads, mt_options.threads, virtual_memlimit);
++
++ }
++
+ ret = lzma_stream_encoder_mt(&lzfile->strm, &mt_options);
+ }
+ #endif
+--
+2.7.4
+
diff --git a/meta/recipes-devtools/rpm/rpm_4.14.2.bb b/meta/recipes-devtools/rpm/rpm_4.14.2.bb
index 46f88375ff..200fe4da26 100644
--- a/meta/recipes-devtools/rpm/rpm_4.14.2.bb
+++ b/meta/recipes-devtools/rpm/rpm_4.14.2.bb
@@ -39,6 +39,7 @@ SRC_URI = "git://github.com/rpm-software-management/rpm;branch=rpm-4.14.x \
file://0003-rpmstrpool.c-make-operations-over-string-pools-threa.patch \
file://0004-build-pack.c-remove-static-local-variables-from-buil.patch \
file://0001-perl-disable-auto-reqs.patch \
+ file://0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch \
"
PE = "1"