diff --git a/PAE.patch b/PAE.patch new file mode 100644 index 0000000..d2f29fb --- /dev/null +++ b/PAE.patch @@ -0,0 +1,159 @@ +From 55f7ba830d40d438f0b0663a505e0c227fc68b6b Mon Sep 17 00:00:00 2001 +From: Phillip Lougher +Date: Tue, 10 Jun 2014 21:51:52 +0100 +Subject: mksquashfs: fix phys mem calculation for 32-bit processes on + PAE/64-bit kernels + +When adding the code to base default memory usage on physical memory +(by default use 25% of physical memory), I made an oversight. I assumed +the process would be able to address 25% of physical memory. + +However, for 32-bit processes running on a PAE kernel or 64-bit kernel, +25% of physical memory can easily exceed the addressible memory for a +32-bit process, e.g. if a machine has 24 GB of physical memory, the +code would asume the process could easily use 6 GB. + +A 32-bit process by definition can only address 4 GB (32-bit pointers). +But, due to the typical kernel/user-space split (1GB/3GB, or 2GB/2GB) +on PAE kernels, a 32-bit process may only be able to address 2 GB. + +So, if Mksquashfs is a 32-bit application running on a PAE/64-bit kernel, +the code assumes it can address much more memory than it really can, which +means it runs out of memory. + +The fix is to impose a maximum default limit on 32-bit kernels, or +otherwise to never use a value more than 25% of the address space. If +we assume the maximum address space is 2 GB, then the maximum becomes +512 MB. But, given most kernels used the 1GB/3GB split, that may be +unduely conservative, and 25% of 3 GB (756 MB) may be better. This +patch compromises on 640 MB, which is mid-way between the 512 MB and 756 MB +values. It is also the fixed default value previously used by Mksquashfs. + +This patch also alters the code which imposes a maximum size. Previously +it was believed limiting to the physical memory size was adequate. But +obviously this needs to be updated to take into account a 32-bit process +may only be able to address 2 GB. In the process I've also taken the +opportunity to limit all requests to no more than 75% of physical memory. + +Signed-off-by: Phillip Lougher + +diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c +index 86f82bb..5370ecf 100644 +--- a/squashfs-tools/mksquashfs.c ++++ b/squashfs-tools/mksquashfs.c +@@ -304,7 +304,7 @@ void restorefs(); + struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth); + void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad); + unsigned short get_checksum_mem(char *buff, int bytes); +-int get_physical_memory(); ++void check_usable_phys_mem(int total_mem); + + + void prep_exit() +@@ -4053,11 +4053,7 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq, + BAD_ERROR("Queue sizes rediculously too large\n"); + total_mem += fwriteq; + +- if(total_mem > get_physical_memory()) { +- ERROR("Total queue sizes larger than physical memory.\n"); +- ERROR("Mksquashfs will exhaust physical memory and thrash.\n"); +- BAD_ERROR("Queues too large\n"); +- } ++ check_usable_phys_mem(total_mem); + + /* + * convert from queue size in Mbytes to queue size in +@@ -4879,6 +4875,72 @@ int get_physical_memory() + } + + ++void check_usable_phys_mem(int total_mem) ++{ ++ /* ++ * We want to allow users to use as much of their physical ++ * memory as they wish. However, for practical reasons there are ++ * limits which need to be imposed, to protect users from themselves ++ * and to prevent people from using Mksquashfs as a DOS attack by using ++ * all physical memory. Mksquashfs uses memory to cache data from disk ++ * to optimise performance. It is pointless to ask it to use more ++ * than 75% of physical memory, as this causes thrashing and it is thus ++ * self-defeating. ++ */ ++ int mem = get_physical_memory(); ++ ++ mem = (mem >> 1) + (mem >> 2); /* 75% */ ++ ++ if(total_mem > mem) { ++ ERROR("Total memory requested is more than 75%% of physical " ++ "memory.\n"); ++ ERROR("Mksquashfs uses memory to cache data from disk to " ++ "optimise performance.\n"); ++ ERROR("It is pointless to ask it to use more than this amount " ++ "of memory, as this\n"); ++ ERROR("causes thrashing and it is thus self-defeating.\n"); ++ BAD_ERROR("Requested memory size too large\n"); ++ } ++ ++ if(sizeof(void *) == 4 && total_mem > 2048) { ++ /* ++ * If we're running on a kernel with PAE or on a 64-bit kernel, ++ * then the 75% physical memory limit can still easily exceed ++ * the addressable memory by this process. ++ * ++ * Due to the typical kernel/user-space split (1GB/3GB, or ++ * 2GB/2GB), we have to conservatively assume the 32-bit ++ * processes can only address 2-3GB. So refuse if the user ++ * tries to allocate more than 2GB. ++ */ ++ ERROR("Total memory requested may exceed maximum " ++ "addressable memory by this process\n"); ++ BAD_ERROR("Requested memory size too large\n"); ++ } ++} ++ ++ ++int get_default_phys_mem() ++{ ++ int mem = get_physical_memory() / SQUASHFS_TAKE; ++ ++ if(sizeof(void *) == 4 && mem > 640) { ++ /* ++ * If we're running on a kernel with PAE or on a 64-bit kernel, ++ * the default memory usage can exceed the addressable ++ * memory by this process. ++ * Due to the typical kernel/user-space split (1GB/3GB, or ++ * 2GB/2GB), we have to conservatively assume the 32-bit ++ * processes can only address 2-3GB. So limit the default ++ * usage to 640M, which gives room for other data. ++ */ ++ mem = 640; ++ } ++ ++ return mem; ++} ++ ++ + void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq, + int *fwriteq) + { +@@ -4890,7 +4952,7 @@ void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq, + + + #define VERSION() \ +- printf("mksquashfs version 4.3 (2014/05/12)\n");\ ++ printf("mksquashfs version 4.3-git (2014/06/09)\n");\ + printf("copyright (C) 2014 Phillip Lougher "\ + "\n\n"); \ + printf("This program is free software; you can redistribute it and/or"\ +@@ -4918,7 +4980,7 @@ int main(int argc, char *argv[]) + int fragq; + int bwriteq; + int fwriteq; +- int total_mem = get_physical_memory() / SQUASHFS_TAKE; ++ int total_mem = get_default_phys_mem(); + int progress = TRUE; + int force_progress = FALSE; + struct file_buffer **fragment = NULL; +-- +cgit v0.10.1 + diff --git a/mem-overflow.patch b/mem-overflow.patch new file mode 100644 index 0000000..1450fe9 --- /dev/null +++ b/mem-overflow.patch @@ -0,0 +1,33 @@ +From 604b607d8ac91eb8afc0b6e3d917d5c073096103 Mon Sep 17 00:00:00 2001 +From: Phillip Lougher +Date: Wed, 11 Jun 2014 04:51:37 +0100 +Subject: mksquashfs: ensure value does not overflow a signed int in -mem + option + +Signed-off-by: Phillip Lougher + +diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c +index 5370ecf..9676dc8 100644 +--- a/squashfs-tools/mksquashfs.c ++++ b/squashfs-tools/mksquashfs.c +@@ -5193,7 +5193,16 @@ print_compressor_options: + argv[0]); + exit(1); + } +- /* convert from bytes to Mbytes */ ++ ++ /* ++ * convert from bytes to Mbytes, ensuring the value ++ * does not overflow a signed int ++ */ ++ if(number >= (1LL << 51)) { ++ ERROR("%s: -mem invalid mem size\n", argv[0]); ++ exit(1); ++ } ++ + total_mem = number / 1048576; + if(total_mem < (SQUASHFS_LOWMEM / SQUASHFS_TAKE)) { + ERROR("%s: -mem should be %d Mbytes or " +-- +cgit v0.10.1 + diff --git a/squashfs-tools.spec b/squashfs-tools.spec index 5dcf4c1..8d52254 100644 --- a/squashfs-tools.spec +++ b/squashfs-tools.spec @@ -1,7 +1,7 @@ Summary: Utility for the creation of squashfs filesystems Name: squashfs-tools Version: 4.3 -Release: 5%{?dist} +Release: 6%{?dist} License: GPLv2+ Group: System Environment/Base URL: http://squashfs.sourceforge.net/ @@ -10,6 +10,12 @@ Source0: http://downloads.sourceforge.net/squashfs/squashfs%{version}.tar.gz # The man pages have been modified for 4.3 for Fedora. Source1: mksquashfs.1 Source2: unsquashfs.1 +# From master branch (55f7ba830d40d438f0b0663a505e0c227fc68b6b). +# 32 bit process can use too much memory when using PAE or 64 bit kernels +Patch0: PAE.patch +# From master branch (604b607d8ac91eb8afc0b6e3d917d5c073096103). +# Prevent overflows when using the -mem option. +Patch1: mem-overflow.patch BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root BuildRequires: zlib-devel BuildRequires: xz-devel @@ -23,6 +29,8 @@ contains the utilities for manipulating squashfs filesystems. %prep %setup -q -n squashfs%{version} +%patch0 +%patch1 %build pushd squashfs-tools @@ -49,6 +57,10 @@ rm -rf %{buildroot} %{_sbindir}/unsquashfs %changelog +* Fri Jun 13 2014 Bruno Wolff III 4.3-6 +- Apply a couple of upstream patches. +- Fixes issue issue with too much memory use under PAE kernels + * Sun Jun 08 2014 Fedora Release Engineering - 4.3-5 - Rebuilt for https://fedoraproject.org/wiki/Fedora_21_Mass_Rebuild