From 55f7ba830d40d438f0b0663a505e0c227fc68b6b Mon Sep 17 00:00:00 2001

From: Phillip Lougher <phillip@squashfs.org.uk>

Date: Tue, 10 Jun 2014 21:51:52 +0100

Subject: mksquashfs: fix phys mem calculation for 32-bit processes on

 PAE/64-bit kernels

When adding the code to base default memory usage on physical memory

(by default use 25% of physical memory), I made an oversight.  I assumed

the process would be able to address 25% of physical memory.

However, for 32-bit processes running on a PAE kernel or 64-bit kernel,

25% of physical memory can easily exceed the addressible memory for a

32-bit process, e.g. if a machine has 24 GB of physical memory, the

code would asume the process could easily use 6 GB.

A 32-bit process by definition can only address 4 GB (32-bit pointers).

But, due to the typical kernel/user-space split (1GB/3GB, or 2GB/2GB)

on PAE kernels, a 32-bit process may only be able to address 2 GB.

So, if Mksquashfs is a 32-bit application running on a PAE/64-bit kernel,

the code assumes it can address much more memory than it really can, which

means it runs out of memory.

The fix is to impose a maximum default limit on 32-bit kernels, or

otherwise to never use a value more than 25% of the address space.  If

we assume the maximum address space is 2 GB, then the maximum becomes

512 MB.  But, given most kernels used the 1GB/3GB split, that may be

unduely conservative, and 25% of 3 GB (756 MB) may be better.  This

patch compromises on 640 MB, which is mid-way between the 512 MB and 756 MB

values.  It is also the fixed default value previously used by Mksquashfs.

This patch also alters the code which imposes a maximum size.  Previously

it was believed limiting to the physical memory size was adequate.  But

obviously this needs to be updated to take into account a 32-bit process

may only be able to address 2 GB.  In the process I've also taken the

opportunity to limit all requests to no more than 75% of physical memory.

Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>

diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c

index 86f82bb..5370ecf 100644

--- a/squashfs-tools/mksquashfs.c

+++ b/squashfs-tools/mksquashfs.c

@@ -304,7 +304,7 @@ void restorefs();

 struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);

 void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);

 unsigned short get_checksum_mem(char *buff, int bytes);

-int get_physical_memory();

+void check_usable_phys_mem(int total_mem);

 void prep_exit()

@@ -4053,11 +4053,7 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,

 		BAD_ERROR("Queue sizes rediculously too large\n");

 	total_mem += fwriteq;

-	if(total_mem > get_physical_memory()) {

-		ERROR("Total queue sizes larger than physical memory.\n");

-		ERROR("Mksquashfs will exhaust physical memory and thrash.\n");

-		BAD_ERROR("Queues too large\n");

-	}

+	check_usable_phys_mem(total_mem);

 	/*

 	 * convert from queue size in Mbytes to queue size in

@@ -4879,6 +4875,72 @@ int get_physical_memory()

 }

+void check_usable_phys_mem(int total_mem)

+{

+	/*

+	 * We want to allow users to use as much of their physical

+	 * memory as they wish.  However, for practical reasons there are

+	 * limits which need to be imposed, to protect users from themselves

+	 * and to prevent people from using Mksquashfs as a DOS attack by using

+	 * all physical memory.   Mksquashfs uses memory to cache data from disk

+	 * to optimise performance.  It is pointless to ask it to use more

+	 * than 75% of physical memory, as this causes thrashing and it is thus

+	 * self-defeating.

+	 */

+	int mem = get_physical_memory();

+

+	mem = (mem >> 1) + (mem >> 2); /* 75% */

+						

+	if(total_mem > mem) {

+		ERROR("Total memory requested is more than 75%% of physical "

+						"memory.\n");

+		ERROR("Mksquashfs uses memory to cache data from disk to "

+						"optimise performance.\n");

+		ERROR("It is pointless to ask it to use more than this amount "

+						"of memory, as this\n");

+		ERROR("causes thrashing and it is thus self-defeating.\n");

+		BAD_ERROR("Requested memory size too large\n");

+	}

+

+	if(sizeof(void *) == 4 && total_mem > 2048) {

+		/*

+		 * If we're running on a kernel with PAE or on a 64-bit kernel,

+		 * then the 75% physical memory limit can still easily exceed

+		 * the addressable memory by this process.

+		 *

+		 * Due to the typical kernel/user-space split (1GB/3GB, or

+		 * 2GB/2GB), we have to conservatively assume the 32-bit

+		 * processes can only address 2-3GB.  So refuse if the user

+		 * tries to allocate more than 2GB.

+		 */

+		ERROR("Total memory requested may exceed maximum "

+				"addressable memory by this process\n");

+		BAD_ERROR("Requested memory size too large\n");

+	}

+}

+

+

+int get_default_phys_mem()

+{

+	int mem = get_physical_memory() / SQUASHFS_TAKE;

+

+	if(sizeof(void *) == 4 && mem > 640) {

+		/*

+		 * If we're running on a kernel with PAE or on a 64-bit kernel,

+		 * the default memory usage can exceed the addressable

+		 * memory by this process.

+		 * Due to the typical kernel/user-space split (1GB/3GB, or

+		 * 2GB/2GB), we have to conservatively assume the 32-bit

+		 * processes can only address 2-3GB.  So limit the  default

+		 * usage to 640M, which gives room for other data.

+		 */

+		mem = 640;

+	}

+

+	return mem;

+}

+

+

 void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,

 							int *fwriteq)

 {

@@ -4890,7 +4952,7 @@ void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,

 #define VERSION() \

-	printf("mksquashfs version 4.3 (2014/05/12)\n");\

+	printf("mksquashfs version 4.3-git (2014/06/09)\n");\

 	printf("copyright (C) 2014 Phillip Lougher "\

 		"<phillip@squashfs.org.uk>\n\n"); \

 	printf("This program is free software; you can redistribute it and/or"\

@@ -4918,7 +4980,7 @@ int main(int argc, char *argv[])

 	int fragq;

 	int bwriteq;

 	int fwriteq;

-	int total_mem = get_physical_memory() / SQUASHFS_TAKE;

+	int total_mem = get_default_phys_mem();

 	int progress = TRUE;

 	int force_progress = FALSE;

 	struct file_buffer **fragment = NULL;

-- 

cgit v0.10.1