[PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Miroslav Rezanina]

Hi,

when running linux as XEN guest and use boot parameter mem= to set memory lower then is assigned to guest, not used memory should be returned to hypervisor as free. This is working with kernel available on xen.org pages, but is not working with kernel 2.6.29. Comparing both kernels I found code for returning unused memory to hypervisor is missing. Following patch add this functionality to 2.6.29 kernel.

Miroslav Rezanina <mrezanin@redhat.com>
--
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 6a8811a..fd6b0e7 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
 struct boot_params boot_params;
 #endif
 
+#ifdef CONFIG_XEN
+void __init xen_return_unused_mem(void);
+#endif
+
 /*
  * Machine setup..
  */
@@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	paravirt_pagetable_setup_done(swapper_pg_dir);
 	paravirt_post_allocator_init();
+#ifdef CONFIG_XEN
+	xen_return_unused_mem();
+#endif
 
 #ifdef CONFIG_X86_64
 	map_vsyscall();
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 15c6c68..bc5d2bc 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -20,6 +20,7 @@
 #include <xen/page.h>
 #include <xen/interface/callback.h>
 #include <xen/interface/physdev.h>
+#include <xen/interface/memory.h>
 #include <xen/features.h>
 
 #include "xen-ops.h"
@@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
 
 
 /**
+ * Author: Miroslav Rezanina <mrezanin@redhat.com>
+ * Function retuns unused memory to hypevisor
+ **/
+void __init xen_return_unused_mem(void)
+{
+	if (xen_start_info->nr_pages > max_pfn) {
+        /*
+	* the max_pfn was shrunk (probably by mem=
+	* kernel parameter); shrink reservation with the HV
+	*/
+	struct xen_memory_reservation reservation = {
+	.address_bits = 0,
+	.extent_order = 0,
+	.domid = DOMID_SELF
+	};
+	unsigned int difference;
+	int ret;
+
+	difference = xen_start_info->nr_pages - max_pfn;
+
+	set_xen_guest_handle(reservation.extent_start,
+		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
+	reservation.nr_extents = difference;
+	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
+		&reservation);
+	BUG_ON (ret != difference);
+	}
+}
+
+/**
  * machine_specific_memory_setup - Hook for machine specific memory setup.
  **/
 

Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Jeremy Fitzhardinge]

On 08/19/09 06:05, Miroslav Rezanina wrote:
> when running linux as XEN guest and use boot parameter mem= to set memory lower then is assigned to guest, not used memory should be returned to hypervisor as free. This is working with kernel available on xen.org pages, but is not working with kernel 2.6.29. Comparing both kernels I found code for returning unused memory to hypervisor is missing. Following patch add this functionality to 2.6.29 kernel.
>   

The idea is sound, but I think it might be better to walk the e820
table, and remove any memory ranges which aren't marked as E820_RAM. 
That makes it possible to carve holes in the address space as well as
simply truncate it.

Also, something appears to have smashed your indentation.

    J

> Miroslav Rezanina <mrezanin@redhat.com>
> --
> diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
> index 6a8811a..fd6b0e7 100644
> --- a/arch/x86/kernel/setup.c
> +++ b/arch/x86/kernel/setup.c
> @@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
>  struct boot_params boot_params;
>  #endif
>  
> +#ifdef CONFIG_XEN
> +void __init xen_return_unused_mem(void);
> +#endif
> +
>  /*
>   * Machine setup..
>   */
> @@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
>  	paging_init();
>  	paravirt_pagetable_setup_done(swapper_pg_dir);
>  	paravirt_post_allocator_init();
> +#ifdef CONFIG_XEN
> +	xen_return_unused_mem();
> +#endif
>  
>  #ifdef CONFIG_X86_64
>  	map_vsyscall();
> diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
> index 15c6c68..bc5d2bc 100644
> --- a/arch/x86/xen/setup.c
> +++ b/arch/x86/xen/setup.c
> @@ -20,6 +20,7 @@
>  #include <xen/page.h>
>  #include <xen/interface/callback.h>
>  #include <xen/interface/physdev.h>
> +#include <xen/interface/memory.h>
>  #include <xen/features.h>
>  
>  #include "xen-ops.h"
> @@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
>  
>  
>  /**
> + * Author: Miroslav Rezanina <mrezanin@redhat.com>
> + * Function retuns unused memory to hypevisor
> + **/
> +void __init xen_return_unused_mem(void)
> +{
> +	if (xen_start_info->nr_pages > max_pfn) {
> +        /*
> +	* the max_pfn was shrunk (probably by mem=
> +	* kernel parameter); shrink reservation with the HV
> +	*/
> +	struct xen_memory_reservation reservation = {
> +	.address_bits = 0,
> +	.extent_order = 0,
> +	.domid = DOMID_SELF
> +	};
> +	unsigned int difference;
> +	int ret;
> +
> +	difference = xen_start_info->nr_pages - max_pfn;
> +
> +	set_xen_guest_handle(reservation.extent_start,
> +		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
> +	reservation.nr_extents = difference;
> +	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
> +		&reservation);
> +	BUG_ON (ret != difference);
> +	}
> +}
> +
> +/**
>   * machine_specific_memory_setup - Hook for machine specific memory setup.
>   **/
>  
>
>   

Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Miroslav Rezanina]

----- Original Message -----
From: "Jeremy Fitzhardinge" <jeremy@goop.org>
To: "Miroslav Rezanina" <mrezanin@redhat.com>
Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com, "Gianluca Guida" <gianluca.guida@citrix.com>
Sent: Wednesday, August 19, 2009 6:16:33 PM GMT +01:00 Amsterdam / Berlin / Bern / Rome / Stockholm / Vienna
Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

>>On 08/19/09 06:05, Miroslav Rezanina wrote:
>> when running linux as XEN guest and use boot parameter mem= to set
>> memory lower then is assigned to guest, not used memory should be 
>> returned to hypervisor as free. This is working with kernel 
>> available on xen.org pages, but is not working with kernel 2.6.29.
>> Comparing both kernels I found code for returning unused memory to 
>> hypervisor is missing. Following patch add this functionality to 
>>2.6.29 kernel.
>>  
>
> The idea is sound, but I think it might be better to walk the e820
> table, and remove any memory ranges which aren't marked as E820_RAM. 
> That makes it possible to carve holes in the address space as well as
> simply truncate it.

Hi Jeremy,
there is handled e820 map in guest. However, this patch informs 
hypervisor, that guest uses less memory than was assigned to it. 
If hypervisor is not informed, memory is reserved for guest that
do not need it. If hypervisor is informed, he decrease memory
reservation for guest and unused memory is marked as free 
for use by other guests.

Mirek

> Also, something appears to have smashed your indentation.
>
>    J

Oh, something goes wrong. Resending patch:

diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 6a8811a..fd6b0e7 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
 struct boot_params boot_params;
 #endif
 
+#ifdef CONFIG_XEN
+void __init xen_return_unused_mem(void);
+#endif
+
 /*
  * Machine setup..
  */
@@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 	paravirt_pagetable_setup_done(swapper_pg_dir);
 	paravirt_post_allocator_init();
+#ifdef CONFIG_XEN
+	xen_return_unused_mem();
+#endif
 
 #ifdef CONFIG_X86_64
 	map_vsyscall();
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 15c6c68..bc5d2bc 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -20,6 +20,7 @@
 #include <xen/page.h>
 #include <xen/interface/callback.h>
 #include <xen/interface/physdev.h>
+#include <xen/interface/memory.h>
 #include <xen/features.h>
 
 #include "xen-ops.h"
@@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
 
 
 /**
+ * Author: Miroslav Rezanina <mrezanin@redhat.com>
+ * Function retuns unused memory to hypevisor
+ **/
+void __init xen_return_unused_mem(void)
+{
+	if (xen_start_info->nr_pages > max_pfn) {
+        /*
+	* the max_pfn was shrunk (probably by mem=
+	* kernel parameter); shrink reservation with the HV
+	*/
+	struct xen_memory_reservation reservation = {
+	.address_bits = 0,
+	.extent_order = 0,
+	.domid = DOMID_SELF
+	};
+	unsigned int difference;
+	int ret;
+
+	difference = xen_start_info->nr_pages - max_pfn;
+
+	set_xen_guest_handle(reservation.extent_start,
+		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
+	reservation.nr_extents = difference;
+	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
+		&reservation);
+	BUG_ON (ret != difference);
+	}
+}
+
+/**
  * machine_specific_memory_setup - Hook for machine specific memory setup.
  **/
 

Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Jeremy Fitzhardinge]

On 08/20/09 00:47, Miroslav Rezanina wrote:
> there is handled e820 map in guest. However, this patch informs 
> hypervisor, that guest uses less memory than was assigned to it. 
> If hypervisor is not informed, memory is reserved for guest that
> do not need it. If hypervisor is informed, he decrease memory
> reservation for guest and unused memory is marked as free 
> for use by other guests.
>   

Yes.  But the guest will modify its own e820 map for a number of
reasons; for example: reducing its own memory, or clearing a space for
the PCI hole.  In general we want to free any underlying pages which
don't correspond to E820_RAM regions.

    J

Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Miroslav Rezanina]

----- "Jeremy Fitzhardinge" <jeremy@goop.org> wrote:

> From: "Jeremy Fitzhardinge" <jeremy@goop.org>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com, "Gianluca Guida" <gianluca.guida@citrix.com>
> Sent: Thursday, August 20, 2009 6:39:02 PM GMT +01:00 Amsterdam / Berlin / Bern / Rome / Stockholm / Vienna
> Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor
>
> On 08/20/09 00:47, Miroslav Rezanina wrote:
> > there is handled e820 map in guest. However, this patch informs 
> > hypervisor, that guest uses less memory than was assigned to it. 
> > If hypervisor is not informed, memory is reserved for guest that
> > do not need it. If hypervisor is informed, he decrease memory
> > reservation for guest and unused memory is marked as free 
> > for use by other guests.
> >   
> 
> Yes.  But the guest will modify its own e820 map for a number of
> reasons; for example: reducing its own memory, or clearing a space
> for
> the PCI hole.  In general we want to free any underlying pages which
> don't correspond to E820_RAM regions.
> 
>     J
> --

Hi Jeremy,
can you give me a practical example, where e820 map can have "hole" inside,
i.e. there will be block of memory not listed in e820 map that have listed 
memory before and after it?
As I checked the source code, there is always removed memory from some point
till end of map, not from one adress till another. And I can't image how would
be such a case handled. Of course, there can be some special regions, as the PCI
hole, but these are marked as "Reserved".
There can be "reserved and returned" some inside memory, but this is already
handled  by balloon driver. My patch returns memory that this driver can't use.
Regards,
Mirek
-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel

Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Jeremy Fitzhardinge]

On 09/07/09 05:41, Miroslav Rezanina wrote:
> can you give me a practical example, where e820 map can have "hole" inside,
> i.e. there will be block of memory not listed in e820 map that have listed 
> memory before and after it?
> As I checked the source code, there is always removed memory from some point
> till end of map, not from one adress till another. And I can't image how would
> be such a case handled. Of course, there can be some special regions, as the PCI
> hole, but these are marked as "Reserved".
> There can be "reserved and returned" some inside memory, but this is already
> handled  by balloon driver. My patch returns memory that this driver can't use.
>   

PCI memory isn't typically reserved; there's just a hole in the address
space for it.  Its up to the BIOS/OS to set the BARs for the devices
into that hole.  Reserved E820 regions are just that - reserved by the
BIOS for its own purposes.

In a Xen domain, the E820 tables are purely synthesized by the kernel's
Xen code, and can have any content we like.  At the moment they tend to
be very simple with linear memory up to the domain's memory size.  For
domains with PCI access - either dom0 or with pcifront - we want to be
able to carve out regions from that memory to place devices.  We may
want to manipulate the E820 tables to reshape memory for other reasons
(like pre-ballooned memory, or memory hotplug).

What I'm looking for is a nice general purpose routine which will walk a
set of E820 entries and release back to Xen any memory which is in an
E820 hole (ie, in a gap between or after E820 entries).  That solves the
simple problem of trimming the end of the memory map, but also copes
with more complex cases that will arise.

    J

[Xen-devel] [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Gianluca Guida]

Miroslav Rezanina writes:
 > Hi,
 > 
 > when running linux as XEN guest and use boot parameter mem= to set memory lower then is assigned to guest, not used memory should be returned to hypervisor as free. This is working with kernel available on xen.org pages, but is not working with kernel 2.6.29. Comparing both kernels I found code for returning unused memory to hypervisor is missing. Following patch add this functionality to 2.6.29 kernel.
 > 

A good idea would be to avoid putting this code in the generic kernel
code. For now just placing it in at the end of Xen's post-allocator
init would make it completely transparent to the non-xen kernel.

There's a patch in Jeremy's rebase/master (at the moment reverted)
that allows guest to boot ballooned (which is the opposite of this
case, roughly) which does this kind of calculations while walking the
e820 table. Perhaps this can be moved there after things get to work
again.

Thanks,
Gianluca

 > Miroslav Rezanina <mrezanin@redhat.com>
 > --
 > diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
 > index 6a8811a..fd6b0e7 100644
 > --- a/arch/x86/kernel/setup.c
 > +++ b/arch/x86/kernel/setup.c
 > @@ -118,6 +118,10 @@ struct boot_params __initdata boot_params;
 >  struct boot_params boot_params;
 >  #endif
 >  
 > +#ifdef CONFIG_XEN
 > +void __init xen_return_unused_mem(void);
 > +#endif
 > +
 >  /*
 >   * Machine setup..
 >   */
 > @@ -920,6 +924,9 @@ void __init setup_arch(char **cmdline_p)
 >  	paging_init();
 >  	paravirt_pagetable_setup_done(swapper_pg_dir);
 >  	paravirt_post_allocator_init();
 > +#ifdef CONFIG_XEN
 > +	xen_return_unused_mem();
 > +#endif
 >  
 >  #ifdef CONFIG_X86_64
 >  	map_vsyscall();
 > diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
 > index 15c6c68..bc5d2bc 100644
 > --- a/arch/x86/xen/setup.c
 > +++ b/arch/x86/xen/setup.c
 > @@ -20,6 +20,7 @@
 >  #include <xen/page.h>
 >  #include <xen/interface/callback.h>
 >  #include <xen/interface/physdev.h>
 > +#include <xen/interface/memory.h>
 >  #include <xen/features.h>
 >  
 >  #include "xen-ops.h"
 > @@ -34,6 +35,36 @@ extern void xen_syscall32_target(void);
 >  
 >  
 >  /**
 > + * Author: Miroslav Rezanina <mrezanin@redhat.com>
 > + * Function retuns unused memory to hypevisor
 > + **/
 > +void __init xen_return_unused_mem(void)
 > +{
 > +	if (xen_start_info->nr_pages > max_pfn) {
 > +        /*
 > +	* the max_pfn was shrunk (probably by mem=
 > +	* kernel parameter); shrink reservation with the HV
 > +	*/
 > +	struct xen_memory_reservation reservation = {
 > +	.address_bits = 0,
 > +	.extent_order = 0,
 > +	.domid = DOMID_SELF
 > +	};
 > +	unsigned int difference;
 > +	int ret;
 > +
 > +	difference = xen_start_info->nr_pages - max_pfn;
 > +
 > +	set_xen_guest_handle(reservation.extent_start,
 > +		((unsigned long *)xen_start_info->mfn_list) + max_pfn);
 > +	reservation.nr_extents = difference;
 > +	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
 > +		&reservation);
 > +	BUG_ON (ret != difference);
 > +	}
 > +}
 > +
 > +/**
 >   * machine_specific_memory_setup - Hook for machine specific memory setup.
 >   **/
 >  
 > 
 > _______________________________________________
 > Xen-devel mailing list
 > Xen-devel@lists.xensource.com
 > http://lists.xensource.com/xen-devel

Re: [PATCH v2][v2.6.29][XEN] Return unused memory to hypervisor

[Miroslav Rezanina]

----- Original Message -----
From: "Gianluca Guida" <gianluca.guida@citrix.com>
To: "Miroslav Rezanina" <mrezanin@redhat.com>
Cc: jeremy@goop.org, xen-devel@lists.xensource.com, linux-kernel@vger.kernel.org
Sent: Thursday, August 20, 2009 11:31:34 AM GMT +01:00 Amsterdam / Berlin / Bern / Rome / Stockholm / Vienna
Subject: [Xen-devel] [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

Miroslav Rezanina writes:
> > Hi,
> > 
> > when running linux as XEN guest and use boot parameter mem= to set memory lower then is assigned to guest, not used memory should be returned to hypervisor as free. This is working with kernel available on xen.org pages, but is not working with kernel 2.6.29. Comparing both kernels I found code for returning unused memory to hypervisor is missing. Following patch add this functionality to 2.6.29 kernel.
> > 
>
> A good idea would be to avoid putting this code in the generic kernel
> code. For now just placing it in at the end of Xen's post-allocator
> init would make it completely transparent to the non-xen kernel.
>

Hi Gianluca,
good point. You're right. I moved calling to xen_post_allocator_init function. 
It is better place for it.

Regards,
Mirek

----diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index b58e963..2a9cc80 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -28,6 +28,7 @@
 #include <linux/console.h>
 
 #include <xen/interface/xen.h>
+#include <xen/interface/memory.h>
 #include <xen/interface/version.h>
 #include <xen/interface/physdev.h>
 #include <xen/interface/vcpu.h>
@@ -122,6 +123,36 @@ static int have_vcpu_info_placement =
 #endif
 	;
 
+/**
+ *  * Author: Miroslav Rezanina <mrezanin@redhat.com>
+ *   * Function retuns unused memory to hypevisor
+ *    **/
+void __init xen_return_unused_mem(void)
+{
+	if (xen_start_info->nr_pages > max_pfn) {
+	/*
+	 * the max_pfn was shrunk (probably by mem=
+         * kernel parameter); shrink reservation with the HV
+	 */
+		struct xen_memory_reservation reservation = {
+			.address_bits = 0,
+			.extent_order = 0,
+			.domid = DOMID_SELF
+		};
+		unsigned int difference;
+		int ret;
+
+		difference = xen_start_info->nr_pages - max_pfn;
+
+		set_xen_guest_handle(reservation.extent_start,
+			((unsigned long *)xen_start_info->mfn_list) + max_pfn);
+		reservation.nr_extents = difference;
+		ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation,
+			&reservation);
+		BUG_ON (ret != difference);
+	}
+}
+
 
 static void xen_vcpu_setup(int cpu)
 {
@@ -1057,6 +1088,8 @@ static __init void xen_post_allocator_init(void)
 	SetPagePinned(virt_to_page(level3_user_vsyscall));
 #endif
 	xen_mark_init_mm_pinned();
+
+	xen_return_unused_mem();
 }
 
 /* This is called once we have the cpu_possible_map */

Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Jeremy Fitzhardinge]

On 08/19/09 06:05, Miroslav Rezanina wrote:
> when running linux as XEN guest and use boot parameter mem= to set memory lower then is assigned to guest, not used memory should be returned to hypervisor as free. This is working with kernel available on xen.org pages, but is not working with kernel 2.6.29. Comparing both kernels I found code for returning unused memory to hypervisor is missing. Following patch add this functionality to 2.6.29 kernel.
>   

Are you planning on submitting a revised patch along the lines I suggested?

Thanks,
    J

Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor

[Miroslav Rezanina]

----- "Jeremy Fitzhardinge" <jeremy@goop.org> wrote:

> From: "Jeremy Fitzhardinge" <jeremy@goop.org>
> To: "Miroslav Rezanina" <mrezanin@redhat.com>
> Cc: linux-kernel@vger.kernel.org, xen-devel@lists.xensource.com
> Sent: Friday, September 4, 2009 1:26:24 AM GMT +01:00 Amsterdam / Berlin / Bern / Rome / Stockholm / Vienna
> Subject: Re: [PATCH][v2.6.29][XEN] Return unused memory to hypervisor
>
> On 08/19/09 06:05, Miroslav Rezanina wrote:
> > when running linux as XEN guest and use boot parameter mem= to set
> memory lower then is assigned to guest, not used memory should be
> returned to hypervisor as free. This is working with kernel available
> on xen.org pages, but is not working with kernel 2.6.29. Comparing
> both kernels I found code for returning unused memory to hypervisor is
> missing. Following patch add this functionality to 2.6.29 kernel.
> >   
> 
> Are you planning on submitting a revised patch along the lines I
> suggested?
> 
> Thanks,
>     J

Hi Jeremy,
yes, I'm planning so. Unfortunatelly, I had some more important tasks to do
so I was not able to rewrite patch as you suggested.

Mirek
> --
> To unsubscribe from this list: send the line "unsubscribe
> linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

-- 
Miroslav Rezanina
Software Engineer - Virtualization Team - XEN kernel