Re: [PATCH v12 12/14] x86/sgx: Turn on per-cgroup EPC reclamation

["Huang, Kai" <kai.huang@intel.com>]

> +/*
> + * Get the per-cgroup or global LRU list that tracks the given reclaimable page.
> + */
>  static inline struct sgx_epc_lru_list *sgx_lru_list(struct sgx_epc_page *epc_page)
>  {
> +#ifdef CONFIG_CGROUP_MISC
> +	/*
> +	 * epc_page->sgx_cg here is never NULL during a reclaimable epc_page's
> +	 * life between sgx_alloc_epc_page() and sgx_free_epc_page():
> +	 *
> +	 * In sgx_alloc_epc_page(), epc_page->sgx_cg is set to the return from
> +	 * sgx_get_current_cg() which is the misc cgroup of the current task, or
> +	 * the root by default even if the misc cgroup is disabled by kernel
> +	 * command line.
> +	 *
> +	 * epc_page->sgx_cg is only unset by sgx_free_epc_page().
> +	 *
> +	 * This function is never used before sgx_alloc_epc_page() or after
> +	 * sgx_free_epc_page().
> +	 */
> +	return &epc_page->sgx_cg->lru;
> +#else
>  	return &sgx_global_lru;
> +#endif
>  }
>  
>  /*
> @@ -42,7 +63,8 @@ static inline struct sgx_epc_lru_list *sgx_lru_list(struct sgx_epc_page *epc_pag
>   */
>  static inline bool sgx_can_reclaim(void)
>  {
> -	return !list_empty(&sgx_global_lru.reclaimable);
> +	return !sgx_cgroup_lru_empty(misc_cg_root()) ||
> +	       !list_empty(&sgx_global_lru.reclaimable);
>  }

Shouldn't this be:

	if (IS_ENABLED(CONFIG_CGROUP_MISC))
		return !sgx_cgroup_lru_empty(misc_cg_root());
	else
		return !list_empty(&sgx_global_lru.reclaimable);
?

In this way, it is consistent with the sgx_reclaim_pages_global() below.

>  
>  static atomic_long_t sgx_nr_free_pages = ATOMIC_LONG_INIT(0);
> @@ -404,7 +426,10 @@ static bool sgx_should_reclaim(unsigned long watermark)
>  
>  static void sgx_reclaim_pages_global(struct mm_struct *charge_mm)
>  {
> -	sgx_reclaim_pages(&sgx_global_lru, charge_mm);
> +	if (IS_ENABLED(CONFIG_CGROUP_MISC))
> +		sgx_cgroup_reclaim_pages(misc_cg_root(), charge_mm);
> +	else
> +		sgx_reclaim_pages(&sgx_global_lru, charge_mm);
>  }
>  
>  /*
> @@ -414,6 +439,14 @@ static void sgx_reclaim_pages_global(struct mm_struct *charge_mm)
>   */
>  void sgx_reclaim_direct(void)
>  {
> +	struct sgx_cgroup *sgx_cg = sgx_get_current_cg();
> +
> +	/* Make sure there are some free pages at cgroup level */
> +	if (sgx_cg && sgx_cgroup_should_reclaim(sgx_cg)) {
> +		sgx_cgroup_reclaim_pages(misc_from_sgx(sgx_cg), current->mm);
> +		sgx_put_cg(sgx_cg);
> +	}

Empty line.

> +	/* Make sure there are some free pages at global level */
>  	if (sgx_should_reclaim(SGX_NR_LOW_PAGES))

Looking at the code, to me sgx_should_reclaim() is a little bit vague
because from the name we don't know whether it interally checks the
current cgroup or the global.  

It's better to rename to sgx_should_reclaim_global().

Ditto for sgx_can_reclaim() -> sgx_can_reclaim_global().

And I think you can do the renaming in the previous patch, because in the
changelog of your previous patch, it seems you have called out the two
functions are for global reclaim.

>  		sgx_reclaim_pages_global(current->mm);
>  }
> @@ -616,6 +649,12 @@ struct sgx_epc_page *sgx_alloc_epc_page(void *owner, enum sgx_reclaim reclaim)
>  			break;
>  		}
>  
> +		/*
> +		 * At this point, the usage within this cgroup is under its
> +		 * limit but there is no physical page left for allocation.
> +		 * Perform a global reclaim to get some pages released from any
> +		 * cgroup with reclaimable pages.
> +		 */
>  		sgx_reclaim_pages_global(current->mm);
>  		cond_resched();
>  	}