From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752376AbdFAQWf (ORCPT ); Thu, 1 Jun 2017 12:22:35 -0400 Received: from mail-eopbgr50126.outbound.protection.outlook.com ([40.107.5.126]:22720 "EHLO EUR03-VE1-obe.outbound.protection.outlook.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1751791AbdFAQWb (ORCPT ); Thu, 1 Jun 2017 12:22:31 -0400 Authentication-Results: linux-foundation.org; dkim=none (message not signed) header.d=none;linux-foundation.org; dmarc=none action=none header.from=virtuozzo.com; From: Andrey Ryabinin To: Andrew Morton CC: Alexander Potapenko , Dmitry Vyukov , , , , Andrey Ryabinin Subject: [PATCH 1/4] mm/kasan: get rid of speculative shadow checks Date: Thu, 1 Jun 2017 19:23:35 +0300 Message-ID: <20170601162338.23540-1-aryabinin@virtuozzo.com> X-Mailer: git-send-email 2.13.0 MIME-Version: 1.0 Content-Type: text/plain X-Originating-IP: [195.214.232.6] X-ClientProxiedBy: VI1P190CA0007.EURP190.PROD.OUTLOOK.COM (10.165.188.148) To DB5PR0801MB2727.eurprd08.prod.outlook.com (10.166.176.23) X-MS-PublicTrafficType: Email X-MS-TrafficTypeDiagnostic: DB5PR0801MB2727: X-MS-Office365-Filtering-Correlation-Id: a3ccca52-e0dc-4b18-a94c-08d4a90a5f9f X-Microsoft-Antispam: UriScan:;BCL:0;PCL:0;RULEID:(22001)(201703131423075)(201703031133081);SRVR:DB5PR0801MB2727; X-Microsoft-Exchange-Diagnostics: 1;DB5PR0801MB2727;3:wrQHJbB7oHe5oIWlSngxMhy/k/zi+zYfpMFEpflKYEOCEVV/k0NdLndvgxG+Ocn2MQM8CSWmfz/L64SvyJj3XHfIRCoZEDXtTXnSAoVG0URCUy6UUndRj0Ey+Vo6yYflgvPltRtBuB3lps6sIL37Irvnu8bcBnh/B4kt/ededeP22lEfe/wjlkStvDwCg1qRuzH+OGFdLPsIXOGzbG3WWdxPBJ1W6JMHuC+CDONmF8qMNj/ZpZ/tnTJ8pGF7nQdCpCFLqNcLSOTzVyw83Af9p/9Y/7pjrRr8DNx1U/CPFxqTsNRgI1Yr/DumHPzfwjmIGBFX4/phgy4kose8YTLEuw==;25:u3rbY8y+pzDBpsse5Y9pZaEifNPdYFnnf13t3mYB4kCAUwqvNr43BQBlZvRiEp8A3LaYbPOo/Xl+aSMjK9pQxhl6ks8t4DXPLE4i8f1EUF0kTyqP7kPRjxIpjoLB9lYR+YEH/zzugaS9vhuz7MSzmVf/IZoYbDxpJ+JUU3oOGiLhW10lxPDHi940vbitQVXQDN04XRdXlFOXhxbwjTIr1hkdMF8Z1+QfKB75rJWD1uVWdCOvfdOvwE8er7o30U27JWkZ8uMariv4HhvT3Dl+ZsrlOCkrtm9yR9mNPPAtXhRdL55k61r32T6wWcZcd4uR7GXB/5uiIl374TCpeNWSHwg7O3MsBrxXYNUDilfxUssZwta8px+y7TAJLt4vDNjOB73bBf2i1FRFiGyE8G9Gx4+prc1lgeVTfclbK7fVkIsP1COLeh6EINXXC2Hg3HgqK4wYESWgtoOOqhdeEysMtrKa4haKitxrjGQ/RDNSSSg= X-Microsoft-Exchange-Diagnostics: 1;DB5PR0801MB2727;31:0gRc3e0kivG4fqs/4d+rGdRKPuJz8iHyvYLHyp3XJ4aTxZI2rj3VvDlQvgmUtW3rQtc5R3tAJguL1TRjajTVrzs1ZxAGwhXIJM0k3rv5008hKI7+YfeM9SmzdcLJ0HlGn21wWmK7Ji7whmbpCxGOpj6uZIJvraQ8n6BIDShXEk8cl5m/Qlr5D6nkQ1orbuFaXCVCwu1+BVvAH2wPtN5iXXW8nb1VSc4ZFwIZOXnhIUk=;20:5sGj9tiy9OMUUl3KcTxWifxIVw628IIsf/M5LVJWvsBZM+T/G/8RTDxwC4CyWco3rrOpkPKvyK1QIhSnnA9theXHAzaNY6BU3ZzNxIA5Q1D7Bs+nb3Mx7+TzTedNvYl6BDl4/tG/OE6uiLjwmQT5+OAlrpb4UoWAugHf94FXENQdg7yoaf1L47u1bNInPuq7SO1OMAPeNMFHx30bLRUNavA/hkoIvX9ZD+CcZMzQvPMzyl2INP+gNkQxVNJq8sqiWSU+Qyzp0pk5DY2cfhXKN6tAGH+eMJRhD5cubENTjoi0nIuPOA7sxgs6WQ3+iEdDxAlyP64p5YzxlWPHktRTjQTq9JbAUVmPsk8e9eWg5wkFb6LzbEsgKKmjxZv6AR/oF4koYbc1jJHclkw2+fxVlUwbqfQozBDabhYbtSZgbnY= X-Microsoft-Antispam-PRVS: X-Exchange-Antispam-Report-Test: UriScan:; X-Exchange-Antispam-Report-CFA-Test: BCL:0;PCL:0;RULEID:(100000700101)(100105000095)(100000701101)(100105300095)(100000702101)(100105100095)(6040450)(601004)(2401047)(5005006)(8121501046)(93006095)(93001095)(100000703101)(100105400095)(10201501046)(3002001)(6041248)(20161123555025)(201703131423075)(201702281528075)(201703061421075)(201703061406153)(20161123558100)(20161123562025)(20161123564025)(20161123560025)(6072148)(100000704101)(100105200095)(100000705101)(100105500095);SRVR:DB5PR0801MB2727;BCL:0;PCL:0;RULEID:(100000800101)(100110000095)(100000801101)(100110300095)(100000802101)(100110100095)(100000803101)(100110400095)(100000804101)(100110200095)(100000805101)(100110500095);SRVR:DB5PR0801MB2727; X-Microsoft-Exchange-Diagnostics: =?us-ascii?Q?1;DB5PR0801MB2727;4:emRxCDsVi1b3d+4woHPjBKRK/JuluOP6Lj+saxQP?= =?us-ascii?Q?EmUwZHRdR/ckyKMbuz+ZyaG4Z9b9+HCjfkCHPt9aL7mydSoMLxHvRDnrWPxW?= =?us-ascii?Q?kRfrWF7lW4DBZTOzWYQM2J4CD5eJbdl1S+Ih/FV5NyMoPRDU4Pax1Vs+9uP8?= =?us-ascii?Q?GbJA/5DxILYXiFWXSgrKJcS6FotOQv4+vPHwf3S5EhuVhlF87HwzgOFdy1KU?= =?us-ascii?Q?tJD/eG76JJibUwRIe/t5fcVdQgC3dJPPBUo7M27lYb72PT5hxL+9Gjt+crVv?= =?us-ascii?Q?qPFqCAgL5zeOphk0Rr5VdCjvhQwIb5IUunRC8zh80jaRUiwmCWS+7g8bp8VY?= =?us-ascii?Q?bA0e9g00hwvG8Clzqw/lCwb6s0aSE2Vp22EzmboT760WHXmpdjCgDRj59tVr?= =?us-ascii?Q?2+AXW9E0coER2fHFgoEDG9i9ty8khJJ6R6/YS0u3TOR8iMtLj7Xg77D7ShYl?= =?us-ascii?Q?YZF8JegAJMJV5pCGAGYr35AzpdOP7baDkK6XGuK0THw9zTNN46Q/YW613FLL?= =?us-ascii?Q?52g76GjUyApGHR9L5vYLjoOr5b93tKaAaU2/Oxg8S/L1L2XDcglJrtMao7H7?= =?us-ascii?Q?h60SN5NMNiVCw8lzLar+y+ndpx1mn/U6UUyjAHvULK6xj3l+jBKW4ffuVZj3?= =?us-ascii?Q?kzkYALw4DxAbKrmHITRBk66HSmQ5QIjT0WzI3lEzcnD9O+wAD3y2gwYKhaua?= =?us-ascii?Q?KAZxqrViJvJVf+EQLwdsDYiJpQm96sVy1ilFtu7qr/uYENgQvBbLX+Oi463S?= =?us-ascii?Q?rUvl+bJhI/4B10RV0jchplpHihOLljLvbjKTCOMIwvi3lbfq6XKR6S7MVbhC?= =?us-ascii?Q?nwwI2u5PCHDoRMi2lAxpGwXLkUfhoWzIYvqNO9amiyeE3JPvMwyBdjht5ELH?= =?us-ascii?Q?9VGPHnohzjz0XbyrbEPyocHctBDZspWYxyvyEM+yTckgTakmn3XJLMSKz2dZ?= =?us-ascii?Q?tm/1Gz+QX+2YWmZkSj7A9N5mDgXfNtepcOp7SV58hSvn2+hlyY2KVVwbMePp?= =?us-ascii?Q?21mr6od9oB5R8JLBbJMbd1I8JBLpHwtznmAVgKQVS5mIb9zlXW4xf9wH3NW9?= =?us-ascii?Q?jZQSCOIYoSEtl0v/qWxIIRwAC5Vur0ybfjeUdLNPb+8xeKlCSMRdzTWD4acl?= =?us-ascii?Q?Skym52ug09hWFKTJph5QGXrW8V8jda+Q?= X-Forefront-PRVS: 0325F6C77B X-Forefront-Antispam-Report: SFV:NSPM;SFS:(10019020)(4630300001)(6069001)(6009001)(39830400002)(39400400002)(39410400002)(39450400003)(81166006)(305945005)(1076002)(6666003)(6916009)(50986999)(5003940100001)(7736002)(36756003)(76506005)(86362001)(42186005)(53416004)(47776003)(5660300001)(66066001)(33646002)(6116002)(3846002)(6506006)(6486002)(48376002)(53936002)(50466002)(110136004)(107886003)(38730400002)(6512007)(189998001)(50226002)(54906002)(8676002)(4326008)(478600001)(2906002)(25786009)(26583001);DIR:OUT;SFP:1102;SCL:1;SRVR:DB5PR0801MB2727;H:localhost.sw.ru;FPR:;SPF:None;MLV:sfv;LANG:en; X-Microsoft-Exchange-Diagnostics: =?us-ascii?Q?1;DB5PR0801MB2727;23:5QYUWQNN/EuoRICsV9Adr/dLbdYrrlXlGpT2cpT?= =?us-ascii?Q?XczmvhVeQRu8pG8iaGSU8bW1zw/4JGeE16qdhai72o+4lNri5Cdepy6bZDGL?= =?us-ascii?Q?FRUQJ/poE4kPtXEU+G2Q9cSHOlKm3E8Nc9UnznaG3+FYIxL0Szj5mveEbqRQ?= =?us-ascii?Q?/OhI7qWhKZ4QDtJGv7wKi1zT0s+J9ZMWYd3TjUapwQ0jfTRuo49UHha+mGgL?= =?us-ascii?Q?AKnid4ER8I0HcB7rpojHr6zTZOAM55XAFPP1p3cTeuGKIfCObzW6vLvOatbP?= =?us-ascii?Q?l1xMGnxzL+7tJj2cn47SLsufQVRhVK12wRSybPAf7fa3/7isnfCCiZz4gNau?= =?us-ascii?Q?W/rc5gzEwJ61ygtmqYjQFpRglwp23zDhuCZG6jJFTfCeBI2bkPTBT5YI3AmZ?= =?us-ascii?Q?aXuVflaM0b4xgAFZWUJomvhj9hfdCLqINQPPN11DhWi8c15QQb521KZnKtXd?= =?us-ascii?Q?HNn274HzcpVho+HKpQ/HjwPcd8zorhV1POMQ65NcfkL0guNEhoecy3ulAkmN?= =?us-ascii?Q?BFTEF6aUH8rd8EyOz12hqq3bsoE4hY0pnJAzA4FyAt0HKv90O7oGtW43oXHo?= =?us-ascii?Q?bQM9l1NRRpFjBEezMKzQp35qy/tTxpamYkaSh+zFTgvvCJbQEeoBCi8r5D4a?= =?us-ascii?Q?Dle0CZAIyiVGmItMnGGJiMJMeIKU9kNE5cmN68AXk4cSFTi90kj8pA7wJ7Jy?= =?us-ascii?Q?Og0n8hn/D6ZWoG5J/JeNnICboIzTw8nUABEHZ8t1j8GXRNM2AiXkyvRPNkah?= =?us-ascii?Q?Vg6UBRQuK2TrYqTq0tysn0x52YMmefScTEw7U2dGpxZOLYBpUyHFXwiMV10g?= =?us-ascii?Q?tF7gacJ9VBJh8zoAzcX5N8yKA7vemQt+6XaxMqA6N+EzYVu4rmsfzv6Pw6it?= =?us-ascii?Q?UiGr4Q7Ghchekydq+5lR4dJaYdDA3WAOw9dyQkOCFNbLTXWaovatcUMgC1mt?= =?us-ascii?Q?HKAgIeVi7E7Y18yq3EHx7VihFgTxxlJME+NidhWkjuRD5xWqNlsZzFatynpy?= =?us-ascii?Q?DUlxnRV9Be06Eio4PlWsRbgqQcP6Pc/mm9mH2O/TDAOi2tw9+d3RB5irA9HB?= =?us-ascii?Q?HyWcCea9Fn5U3z6cG/RDLWEv/fGF0smGUtQFVTLTjVafAcQAZoA=3D=3D?= X-Microsoft-Exchange-Diagnostics: 1;DB5PR0801MB2727;6:jZzNRwS1B0+tHlAgj9zOytIafmuEQFi43s6qQtFf1PfAMD1Tl3gqR80FeMX5Mm2KPoigKfHpFTuGQTtZ+9r5QxvOY5iFxcqQuR50wOoDxsf9WnzenUCQfcInYGcMZaOwQ2DKU7U8GSMGGd6i4XgGCdtJWg47q2/onKnVwURbrrhqnFJvqLcVmBIbdqTdduFTQkzESVS/ZZqJpWOUNNj93VDNxzyCvIX2o5ZUHSB7k+pQFv2LeK/vNcHKF/Ga5ZNdFRbqnOIr5R/CWUNE/nuA5bKmePMDYzlUUMVTLCZexGQNClqKhAU/Rc4AqIdw6CS9EhxY8/11WqU1TUCkxxIHK1JEAqnkWwMwX5w6YVZTx6GEDW+Ive/LexUJnPzJegdopzKKqedV4rE+is2W5WJsxq0YRzIQtHiiFxXe/YTB4OeWeWbT9jGEpJuPToZydc4lxMNd7M63FraJ1d7hYObOz7R7jobTDqi7hciRjICGwpNvsZ8zP6eH7TniAMasM9iFPFNBIjB1hos5hEV3+ul/0Q== X-Microsoft-Exchange-Diagnostics: 1;DB5PR0801MB2727;5:zPAN2rOpZ8WM5jzo0Iz/WbO4gjuVcTjF0Eu8L7BmJkzTTwOTi4ue5RTvYbKR7OIdxL8IbKt9RX8yH8mtWi6Qee5elNZTRRlgrV8elhky2zw3iuLWegGjPSxYrE8o2JYwT1q5u6/12WawVDpJNHNHSj0caIzOVDLaFMa3RWgWVg+0KHdWSSrWt+aETbIAHOa9V/ieRI4O21st+koCKzsIeDPUaUVJ3we4eoJ5BcDv9pw2czr3oiECWsxM5lPVXhrEAdJiAMNEoE3GQsVgtvi77nggarVYrPciQsfrxo+9t7wGa6ZEsBjSaMXbJ9tJi5Vy2ijD4a5BSuEtxfDw+fnifc6nFJ+9+KBZ22XitETYsRm4Wb4fFDu9rcth/YPZwymuKutiSjbyOE10VsddjV6SoKaKgFIeET+4yH5JSdtiVZcmStCk5lh7ymWhKvR9nasoi//cKNLyUX1WQd8utwzgpTChgEB+WqIuz+vLy3FI6Z6v7p1GONRQm4DWs35KZ9kb;24:qljCuA7L2W+/geegb+jYUfMZfPCOvV9P2aJG0fSO0zVA8jrMviUdDOqhBrIAP+IHAEbeXQnN13p8w2L6dOCLF0/Z1XXpEs71JQBqXcGBbIc= SpamDiagnosticOutput: 1:99 SpamDiagnosticMetadata: NSPM X-Microsoft-Exchange-Diagnostics: 1;DB5PR0801MB2727;7:st9OXCrpiwFbYWj2nJg4pnN0YJRfh8jFNin7x5th1C7gACz6DXXr4oBTEYoqJN/YuORq09TeGU8exG3/NS/6vc8jECclFdNeK+vYrQb6lCeUfFRV2Z5Av1ZuQkpy7/1h5hQlZLwYdVBeY5B26GLzSFZWWcwi9JeLqYUlmGubbXHtkBxAmMsS5djqK+p1xf1iQI8ci0wu+wG9LJW36S6fh++HpkfO/M/GRbJ2u8zixSzd81anwWF8BE5yA2aWO4AzYOZIMPH9W9cg6SIrm+2FZ4+2vZcac4jNc6h4GQEyl8roO5WMTjedf+7WR76ipkLC0iPZrXse1uJQlLfrSGkvBg==;20:vYvFHG7P0TDyTec1iaQLG1A/JpB8ItOqAyQxdsPEI0PHuXy1cGUKB2iAj77eGaz+8STYsn4EPbp33BLcO9DI0GVOnpj1NkNGv8n8vFackFrR0UK2Pp1GXGtqYr5dSvTDPYiAJmD1/X+g4czOPn0sNqQCoqS9BXUAtYsugOixO1s= X-OriginatorOrg: virtuozzo.com X-MS-Exchange-CrossTenant-OriginalArrivalTime: 01 Jun 2017 16:22:17.6711 (UTC) X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted X-MS-Exchange-Transport-CrossTenantHeadersStamped: DB5PR0801MB2727 Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org For some unaligned memory accesses we have to check additional byte of the shadow memory. Currently we load that byte speculatively to have only single load + branch on the optimistic fast path. However, this approach have some downsides: - It's unaligned access, so this prevents porting KASAN on architectures which doesn't support unaligned accesses. - We have to map additional shadow page to prevent crash if speculative load happens near the end of the mapped memory. This would significantly complicate upcoming memory hotplug support. I wasn't able to notice any performance degradation with this patch. So these speculative loads is just a pain with no gain, let's remove them. Signed-off-by: Andrey Ryabinin --- mm/kasan/kasan.c | 98 +++++++++----------------------------------------------- 1 file changed, 16 insertions(+), 82 deletions(-) diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c index 85ee45b07615..e6fe07a98677 100644 --- a/mm/kasan/kasan.c +++ b/mm/kasan/kasan.c @@ -134,94 +134,30 @@ static __always_inline bool memory_is_poisoned_1(unsigned long addr) return false; } -static __always_inline bool memory_is_poisoned_2(unsigned long addr) +static __always_inline bool memory_is_poisoned_2_4_8(unsigned long addr, + unsigned long size) { - u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); - - if (unlikely(*shadow_addr)) { - if (memory_is_poisoned_1(addr + 1)) - return true; - - /* - * If single shadow byte covers 2-byte access, we don't - * need to do anything more. Otherwise, test the first - * shadow byte. - */ - if (likely(((addr + 1) & KASAN_SHADOW_MASK) != 0)) - return false; - - return unlikely(*(u8 *)shadow_addr); - } + u8 *shadow_addr = (u8 *)kasan_mem_to_shadow((void *)addr); - return false; -} - -static __always_inline bool memory_is_poisoned_4(unsigned long addr) -{ - u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); - - if (unlikely(*shadow_addr)) { - if (memory_is_poisoned_1(addr + 3)) - return true; - - /* - * If single shadow byte covers 4-byte access, we don't - * need to do anything more. Otherwise, test the first - * shadow byte. - */ - if (likely(((addr + 3) & KASAN_SHADOW_MASK) >= 3)) - return false; - - return unlikely(*(u8 *)shadow_addr); - } - - return false; -} - -static __always_inline bool memory_is_poisoned_8(unsigned long addr) -{ - u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); - - if (unlikely(*shadow_addr)) { - if (memory_is_poisoned_1(addr + 7)) - return true; - - /* - * If single shadow byte covers 8-byte access, we don't - * need to do anything more. Otherwise, test the first - * shadow byte. - */ - if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) - return false; - - return unlikely(*(u8 *)shadow_addr); - } + /* + * Access crosses 8(shadow size)-byte boundary. Such access maps + * into 2 shadow bytes, so we need to check them both. + */ + if (unlikely(((addr + size - 1) & KASAN_SHADOW_MASK) < size - 1)) + return *shadow_addr || memory_is_poisoned_1(addr + size - 1); - return false; + return memory_is_poisoned_1(addr + size - 1); } static __always_inline bool memory_is_poisoned_16(unsigned long addr) { - u32 *shadow_addr = (u32 *)kasan_mem_to_shadow((void *)addr); - - if (unlikely(*shadow_addr)) { - u16 shadow_first_bytes = *(u16 *)shadow_addr; - - if (unlikely(shadow_first_bytes)) - return true; - - /* - * If two shadow bytes covers 16-byte access, we don't - * need to do anything more. Otherwise, test the last - * shadow byte. - */ - if (likely(IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) - return false; + u16 *shadow_addr = (u16 *)kasan_mem_to_shadow((void *)addr); - return memory_is_poisoned_1(addr + 15); - } + /* Unaligned 16-bytes access maps into 3 shadow bytes. */ + if (unlikely(!IS_ALIGNED(addr, KASAN_SHADOW_SCALE_SIZE))) + return *shadow_addr || memory_is_poisoned_1(addr + 15); - return false; + return *shadow_addr; } static __always_inline unsigned long bytes_is_nonzero(const u8 *start, @@ -292,11 +228,9 @@ static __always_inline bool memory_is_poisoned(unsigned long addr, size_t size) case 1: return memory_is_poisoned_1(addr); case 2: - return memory_is_poisoned_2(addr); case 4: - return memory_is_poisoned_4(addr); case 8: - return memory_is_poisoned_8(addr); + return memory_is_poisoned_2_4_8(addr, size); case 16: return memory_is_poisoned_16(addr); default: -- 2.13.0