1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2020 Google LLC 4 */ 5#define _GNU_SOURCE 6 7#include <errno.h> 8#include <stdlib.h> 9#include <stdio.h> 10#include <string.h> 11#include <sys/mman.h> 12#include <time.h> 13#include <stdbool.h> 14 15#include "../kselftest.h" 16 17#define EXPECT_SUCCESS 0 18#define EXPECT_FAILURE 1 19#define NON_OVERLAPPING 0 20#define OVERLAPPING 1 21#define NS_PER_SEC 1000000000ULL 22#define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */ 23#define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ 24 25#define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) 26#define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) 27#define SIZE_MB(m) ((size_t)m * (1024 * 1024)) 28#define SIZE_KB(k) ((size_t)k * 1024) 29 30struct config { 31 unsigned long long src_alignment; 32 unsigned long long dest_alignment; 33 unsigned long long region_size; 34 int overlapping; 35 int dest_preamble_size; 36}; 37 38struct test { 39 const char *name; 40 struct config config; 41 int expect_failure; 42}; 43 44enum { 45 _1KB = 1ULL << 10, /* 1KB -> not page aligned */ 46 _4KB = 4ULL << 10, 47 _8KB = 8ULL << 10, 48 _1MB = 1ULL << 20, 49 _2MB = 2ULL << 20, 50 _4MB = 4ULL << 20, 51 _5MB = 5ULL << 20, 52 _1GB = 1ULL << 30, 53 _2GB = 2ULL << 30, 54 PMD = _2MB, 55 PUD = _1GB, 56}; 57 58#define PTE page_size 59 60#define MAKE_TEST(source_align, destination_align, size, \ 61 overlaps, should_fail, test_name) \ 62(struct test){ \ 63 .name = test_name, \ 64 .config = { \ 65 .src_alignment = source_align, \ 66 .dest_alignment = destination_align, \ 67 .region_size = size, \ 68 .overlapping = overlaps, \ 69 }, \ 70 .expect_failure = should_fail \ 71} 72 73/* compute square root using binary search */ 74static unsigned long get_sqrt(unsigned long val) 75{ 76 unsigned long low = 1; 77 78 /* assuming rand_size is less than 1TB */ 79 unsigned long high = (1UL << 20); 80 81 while (low <= high) { 82 unsigned long mid = low + (high - low) / 2; 83 unsigned long temp = mid * mid; 84 85 if (temp == val) 86 return mid; 87 if (temp < val) 88 low = mid + 1; 89 high = mid - 1; 90 } 91 return low; 92} 93 94/* 95 * Returns false if the requested remap region overlaps with an 96 * existing mapping (e.g text, stack) else returns true. 97 */ 98static bool is_remap_region_valid(void *addr, unsigned long long size) 99{ 100 void *remap_addr = NULL; 101 bool ret = true; 102 103 /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */ 104 remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE, 105 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 106 -1, 0); 107 108 if (remap_addr == MAP_FAILED) { 109 if (errno == EEXIST) 110 ret = false; 111 } else { 112 munmap(remap_addr, size); 113 } 114 115 return ret; 116} 117 118/* Returns mmap_min_addr sysctl tunable from procfs */ 119static unsigned long long get_mmap_min_addr(void) 120{ 121 FILE *fp; 122 int n_matched; 123 static unsigned long long addr; 124 125 if (addr) 126 return addr; 127 128 fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); 129 if (fp == NULL) { 130 ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", 131 strerror(errno)); 132 exit(KSFT_SKIP); 133 } 134 135 n_matched = fscanf(fp, "%llu", &addr); 136 if (n_matched != 1) { 137 ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", 138 strerror(errno)); 139 fclose(fp); 140 exit(KSFT_SKIP); 141 } 142 143 fclose(fp); 144 return addr; 145} 146 147/* 148 * Using /proc/self/maps, assert that the specified address range is contained 149 * within a single mapping. 150 */ 151static bool is_range_mapped(FILE *maps_fp, unsigned long start, 152 unsigned long end) 153{ 154 char *line = NULL; 155 size_t len = 0; 156 bool success = false; 157 unsigned long first_val, second_val; 158 159 rewind(maps_fp); 160 161 while (getline(&line, &len, maps_fp) != -1) { 162 if (sscanf(line, "%lx-%lx", &first_val, &second_val) != 2) { 163 ksft_exit_fail_msg("cannot parse /proc/self/maps\n"); 164 break; 165 } 166 167 if (first_val <= start && second_val >= end) { 168 success = true; 169 break; 170 } 171 } 172 173 return success; 174} 175 176/* 177 * Returns the start address of the mapping on success, else returns 178 * NULL on failure. 179 */ 180static void *get_source_mapping(struct config c) 181{ 182 unsigned long long addr = 0ULL; 183 void *src_addr = NULL; 184 unsigned long long mmap_min_addr; 185 186 mmap_min_addr = get_mmap_min_addr(); 187 /* 188 * For some tests, we need to not have any mappings below the 189 * source mapping. Add some headroom to mmap_min_addr for this. 190 */ 191 mmap_min_addr += 10 * _4MB; 192 193retry: 194 addr += c.src_alignment; 195 if (addr < mmap_min_addr) 196 goto retry; 197 198 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE, 199 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 200 -1, 0); 201 if (src_addr == MAP_FAILED) { 202 if (errno == EPERM || errno == EEXIST) 203 goto retry; 204 goto error; 205 } 206 /* 207 * Check that the address is aligned to the specified alignment. 208 * Addresses which have alignments that are multiples of that 209 * specified are not considered valid. For instance, 1GB address is 210 * 2MB-aligned, however it will not be considered valid for a 211 * requested alignment of 2MB. This is done to reduce coincidental 212 * alignment in the tests. 213 */ 214 if (((unsigned long long) src_addr & (c.src_alignment - 1)) || 215 !((unsigned long long) src_addr & c.src_alignment)) { 216 munmap(src_addr, c.region_size); 217 goto retry; 218 } 219 220 if (!src_addr) 221 goto error; 222 223 return src_addr; 224error: 225 ksft_print_msg("Failed to map source region: %s\n", 226 strerror(errno)); 227 return NULL; 228} 229 230/* 231 * This test validates that merge is called when expanding a mapping. 232 * Mapping containing three pages is created, middle page is unmapped 233 * and then the mapping containing the first page is expanded so that 234 * it fills the created hole. The two parts should merge creating 235 * single mapping with three pages. 236 */ 237static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size) 238{ 239 char *test_name = "mremap expand merge"; 240 bool success = false; 241 char *remap, *start; 242 243 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 244 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 245 246 if (start == MAP_FAILED) { 247 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 248 goto out; 249 } 250 251 munmap(start + page_size, page_size); 252 remap = mremap(start, page_size, 2 * page_size, 0); 253 if (remap == MAP_FAILED) { 254 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 255 munmap(start, page_size); 256 munmap(start + 2 * page_size, page_size); 257 goto out; 258 } 259 260 success = is_range_mapped(maps_fp, (unsigned long)start, 261 (unsigned long)(start + 3 * page_size)); 262 munmap(start, 3 * page_size); 263 264out: 265 if (success) 266 ksft_test_result_pass("%s\n", test_name); 267 else 268 ksft_test_result_fail("%s\n", test_name); 269} 270 271/* 272 * Similar to mremap_expand_merge() except instead of removing the middle page, 273 * we remove the last then attempt to remap offset from the second page. This 274 * should result in the mapping being restored to its former state. 275 */ 276static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) 277{ 278 279 char *test_name = "mremap expand merge offset"; 280 bool success = false; 281 char *remap, *start; 282 283 start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, 284 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 285 286 if (start == MAP_FAILED) { 287 ksft_print_msg("mmap failed: %s\n", strerror(errno)); 288 goto out; 289 } 290 291 /* Unmap final page to ensure we have space to expand. */ 292 munmap(start + 2 * page_size, page_size); 293 remap = mremap(start + page_size, page_size, 2 * page_size, 0); 294 if (remap == MAP_FAILED) { 295 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 296 munmap(start, 2 * page_size); 297 goto out; 298 } 299 300 success = is_range_mapped(maps_fp, (unsigned long)start, 301 (unsigned long)(start + 3 * page_size)); 302 munmap(start, 3 * page_size); 303 304out: 305 if (success) 306 ksft_test_result_pass("%s\n", test_name); 307 else 308 ksft_test_result_fail("%s\n", test_name); 309} 310 311/* 312 * Verify that an mremap within a range does not cause corruption 313 * of unrelated part of range. 314 * 315 * Consider the following range which is 2MB aligned and is 316 * a part of a larger 20MB range which is not shown. Each 317 * character is 256KB below making the source and destination 318 * 2MB each. The lower case letters are moved (s to d) and the 319 * upper case letters are not moved. The below test verifies 320 * that the upper case S letters are not corrupted by the 321 * adjacent mremap. 322 * 323 * |DDDDddddSSSSssss| 324 */ 325static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr) 326{ 327 char *test_name = "mremap mremap move within range"; 328 void *src, *dest; 329 int i, success = 1; 330 331 size_t size = SIZE_MB(20); 332 void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, 333 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 334 if (ptr == MAP_FAILED) { 335 perror("mmap"); 336 success = 0; 337 goto out; 338 } 339 memset(ptr, 0, size); 340 341 src = ptr + SIZE_MB(6); 342 src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1)); 343 344 /* Set byte pattern for source block. */ 345 memcpy(src, rand_addr, SIZE_MB(2)); 346 347 dest = src - SIZE_MB(2); 348 349 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 350 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 351 if (new_ptr == MAP_FAILED) { 352 perror("mremap"); 353 success = 0; 354 goto out; 355 } 356 357 /* Verify byte pattern after remapping */ 358 srand(pattern_seed); 359 for (i = 0; i < SIZE_MB(1); i++) { 360 char c = (char) rand(); 361 362 if (((char *)src)[i] != c) { 363 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 364 i); 365 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 366 ((char *) src)[i] & 0xff); 367 success = 0; 368 } 369 } 370 371out: 372 if (munmap(ptr, size) == -1) 373 perror("munmap"); 374 375 if (success) 376 ksft_test_result_pass("%s\n", test_name); 377 else 378 ksft_test_result_fail("%s\n", test_name); 379} 380 381/* Returns the time taken for the remap on success else returns -1. */ 382static long long remap_region(struct config c, unsigned int threshold_mb, 383 char *rand_addr) 384{ 385 void *addr, *src_addr, *dest_addr, *dest_preamble_addr; 386 unsigned long long t, d; 387 struct timespec t_start = {0, 0}, t_end = {0, 0}; 388 long long start_ns, end_ns, align_mask, ret, offset; 389 unsigned long long threshold; 390 unsigned long num_chunks; 391 392 if (threshold_mb == VALIDATION_NO_THRESHOLD) 393 threshold = c.region_size; 394 else 395 threshold = MIN(threshold_mb * _1MB, c.region_size); 396 397 src_addr = get_source_mapping(c); 398 if (!src_addr) { 399 ret = -1; 400 goto out; 401 } 402 403 /* Set byte pattern for source block. */ 404 memcpy(src_addr, rand_addr, threshold); 405 406 /* Mask to zero out lower bits of address for alignment */ 407 align_mask = ~(c.dest_alignment - 1); 408 /* Offset of destination address from the end of the source region */ 409 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment; 410 addr = (void *) (((unsigned long long) src_addr + c.region_size 411 + offset) & align_mask); 412 413 /* Remap after the destination block preamble. */ 414 addr += c.dest_preamble_size; 415 416 /* See comment in get_source_mapping() */ 417 if (!((unsigned long long) addr & c.dest_alignment)) 418 addr = (void *) ((unsigned long long) addr | c.dest_alignment); 419 420 /* Don't destroy existing mappings unless expected to overlap */ 421 while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) { 422 /* Check for unsigned overflow */ 423 if (addr + c.dest_alignment < addr) { 424 ksft_print_msg("Couldn't find a valid region to remap to\n"); 425 ret = -1; 426 goto clean_up_src; 427 } 428 addr += c.dest_alignment; 429 } 430 431 if (c.dest_preamble_size) { 432 dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size, 433 PROT_READ | PROT_WRITE, 434 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, 435 -1, 0); 436 if (dest_preamble_addr == MAP_FAILED) { 437 ksft_print_msg("Failed to map dest preamble region: %s\n", 438 strerror(errno)); 439 ret = -1; 440 goto clean_up_src; 441 } 442 443 /* Set byte pattern for the dest preamble block. */ 444 memcpy(dest_preamble_addr, rand_addr, c.dest_preamble_size); 445 } 446 447 clock_gettime(CLOCK_MONOTONIC, &t_start); 448 dest_addr = mremap(src_addr, c.region_size, c.region_size, 449 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr); 450 clock_gettime(CLOCK_MONOTONIC, &t_end); 451 452 if (dest_addr == MAP_FAILED) { 453 ksft_print_msg("mremap failed: %s\n", strerror(errno)); 454 ret = -1; 455 goto clean_up_dest_preamble; 456 } 457 458 /* 459 * Verify byte pattern after remapping. Employ an algorithm with a 460 * square root time complexity in threshold: divide the range into 461 * chunks, if memcmp() returns non-zero, only then perform an 462 * iteration in that chunk to find the mismatch index. 463 */ 464 num_chunks = get_sqrt(threshold); 465 for (unsigned long i = 0; i < num_chunks; ++i) { 466 size_t chunk_size = threshold / num_chunks; 467 unsigned long shift = i * chunk_size; 468 469 if (!memcmp(dest_addr + shift, rand_addr + shift, chunk_size)) 470 continue; 471 472 /* brute force iteration only over mismatch segment */ 473 for (t = shift; t < shift + chunk_size; ++t) { 474 if (((char *) dest_addr)[t] != rand_addr[t]) { 475 ksft_print_msg("Data after remap doesn't match at offset %llu\n", 476 t); 477 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, 478 ((char *) dest_addr)[t] & 0xff); 479 ret = -1; 480 goto clean_up_dest; 481 } 482 } 483 } 484 485 /* 486 * if threshold is not divisible by num_chunks, then check the 487 * last chunk 488 */ 489 for (t = num_chunks * (threshold / num_chunks); t < threshold; ++t) { 490 if (((char *) dest_addr)[t] != rand_addr[t]) { 491 ksft_print_msg("Data after remap doesn't match at offset %llu\n", 492 t); 493 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, 494 ((char *) dest_addr)[t] & 0xff); 495 ret = -1; 496 goto clean_up_dest; 497 } 498 } 499 500 /* Verify the dest preamble byte pattern after remapping */ 501 if (!c.dest_preamble_size) 502 goto no_preamble; 503 504 num_chunks = get_sqrt(c.dest_preamble_size); 505 506 for (unsigned long i = 0; i < num_chunks; ++i) { 507 size_t chunk_size = c.dest_preamble_size / num_chunks; 508 unsigned long shift = i * chunk_size; 509 510 if (!memcmp(dest_preamble_addr + shift, rand_addr + shift, 511 chunk_size)) 512 continue; 513 514 /* brute force iteration only over mismatched segment */ 515 for (d = shift; d < shift + chunk_size; ++d) { 516 if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { 517 ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", 518 d); 519 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, 520 ((char *) dest_preamble_addr)[d] & 0xff); 521 ret = -1; 522 goto clean_up_dest; 523 } 524 } 525 } 526 527 for (d = num_chunks * (c.dest_preamble_size / num_chunks); d < c.dest_preamble_size; ++d) { 528 if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { 529 ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", 530 d); 531 ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, 532 ((char *) dest_preamble_addr)[d] & 0xff); 533 ret = -1; 534 goto clean_up_dest; 535 } 536 } 537 538no_preamble: 539 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; 540 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; 541 ret = end_ns - start_ns; 542 543/* 544 * Since the destination address is specified using MREMAP_FIXED, subsequent 545 * mremap will unmap any previous mapping at the address range specified by 546 * dest_addr and region_size. This significantly affects the remap time of 547 * subsequent tests. So we clean up mappings after each test. 548 */ 549clean_up_dest: 550 munmap(dest_addr, c.region_size); 551clean_up_dest_preamble: 552 if (c.dest_preamble_size && dest_preamble_addr) 553 munmap(dest_preamble_addr, c.dest_preamble_size); 554clean_up_src: 555 munmap(src_addr, c.region_size); 556out: 557 return ret; 558} 559 560/* 561 * Verify that an mremap aligning down does not destroy 562 * the beginning of the mapping just because the aligned 563 * down address landed on a mapping that maybe does not exist. 564 */ 565static void mremap_move_1mb_from_start(unsigned int pattern_seed, 566 char *rand_addr) 567{ 568 char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src"; 569 void *src = NULL, *dest = NULL; 570 int i, success = 1; 571 572 /* Config to reuse get_source_mapping() to do an aligned mmap. */ 573 struct config c = { 574 .src_alignment = SIZE_MB(1) + SIZE_KB(256), 575 .region_size = SIZE_MB(6) 576 }; 577 578 src = get_source_mapping(c); 579 if (!src) { 580 success = 0; 581 goto out; 582 } 583 584 c.src_alignment = SIZE_MB(1) + SIZE_KB(256); 585 dest = get_source_mapping(c); 586 if (!dest) { 587 success = 0; 588 goto out; 589 } 590 591 /* Set byte pattern for source block. */ 592 memcpy(src, rand_addr, SIZE_MB(2)); 593 594 /* 595 * Unmap the beginning of dest so that the aligned address 596 * falls on no mapping. 597 */ 598 munmap(dest, SIZE_MB(1)); 599 600 void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), 601 MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); 602 if (new_ptr == MAP_FAILED) { 603 perror("mremap"); 604 success = 0; 605 goto out; 606 } 607 608 /* Verify byte pattern after remapping */ 609 srand(pattern_seed); 610 for (i = 0; i < SIZE_MB(1); i++) { 611 char c = (char) rand(); 612 613 if (((char *)src)[i] != c) { 614 ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", 615 i); 616 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, 617 ((char *) src)[i] & 0xff); 618 success = 0; 619 } 620 } 621 622out: 623 if (src && munmap(src, c.region_size) == -1) 624 perror("munmap src"); 625 626 if (dest && munmap(dest, c.region_size) == -1) 627 perror("munmap dest"); 628 629 if (success) 630 ksft_test_result_pass("%s\n", test_name); 631 else 632 ksft_test_result_fail("%s\n", test_name); 633} 634 635static void run_mremap_test_case(struct test test_case, int *failures, 636 unsigned int threshold_mb, 637 unsigned int pattern_seed, char *rand_addr) 638{ 639 long long remap_time = remap_region(test_case.config, threshold_mb, 640 rand_addr); 641 642 if (remap_time < 0) { 643 if (test_case.expect_failure) 644 ksft_test_result_xfail("%s\n\tExpected mremap failure\n", 645 test_case.name); 646 else { 647 ksft_test_result_fail("%s\n", test_case.name); 648 *failures += 1; 649 } 650 } else { 651 /* 652 * Comparing mremap time is only applicable if entire region 653 * was faulted in. 654 */ 655 if (threshold_mb == VALIDATION_NO_THRESHOLD || 656 test_case.config.region_size <= threshold_mb * _1MB) 657 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", 658 test_case.name, remap_time); 659 else 660 ksft_test_result_pass("%s\n", test_case.name); 661 } 662} 663 664static void usage(const char *cmd) 665{ 666 fprintf(stderr, 667 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n" 668 "-t\t only validate threshold_mb of the remapped region\n" 669 " \t if 0 is supplied no threshold is used; all tests\n" 670 " \t are run and remapped regions validated fully.\n" 671 " \t The default threshold used is 4MB.\n" 672 "-p\t provide a seed to generate the random pattern for\n" 673 " \t validating the remapped region.\n", cmd); 674} 675 676static int parse_args(int argc, char **argv, unsigned int *threshold_mb, 677 unsigned int *pattern_seed) 678{ 679 const char *optstr = "t:p:"; 680 int opt; 681 682 while ((opt = getopt(argc, argv, optstr)) != -1) { 683 switch (opt) { 684 case 't': 685 *threshold_mb = atoi(optarg); 686 break; 687 case 'p': 688 *pattern_seed = atoi(optarg); 689 break; 690 default: 691 usage(argv[0]); 692 return -1; 693 } 694 } 695 696 if (optind < argc) { 697 usage(argv[0]); 698 return -1; 699 } 700 701 return 0; 702} 703 704#define MAX_TEST 15 705#define MAX_PERF_TEST 3 706int main(int argc, char **argv) 707{ 708 int failures = 0; 709 int i, run_perf_tests; 710 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; 711 712 /* hard-coded test configs */ 713 size_t max_test_variable_region_size = _2GB; 714 size_t max_test_constant_region_size = _2MB; 715 size_t dest_preamble_size = 10 * _4MB; 716 717 unsigned int pattern_seed; 718 char *rand_addr; 719 size_t rand_size; 720 int num_expand_tests = 2; 721 int num_misc_tests = 2; 722 struct test test_cases[MAX_TEST] = {}; 723 struct test perf_test_cases[MAX_PERF_TEST]; 724 int page_size; 725 time_t t; 726 FILE *maps_fp; 727 728 pattern_seed = (unsigned int) time(&t); 729 730 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) 731 exit(EXIT_FAILURE); 732 733 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", 734 threshold_mb, pattern_seed); 735 736 /* 737 * set preallocated random array according to test configs; see the 738 * functions for the logic of setting the size 739 */ 740 if (!threshold_mb) 741 rand_size = MAX(max_test_variable_region_size, 742 max_test_constant_region_size); 743 else 744 rand_size = MAX(MIN(threshold_mb * _1MB, 745 max_test_variable_region_size), 746 max_test_constant_region_size); 747 rand_size = MAX(dest_preamble_size, rand_size); 748 749 rand_addr = (char *)mmap(NULL, rand_size, PROT_READ | PROT_WRITE, 750 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 751 if (rand_addr == MAP_FAILED) { 752 perror("mmap"); 753 ksft_exit_fail_msg("cannot mmap rand_addr\n"); 754 } 755 756 /* fill stream of random bytes */ 757 srand(pattern_seed); 758 for (unsigned long i = 0; i < rand_size; ++i) 759 rand_addr[i] = (char) rand(); 760 761 page_size = sysconf(_SC_PAGESIZE); 762 763 /* Expected mremap failures */ 764 test_cases[0] = MAKE_TEST(page_size, page_size, page_size, 765 OVERLAPPING, EXPECT_FAILURE, 766 "mremap - Source and Destination Regions Overlapping"); 767 768 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size, 769 NON_OVERLAPPING, EXPECT_FAILURE, 770 "mremap - Destination Address Misaligned (1KB-aligned)"); 771 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size, 772 NON_OVERLAPPING, EXPECT_FAILURE, 773 "mremap - Source Address Misaligned (1KB-aligned)"); 774 775 /* Src addr PTE aligned */ 776 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2, 777 NON_OVERLAPPING, EXPECT_SUCCESS, 778 "8KB mremap - Source PTE-aligned, Destination PTE-aligned"); 779 780 /* Src addr 1MB aligned */ 781 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 782 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"); 783 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, 784 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 785 786 /* Src addr PMD aligned */ 787 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 788 "4MB mremap - Source PMD-aligned, Destination PTE-aligned"); 789 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 790 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"); 791 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, 792 "4MB mremap - Source PMD-aligned, Destination PMD-aligned"); 793 794 /* Src addr PUD aligned */ 795 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 796 "2GB mremap - Source PUD-aligned, Destination PTE-aligned"); 797 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 798 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"); 799 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 800 "2GB mremap - Source PUD-aligned, Destination PMD-aligned"); 801 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, 802 "2GB mremap - Source PUD-aligned, Destination PUD-aligned"); 803 804 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 805 test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 806 "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); 807 808 /* Src and Dest addr 1MB aligned. 5MB mremap. */ 809 test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, 810 "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble"); 811 test_cases[14].config.dest_preamble_size = 10 * _4MB; 812 813 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 814 "1GB mremap - Source PTE-aligned, Destination PTE-aligned"); 815 /* 816 * mremap 1GB region - Page table level aligned time 817 * comparison. 818 */ 819 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 820 "1GB mremap - Source PMD-aligned, Destination PMD-aligned"); 821 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, 822 "1GB mremap - Source PUD-aligned, Destination PUD-aligned"); 823 824 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) || 825 (threshold_mb * _1MB >= _1GB); 826 827 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ? 828 ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests); 829 830 for (i = 0; i < ARRAY_SIZE(test_cases); i++) 831 run_mremap_test_case(test_cases[i], &failures, threshold_mb, 832 pattern_seed, rand_addr); 833 834 maps_fp = fopen("/proc/self/maps", "r"); 835 836 if (maps_fp == NULL) { 837 munmap(rand_addr, rand_size); 838 ksft_exit_fail_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); 839 } 840 841 mremap_expand_merge(maps_fp, page_size); 842 mremap_expand_merge_offset(maps_fp, page_size); 843 844 fclose(maps_fp); 845 846 mremap_move_within_range(pattern_seed, rand_addr); 847 mremap_move_1mb_from_start(pattern_seed, rand_addr); 848 849 if (run_perf_tests) { 850 ksft_print_msg("\n%s\n", 851 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); 852 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++) 853 run_mremap_test_case(perf_test_cases[i], &failures, 854 threshold_mb, pattern_seed, 855 rand_addr); 856 } 857 858 munmap(rand_addr, rand_size); 859 860 if (failures > 0) 861 ksft_exit_fail(); 862 else 863 ksft_exit_pass(); 864} 865