1289177Speter/* verify.c --- verification of FSX filesystems 2289177Speter * 3289177Speter * ==================================================================== 4289177Speter * Licensed to the Apache Software Foundation (ASF) under one 5289177Speter * or more contributor license agreements. See the NOTICE file 6289177Speter * distributed with this work for additional information 7289177Speter * regarding copyright ownership. The ASF licenses this file 8289177Speter * to you under the Apache License, Version 2.0 (the 9289177Speter * "License"); you may not use this file except in compliance 10289177Speter * with the License. You may obtain a copy of the License at 11289177Speter * 12289177Speter * http://www.apache.org/licenses/LICENSE-2.0 13289177Speter * 14289177Speter * Unless required by applicable law or agreed to in writing, 15289177Speter * software distributed under the License is distributed on an 16289177Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17289177Speter * KIND, either express or implied. See the License for the 18289177Speter * specific language governing permissions and limitations 19289177Speter * under the License. 20289177Speter * ==================================================================== 21289177Speter */ 22289177Speter 23289177Speter#include "verify.h" 24289177Speter#include "fs_x.h" 25289177Speter#include "svn_time.h" 26289177Speter#include "private/svn_subr_private.h" 27289177Speter 28289177Speter#include "cached_data.h" 29289177Speter#include "rep-cache.h" 30289177Speter#include "util.h" 31289177Speter#include "index.h" 32289177Speter 33289177Speter#include "../libsvn_fs/fs-loader.h" 34289177Speter 35289177Speter#include "svn_private_config.h" 36289177Speter 37289177Speter 38289177Speter/** Verifying. **/ 39289177Speter 40289177Speter/* Baton type expected by verify_walker(). The purpose is to limit the 41289177Speter * number of notifications sent. 42289177Speter */ 43289177Spetertypedef struct verify_walker_baton_t 44289177Speter{ 45289177Speter /* number of calls to verify_walker() since the last clean */ 46289177Speter int iteration_count; 47289177Speter 48289177Speter /* progress notification callback to invoke periodically (may be NULL) */ 49289177Speter svn_fs_progress_notify_func_t notify_func; 50289177Speter 51289177Speter /* baton to use with NOTIFY_FUNC */ 52289177Speter void *notify_baton; 53289177Speter 54289177Speter /* remember the last revision for which we called notify_func */ 55289177Speter svn_revnum_t last_notified_revision; 56289177Speter} verify_walker_baton_t; 57289177Speter 58289177Speter/* Used by svn_fs_x__verify(). 59289177Speter Implements svn_fs_x__walk_rep_reference().walker. */ 60289177Speterstatic svn_error_t * 61289177Speterverify_walker(svn_fs_x__representation_t *rep, 62289177Speter void *baton, 63289177Speter svn_fs_t *fs, 64289177Speter apr_pool_t *scratch_pool) 65289177Speter{ 66289177Speter verify_walker_baton_t *walker_baton = baton; 67289177Speter 68289177Speter /* notify and free resources periodically */ 69289177Speter if (walker_baton->iteration_count > 1000) 70289177Speter { 71289177Speter svn_revnum_t revision = svn_fs_x__get_revnum(rep->id.change_set); 72289177Speter if ( walker_baton->notify_func 73289177Speter && revision != walker_baton->last_notified_revision) 74289177Speter { 75289177Speter walker_baton->notify_func(revision, 76289177Speter walker_baton->notify_baton, 77289177Speter scratch_pool); 78289177Speter walker_baton->last_notified_revision = revision; 79289177Speter } 80289177Speter 81289177Speter walker_baton->iteration_count = 0; 82289177Speter } 83289177Speter 84289177Speter /* access the repo data */ 85289177Speter SVN_ERR(svn_fs_x__check_rep(rep, fs, scratch_pool)); 86289177Speter 87289177Speter /* update resource usage counters */ 88289177Speter walker_baton->iteration_count++; 89289177Speter 90289177Speter return SVN_NO_ERROR; 91289177Speter} 92289177Speter 93289177Speter/* Verify the rep cache DB's consistency with our rev / pack data. 94289177Speter * The function signature is similar to svn_fs_x__verify. 95289177Speter * The values of START and END have already been auto-selected and 96289177Speter * verified. 97289177Speter */ 98289177Speterstatic svn_error_t * 99289177Speterverify_rep_cache(svn_fs_t *fs, 100289177Speter svn_revnum_t start, 101289177Speter svn_revnum_t end, 102289177Speter svn_fs_progress_notify_func_t notify_func, 103289177Speter void *notify_baton, 104289177Speter svn_cancel_func_t cancel_func, 105289177Speter void *cancel_baton, 106289177Speter apr_pool_t *scratch_pool) 107289177Speter{ 108289177Speter svn_boolean_t exists; 109289177Speter 110289177Speter /* rep-cache verification. */ 111289177Speter SVN_ERR(svn_fs_x__exists_rep_cache(&exists, fs, scratch_pool)); 112289177Speter if (exists) 113289177Speter { 114289177Speter /* provide a baton to allow the reuse of open file handles between 115289177Speter iterations (saves 2/3 of OS level file operations). */ 116289177Speter verify_walker_baton_t *baton 117289177Speter = apr_pcalloc(scratch_pool, sizeof(*baton)); 118289177Speter 119289177Speter baton->last_notified_revision = SVN_INVALID_REVNUM; 120289177Speter baton->notify_func = notify_func; 121289177Speter baton->notify_baton = notify_baton; 122289177Speter 123289177Speter /* tell the user that we are now ready to do *something* */ 124289177Speter if (notify_func) 125289177Speter notify_func(SVN_INVALID_REVNUM, notify_baton, scratch_pool); 126289177Speter 127289177Speter /* Do not attempt to walk the rep-cache database if its file does 128289177Speter not exist, since doing so would create it --- which may confuse 129289177Speter the administrator. Don't take any lock. */ 130289177Speter SVN_ERR(svn_fs_x__walk_rep_reference(fs, start, end, 131289177Speter verify_walker, baton, 132289177Speter cancel_func, cancel_baton, 133289177Speter scratch_pool)); 134289177Speter } 135289177Speter 136289177Speter return SVN_NO_ERROR; 137289177Speter} 138289177Speter 139289177Speter/* Verify that the MD5 checksum of the data between offsets START and END 140289177Speter * in FILE matches the EXPECTED checksum. If there is a mismatch use the 141289177Speter * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC 142289177Speter * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ 143289177Speterstatic svn_error_t * 144289177Speterverify_index_checksum(apr_file_t *file, 145289177Speter const char *name, 146289177Speter apr_off_t start, 147289177Speter apr_off_t end, 148289177Speter svn_checksum_t *expected, 149289177Speter svn_cancel_func_t cancel_func, 150289177Speter void *cancel_baton, 151289177Speter apr_pool_t *scratch_pool) 152289177Speter{ 153289177Speter unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; 154289177Speter apr_off_t size = end - start; 155289177Speter svn_checksum_t *actual; 156289177Speter svn_checksum_ctx_t *context 157289177Speter = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); 158289177Speter 159289177Speter /* Calculate the index checksum. */ 160289177Speter SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool)); 161289177Speter while (size > 0) 162289177Speter { 163289177Speter apr_size_t to_read = size > sizeof(buffer) 164289177Speter ? sizeof(buffer) 165289177Speter : (apr_size_t)size; 166289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 167289177Speter scratch_pool)); 168289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 169289177Speter size -= to_read; 170289177Speter 171289177Speter if (cancel_func) 172289177Speter SVN_ERR(cancel_func(cancel_baton)); 173289177Speter } 174289177Speter 175289177Speter SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); 176289177Speter 177289177Speter /* Verify that it matches the expected checksum. */ 178289177Speter if (!svn_checksum_match(expected, actual)) 179289177Speter { 180289177Speter const char *file_name; 181289177Speter 182289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 183289177Speter SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, 184289177Speter _("%s checksum mismatch in file %s"), 185289177Speter name, file_name)); 186289177Speter } 187289177Speter 188289177Speter return SVN_NO_ERROR; 189289177Speter} 190289177Speter 191289177Speter/* Verify the MD5 checksums of the index data in the rev / pack file 192289177Speter * containing revision START in FS. If given, invoke CANCEL_FUNC with 193289177Speter * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary 194289177Speter * allocations. 195289177Speter */ 196289177Speterstatic svn_error_t * 197289177Speterverify_index_checksums(svn_fs_t *fs, 198289177Speter svn_revnum_t start, 199289177Speter svn_cancel_func_t cancel_func, 200289177Speter void *cancel_baton, 201289177Speter apr_pool_t *scratch_pool) 202289177Speter{ 203289177Speter svn_fs_x__revision_file_t *rev_file; 204289177Speter 205289177Speter /* Open the rev / pack file and read the footer */ 206289177Speter SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, 207289177Speter scratch_pool, scratch_pool)); 208289177Speter SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); 209289177Speter 210289177Speter /* Verify the index contents against the checksum from the footer. */ 211289177Speter SVN_ERR(verify_index_checksum(rev_file->file, "L2P index", 212289177Speter rev_file->l2p_offset, rev_file->p2l_offset, 213289177Speter rev_file->l2p_checksum, 214289177Speter cancel_func, cancel_baton, scratch_pool)); 215289177Speter SVN_ERR(verify_index_checksum(rev_file->file, "P2L index", 216289177Speter rev_file->p2l_offset, rev_file->footer_offset, 217289177Speter rev_file->p2l_checksum, 218289177Speter cancel_func, cancel_baton, scratch_pool)); 219289177Speter 220289177Speter /* Done. */ 221289177Speter SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 222289177Speter 223289177Speter return SVN_NO_ERROR; 224289177Speter} 225289177Speter 226289177Speter/* Verify that for all log-to-phys index entries for revisions START to 227289177Speter * START + COUNT-1 in FS there is a consistent entry in the phys-to-log 228289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 229289177Speter * intervals. Use SCRATCH_POOL for temporary allocations. 230289177Speter */ 231289177Speterstatic svn_error_t * 232289177Spetercompare_l2p_to_p2l_index(svn_fs_t *fs, 233289177Speter svn_revnum_t start, 234289177Speter svn_revnum_t count, 235289177Speter svn_cancel_func_t cancel_func, 236289177Speter void *cancel_baton, 237289177Speter apr_pool_t *scratch_pool) 238289177Speter{ 239289177Speter svn_revnum_t i; 240289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 241289177Speter apr_array_header_t *max_ids; 242289177Speter 243289177Speter /* common file access structure */ 244289177Speter svn_fs_x__revision_file_t *rev_file; 245289177Speter SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, 246289177Speter iterpool)); 247289177Speter 248289177Speter /* determine the range of items to check for each revision */ 249289177Speter SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, scratch_pool, 250289177Speter iterpool)); 251289177Speter 252289177Speter /* check all items in all revisions if the given range */ 253289177Speter for (i = 0; i < max_ids->nelts; ++i) 254289177Speter { 255289177Speter apr_uint64_t k; 256289177Speter apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t); 257289177Speter svn_revnum_t revision = start + i; 258289177Speter 259289177Speter for (k = 0; k < max_id; ++k) 260289177Speter { 261289177Speter apr_off_t offset; 262289177Speter apr_uint32_t sub_item; 263289177Speter svn_fs_x__id_t l2p_item; 264289177Speter svn_fs_x__id_t *p2l_item; 265289177Speter 266289177Speter l2p_item.change_set = svn_fs_x__change_set_by_rev(revision); 267289177Speter l2p_item.number = k; 268289177Speter 269289177Speter /* get L2P entry. Ignore unused entries. */ 270289177Speter SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, 271289177Speter &l2p_item, iterpool)); 272289177Speter if (offset == -1) 273289177Speter continue; 274289177Speter 275289177Speter /* find the corresponding P2L entry */ 276289177Speter SVN_ERR(svn_fs_x__p2l_item_lookup(&p2l_item, fs, rev_file, 277289177Speter revision, offset, sub_item, 278289177Speter iterpool, iterpool)); 279289177Speter 280289177Speter if (p2l_item == NULL) 281289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 282289177Speter NULL, 283289177Speter _("p2l index entry not found for " 284289177Speter "PHYS o%s:s%ld returned by " 285289177Speter "l2p index for LOG r%ld:i%ld"), 286289177Speter apr_off_t_toa(scratch_pool, offset), 287289177Speter (long)sub_item, revision, (long)k); 288289177Speter 289289177Speter if (!svn_fs_x__id_eq(&l2p_item, p2l_item)) 290289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 291289177Speter NULL, 292289177Speter _("p2l index info LOG r%ld:i%ld" 293289177Speter " does not match " 294289177Speter "l2p index for LOG r%ld:i%ld"), 295289177Speter svn_fs_x__get_revnum(p2l_item->change_set), 296289177Speter (long)p2l_item->number, revision, 297289177Speter (long)k); 298289177Speter 299289177Speter svn_pool_clear(iterpool); 300289177Speter } 301289177Speter 302289177Speter if (cancel_func) 303289177Speter SVN_ERR(cancel_func(cancel_baton)); 304289177Speter } 305289177Speter 306289177Speter svn_pool_destroy(iterpool); 307289177Speter 308289177Speter SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 309289177Speter 310289177Speter return SVN_NO_ERROR; 311289177Speter} 312289177Speter 313289177Speter/* Verify that for all phys-to-log index entries for revisions START to 314289177Speter * START + COUNT-1 in FS there is a consistent entry in the log-to-phys 315289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 316289177Speter * intervals. Use SCRATCH_POOL for temporary allocations. 317289177Speter * 318289177Speter * Please note that we can only check on pack / rev file granularity and 319289177Speter * must only be called for a single rev / pack file. 320289177Speter */ 321289177Speterstatic svn_error_t * 322289177Spetercompare_p2l_to_l2p_index(svn_fs_t *fs, 323289177Speter svn_revnum_t start, 324289177Speter svn_revnum_t count, 325289177Speter svn_cancel_func_t cancel_func, 326289177Speter void *cancel_baton, 327289177Speter apr_pool_t *scratch_pool) 328289177Speter{ 329289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 330289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 331289177Speter apr_pool_t *iterpool2 = svn_pool_create(scratch_pool); 332289177Speter apr_off_t max_offset; 333289177Speter apr_off_t offset = 0; 334289177Speter 335289177Speter /* common file access structure */ 336289177Speter svn_fs_x__revision_file_t *rev_file; 337289177Speter SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, 338289177Speter iterpool)); 339289177Speter 340289177Speter /* get the size of the rev / pack file as covered by the P2L index */ 341289177Speter SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 342289177Speter scratch_pool)); 343289177Speter 344289177Speter /* for all offsets in the file, get the P2L index entries and check 345289177Speter them against the L2P index */ 346289177Speter for (offset = 0; offset < max_offset; ) 347289177Speter { 348289177Speter apr_array_header_t *entries; 349289177Speter svn_fs_x__p2l_entry_t *last_entry; 350289177Speter int i; 351289177Speter 352289177Speter svn_pool_clear(iterpool); 353289177Speter 354289177Speter /* get all entries for the current block */ 355289177Speter SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 356289177Speter offset, ffd->p2l_page_size, 357289177Speter iterpool, iterpool)); 358289177Speter if (entries->nelts == 0) 359289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 360289177Speter NULL, 361289177Speter _("p2l does not cover offset %s" 362289177Speter " for revision %ld"), 363289177Speter apr_off_t_toa(scratch_pool, offset), start); 364289177Speter 365289177Speter /* process all entries (and later continue with the next block) */ 366289177Speter last_entry 367289177Speter = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t); 368289177Speter offset = last_entry->offset + last_entry->size; 369289177Speter 370289177Speter for (i = 0; i < entries->nelts; ++i) 371289177Speter { 372289177Speter apr_uint32_t k; 373289177Speter svn_fs_x__p2l_entry_t *entry 374289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 375289177Speter 376289177Speter /* check all sub-items for consist entries in the L2P index */ 377289177Speter for (k = 0; k < entry->item_count; ++k) 378289177Speter { 379289177Speter apr_off_t l2p_offset; 380289177Speter apr_uint32_t sub_item; 381289177Speter svn_fs_x__id_t *p2l_item = &entry->items[k]; 382289177Speter svn_revnum_t revision 383289177Speter = svn_fs_x__get_revnum(p2l_item->change_set); 384289177Speter 385289177Speter svn_pool_clear(iterpool2); 386289177Speter SVN_ERR(svn_fs_x__item_offset(&l2p_offset, &sub_item, fs, 387289177Speter rev_file, p2l_item, iterpool2)); 388289177Speter 389289177Speter if (sub_item != k || l2p_offset != entry->offset) 390289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 391289177Speter NULL, 392289177Speter _("l2p index entry PHYS o%s:s%ld " 393289177Speter "does not match p2l index value " 394289177Speter "LOG r%ld:i%ld for PHYS o%s:s%ld"), 395289177Speter apr_off_t_toa(scratch_pool, 396289177Speter l2p_offset), 397289177Speter (long)sub_item, 398289177Speter revision, 399289177Speter (long)p2l_item->number, 400289177Speter apr_off_t_toa(scratch_pool, 401289177Speter entry->offset), 402289177Speter (long)k); 403289177Speter } 404289177Speter } 405289177Speter 406289177Speter if (cancel_func) 407289177Speter SVN_ERR(cancel_func(cancel_baton)); 408289177Speter } 409289177Speter 410289177Speter svn_pool_destroy(iterpool2); 411289177Speter svn_pool_destroy(iterpool); 412289177Speter 413289177Speter SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 414289177Speter 415289177Speter return SVN_NO_ERROR; 416289177Speter} 417289177Speter 418289177Speter/* Items smaller than this can be read at once into a buffer and directly 419289177Speter * be checksummed. Larger items require stream processing. 420289177Speter * Must be a multiple of 8. */ 421289177Speter#define STREAM_THRESHOLD 4096 422289177Speter 423289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. SIZE must not 424289177Speter * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 425289177Speter */ 426289177Speterstatic svn_error_t * 427289177Speterexpect_buffer_nul(apr_file_t *file, 428289177Speter apr_off_t size, 429289177Speter apr_pool_t *scratch_pool) 430289177Speter{ 431289177Speter union 432289177Speter { 433289177Speter unsigned char buffer[STREAM_THRESHOLD]; 434289177Speter apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)]; 435289177Speter } data; 436289177Speter 437289177Speter apr_size_t i; 438289177Speter SVN_ERR_ASSERT(size <= STREAM_THRESHOLD); 439289177Speter 440289177Speter /* read the whole data block; error out on failure */ 441289177Speter data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0; 442289177Speter SVN_ERR(svn_io_file_read_full2(file, data.buffer, size, NULL, NULL, 443289177Speter scratch_pool)); 444289177Speter 445289177Speter /* chunky check */ 446289177Speter for (i = 0; i < size / sizeof(apr_uint64_t); ++i) 447289177Speter if (data.chunks[i] != 0) 448289177Speter break; 449289177Speter 450289177Speter /* byte-wise check upon mismatch or at the end of the block */ 451289177Speter for (i *= sizeof(apr_uint64_t); i < size; ++i) 452289177Speter if (data.buffer[i] != 0) 453289177Speter { 454289177Speter const char *file_name; 455289177Speter apr_off_t offset; 456289177Speter 457289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 458289177Speter SVN_ERR(svn_fs_x__get_file_offset(&offset, file, scratch_pool)); 459289177Speter offset -= size - i; 460289177Speter 461289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 462289177Speter _("Empty section in file %s contains " 463289177Speter "non-NUL data at offset %s"), 464289177Speter file_name, 465289177Speter apr_off_t_toa(scratch_pool, offset)); 466289177Speter } 467289177Speter 468289177Speter return SVN_NO_ERROR; 469289177Speter} 470289177Speter 471289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. 472289177Speter * Use SCRATCH_POOL for temporary allocations. 473289177Speter */ 474289177Speterstatic svn_error_t * 475289177Speterread_all_nul(apr_file_t *file, 476289177Speter apr_off_t size, 477289177Speter apr_pool_t *scratch_pool) 478289177Speter{ 479289177Speter for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD) 480289177Speter SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, scratch_pool)); 481289177Speter 482289177Speter if (size) 483289177Speter SVN_ERR(expect_buffer_nul(file, size, scratch_pool)); 484289177Speter 485289177Speter return SVN_NO_ERROR; 486289177Speter} 487289177Speter 488289177Speter/* Compare the ACTUAL checksum with the one expected by ENTRY. 489289177Speter * Return an error in case of mismatch. Use the name of FILE 490289177Speter * in error message. Allocate temporary data in SCRATCH_POOL. 491289177Speter */ 492289177Speterstatic svn_error_t * 493289177Speterexpected_checksum(apr_file_t *file, 494289177Speter svn_fs_x__p2l_entry_t *entry, 495289177Speter apr_uint32_t actual, 496289177Speter apr_pool_t *scratch_pool) 497289177Speter{ 498289177Speter if (actual != entry->fnv1_checksum) 499289177Speter { 500289177Speter const char *file_name; 501289177Speter 502289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 503289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 504289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 505289177Speter _("Checksum mismatch in item at offset %s of " 506289177Speter "length %s bytes in file %s"), 507289177Speter apr_off_t_toa(scratch_pool, entry->offset), 508289177Speter apr_off_t_toa(scratch_pool, entry->size), 509289177Speter file_name); 510289177Speter } 511289177Speter 512289177Speter return SVN_NO_ERROR; 513289177Speter} 514289177Speter 515289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read 516289177Speter * from FILE will match ENTRY's expected checksum. SIZE must not 517289177Speter * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 518289177Speter */ 519289177Speterstatic svn_error_t * 520289177Speterexpected_buffered_checksum(apr_file_t *file, 521289177Speter svn_fs_x__p2l_entry_t *entry, 522289177Speter apr_pool_t *scratch_pool) 523289177Speter{ 524289177Speter unsigned char buffer[STREAM_THRESHOLD]; 525289177Speter SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD); 526289177Speter 527289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, (apr_size_t)entry->size, 528289177Speter NULL, NULL, scratch_pool)); 529289177Speter SVN_ERR(expected_checksum(file, entry, 530289177Speter svn__fnv1a_32x4(buffer, (apr_size_t)entry->size), 531289177Speter scratch_pool)); 532289177Speter 533289177Speter return SVN_NO_ERROR; 534289177Speter} 535289177Speter 536289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from 537289177Speter * FILE will match ENTRY's expected checksum. 538289177Speter * Use SCRATCH_POOL for temporary allocations. 539289177Speter */ 540289177Speterstatic svn_error_t * 541289177Speterexpected_streamed_checksum(apr_file_t *file, 542289177Speter svn_fs_x__p2l_entry_t *entry, 543289177Speter apr_pool_t *scratch_pool) 544289177Speter{ 545289177Speter unsigned char buffer[STREAM_THRESHOLD]; 546289177Speter svn_checksum_t *checksum; 547289177Speter svn_checksum_ctx_t *context 548289177Speter = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, scratch_pool); 549289177Speter apr_off_t size = entry->size; 550289177Speter 551289177Speter while (size > 0) 552289177Speter { 553289177Speter apr_size_t to_read = size > sizeof(buffer) 554289177Speter ? sizeof(buffer) 555289177Speter : (apr_size_t)size; 556289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 557289177Speter scratch_pool)); 558289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 559289177Speter size -= to_read; 560289177Speter } 561289177Speter 562289177Speter SVN_ERR(svn_checksum_final(&checksum, context, scratch_pool)); 563289177Speter SVN_ERR(expected_checksum(file, entry, 564289177Speter ntohl(*(const apr_uint32_t *)checksum->digest), 565289177Speter scratch_pool)); 566289177Speter 567289177Speter return SVN_NO_ERROR; 568289177Speter} 569289177Speter 570289177Speter/* Verify that for all phys-to-log index entries for revisions START to 571289177Speter * START + COUNT-1 in FS match the actual pack / rev file contents. 572289177Speter * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 573289177Speter * Use SCRATCH_POOL for temporary allocations. 574289177Speter * 575289177Speter * Please note that we can only check on pack / rev file granularity and 576289177Speter * must only be called for a single rev / pack file. 577289177Speter */ 578289177Speterstatic svn_error_t * 579289177Spetercompare_p2l_to_rev(svn_fs_t *fs, 580289177Speter svn_revnum_t start, 581289177Speter svn_revnum_t count, 582289177Speter svn_cancel_func_t cancel_func, 583289177Speter void *cancel_baton, 584289177Speter apr_pool_t *scratch_pool) 585289177Speter{ 586289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 587289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 588289177Speter apr_off_t max_offset; 589289177Speter apr_off_t offset = 0; 590289177Speter svn_fs_x__revision_file_t *rev_file; 591289177Speter 592289177Speter /* open the pack / rev file that is covered by the p2l index */ 593289177Speter SVN_ERR(svn_fs_x__open_pack_or_rev_file(&rev_file, fs, start, scratch_pool, 594289177Speter iterpool)); 595289177Speter 596289177Speter /* check file size vs. range covered by index */ 597289177Speter SVN_ERR(svn_fs_x__auto_read_footer(rev_file)); 598289177Speter SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 599289177Speter scratch_pool)); 600289177Speter 601289177Speter if (rev_file->l2p_offset != max_offset) 602289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, 603289177Speter _("File size of %s for revision r%ld does " 604289177Speter "not match p2l index size of %s"), 605289177Speter apr_off_t_toa(scratch_pool, 606289177Speter rev_file->l2p_offset), 607289177Speter start, 608289177Speter apr_off_t_toa(scratch_pool, 609289177Speter max_offset)); 610289177Speter 611289177Speter SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0, 612289177Speter scratch_pool)); 613289177Speter 614289177Speter /* for all offsets in the file, get the P2L index entries and check 615289177Speter them against the L2P index */ 616289177Speter for (offset = 0; offset < max_offset; ) 617289177Speter { 618289177Speter apr_array_header_t *entries; 619289177Speter int i; 620289177Speter 621289177Speter svn_pool_clear(iterpool); 622289177Speter 623289177Speter /* get all entries for the current block */ 624289177Speter SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 625289177Speter offset, ffd->p2l_page_size, 626289177Speter iterpool, iterpool)); 627289177Speter 628289177Speter /* The above might have moved the file pointer. 629289177Speter * Ensure we actually start reading at OFFSET. */ 630289177Speter SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, 631289177Speter NULL, offset, iterpool)); 632289177Speter 633289177Speter /* process all entries (and later continue with the next block) */ 634289177Speter for (i = 0; i < entries->nelts; ++i) 635289177Speter { 636289177Speter svn_fs_x__p2l_entry_t *entry 637289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 638289177Speter 639289177Speter /* skip bits we previously checked */ 640289177Speter if (i == 0 && entry->offset < offset) 641289177Speter continue; 642289177Speter 643289177Speter /* skip zero-sized entries */ 644289177Speter if (entry->size == 0) 645289177Speter continue; 646289177Speter 647289177Speter /* p2l index must cover all rev / pack file offsets exactly once */ 648289177Speter if (entry->offset != offset) 649289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 650289177Speter NULL, 651289177Speter _("p2l index entry for revision r%ld" 652289177Speter " is non-contiguous between offsets " 653289177Speter " %s and %s"), 654289177Speter start, 655289177Speter apr_off_t_toa(scratch_pool, offset), 656289177Speter apr_off_t_toa(scratch_pool, 657289177Speter entry->offset)); 658289177Speter 659289177Speter /* empty sections must contain NUL bytes only */ 660289177Speter if (entry->type == SVN_FS_X__ITEM_TYPE_UNUSED) 661289177Speter { 662289177Speter /* skip filler entry at the end of the p2l index */ 663289177Speter if (entry->offset != max_offset) 664289177Speter SVN_ERR(read_all_nul(rev_file->file, entry->size, iterpool)); 665289177Speter } 666289177Speter else 667289177Speter { 668289177Speter if (entry->size < STREAM_THRESHOLD) 669289177Speter SVN_ERR(expected_buffered_checksum(rev_file->file, entry, 670289177Speter iterpool)); 671289177Speter else 672289177Speter SVN_ERR(expected_streamed_checksum(rev_file->file, entry, 673289177Speter iterpool)); 674289177Speter } 675289177Speter 676289177Speter /* advance offset */ 677289177Speter offset += entry->size; 678289177Speter } 679289177Speter 680289177Speter if (cancel_func) 681289177Speter SVN_ERR(cancel_func(cancel_baton)); 682289177Speter } 683289177Speter 684289177Speter svn_pool_destroy(iterpool); 685289177Speter 686289177Speter return SVN_NO_ERROR; 687289177Speter} 688289177Speter 689289177Speter/* Verify that the revprops of the revisions START to END in FS can be 690289177Speter * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 691289177Speter * 692289177Speter * The values of START and END have already been auto-selected and 693289177Speter * verified. 694289177Speter */ 695289177Speterstatic svn_error_t * 696289177Speterverify_revprops(svn_fs_t *fs, 697289177Speter svn_revnum_t start, 698289177Speter svn_revnum_t end, 699289177Speter svn_cancel_func_t cancel_func, 700289177Speter void *cancel_baton, 701289177Speter apr_pool_t *scratch_pool) 702289177Speter{ 703289177Speter svn_revnum_t revision; 704289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 705289177Speter 706289177Speter for (revision = start; revision < end; ++revision) 707289177Speter { 708289177Speter svn_string_t *date; 709289177Speter apr_time_t timetemp; 710289177Speter 711289177Speter svn_pool_clear(iterpool); 712289177Speter 713289177Speter /* Access the svn:date revprop. 714289177Speter * This implies parsing all revprops for that revision. */ 715289177Speter SVN_ERR(svn_fs_x__revision_prop(&date, fs, revision, 716289177Speter SVN_PROP_REVISION_DATE, 717289177Speter iterpool, iterpool)); 718289177Speter 719289177Speter /* The time stamp is the only revprop that, if given, needs to 720289177Speter * have a valid content. */ 721289177Speter if (date) 722289177Speter SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool)); 723289177Speter 724289177Speter if (cancel_func) 725289177Speter SVN_ERR(cancel_func(cancel_baton)); 726289177Speter } 727289177Speter 728289177Speter svn_pool_destroy(iterpool); 729289177Speter 730289177Speter return SVN_NO_ERROR; 731289177Speter} 732289177Speter 733289177Speter/* Verify that on-disk representation has not been tempered with (in a way 734289177Speter * that leaves the repository in a corrupted state). This compares log-to- 735289177Speter * phys with phys-to-log indexes, verifies the low-level checksums and 736289177Speter * checks that all revprops are available. The function signature is 737289177Speter * similar to svn_fs_x__verify. 738289177Speter * 739289177Speter * The values of START and END have already been auto-selected and 740289177Speter * verified. 741289177Speter */ 742289177Speterstatic svn_error_t * 743289177Speterverify_metadata_consistency(svn_fs_t *fs, 744289177Speter svn_revnum_t start, 745289177Speter svn_revnum_t end, 746289177Speter svn_fs_progress_notify_func_t notify_func, 747289177Speter void *notify_baton, 748289177Speter svn_cancel_func_t cancel_func, 749289177Speter void *cancel_baton, 750289177Speter apr_pool_t *scratch_pool) 751289177Speter{ 752289177Speter svn_error_t *err; 753289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 754289177Speter svn_revnum_t revision, next_revision; 755289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 756289177Speter 757289177Speter for (revision = start; revision <= end; revision = next_revision) 758289177Speter { 759289177Speter svn_revnum_t count = svn_fs_x__packed_base_rev(fs, revision); 760289177Speter svn_revnum_t pack_start = count; 761289177Speter svn_revnum_t pack_end = pack_start + svn_fs_x__pack_size(fs, revision); 762289177Speter 763289177Speter svn_pool_clear(iterpool); 764289177Speter 765289177Speter if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) 766289177Speter notify_func(pack_start, notify_baton, iterpool); 767289177Speter 768289177Speter /* Check for external corruption to the indexes. */ 769289177Speter err = verify_index_checksums(fs, pack_start, cancel_func, 770289177Speter cancel_baton, iterpool); 771289177Speter 772289177Speter /* two-way index check */ 773289177Speter if (!err) 774289177Speter err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, 775289177Speter cancel_func, cancel_baton, iterpool); 776289177Speter if (!err) 777289177Speter err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, 778289177Speter cancel_func, cancel_baton, iterpool); 779289177Speter 780289177Speter /* verify in-index checksums and types vs. actual rev / pack files */ 781289177Speter if (!err) 782289177Speter err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, 783289177Speter cancel_func, cancel_baton, iterpool); 784289177Speter 785289177Speter /* ensure that revprops are available and accessible */ 786289177Speter if (!err) 787289177Speter err = verify_revprops(fs, pack_start, pack_end, 788289177Speter cancel_func, cancel_baton, iterpool); 789289177Speter 790289177Speter /* concurrent packing is one of the reasons why verification may fail. 791289177Speter Make sure, we operate on up-to-date information. */ 792289177Speter if (err) 793289177Speter SVN_ERR(svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, 794289177Speter fs, scratch_pool)); 795289177Speter 796289177Speter /* retry the whole shard if it got packed in the meantime */ 797289177Speter if (err && count != svn_fs_x__pack_size(fs, revision)) 798289177Speter { 799289177Speter svn_error_clear(err); 800289177Speter 801289177Speter /* We could simply assign revision here but the code below is 802289177Speter more intuitive to maintainers. */ 803289177Speter next_revision = svn_fs_x__packed_base_rev(fs, revision); 804289177Speter } 805289177Speter else 806289177Speter { 807289177Speter SVN_ERR(err); 808289177Speter next_revision = pack_end; 809289177Speter } 810289177Speter } 811289177Speter 812289177Speter svn_pool_destroy(iterpool); 813289177Speter 814289177Speter return SVN_NO_ERROR; 815289177Speter} 816289177Speter 817289177Spetersvn_error_t * 818289177Spetersvn_fs_x__verify(svn_fs_t *fs, 819289177Speter svn_revnum_t start, 820289177Speter svn_revnum_t end, 821289177Speter svn_fs_progress_notify_func_t notify_func, 822289177Speter void *notify_baton, 823289177Speter svn_cancel_func_t cancel_func, 824289177Speter void *cancel_baton, 825289177Speter apr_pool_t *scratch_pool) 826289177Speter{ 827289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 828289177Speter svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ 829289177Speter 830289177Speter /* Input validation. */ 831289177Speter if (! SVN_IS_VALID_REVNUM(start)) 832289177Speter start = 0; 833289177Speter if (! SVN_IS_VALID_REVNUM(end)) 834289177Speter end = youngest; 835289177Speter SVN_ERR(svn_fs_x__ensure_revision_exists(start, fs, scratch_pool)); 836289177Speter SVN_ERR(svn_fs_x__ensure_revision_exists(end, fs, scratch_pool)); 837289177Speter 838289177Speter /* log/phys index consistency. We need to check them first to make 839289177Speter sure we can access the rev / pack files in format7. */ 840289177Speter SVN_ERR(verify_metadata_consistency(fs, start, end, 841289177Speter notify_func, notify_baton, 842289177Speter cancel_func, cancel_baton, 843289177Speter scratch_pool)); 844289177Speter 845289177Speter /* rep cache consistency */ 846289177Speter SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton, 847289177Speter cancel_func, cancel_baton, scratch_pool)); 848289177Speter 849289177Speter return SVN_NO_ERROR; 850289177Speter} 851