1289177Speter/* verify.c --- verification of FSX filesystems 2289177Speter * 3289177Speter * ==================================================================== 4289177Speter * Licensed to the Apache Software Foundation (ASF) under one 5289177Speter * or more contributor license agreements. See the NOTICE file 6289177Speter * distributed with this work for additional information 7289177Speter * regarding copyright ownership. The ASF licenses this file 8289177Speter * to you under the Apache License, Version 2.0 (the 9289177Speter * "License"); you may not use this file except in compliance 10289177Speter * with the License. You may obtain a copy of the License at 11289177Speter * 12289177Speter * http://www.apache.org/licenses/LICENSE-2.0 13289177Speter * 14289177Speter * Unless required by applicable law or agreed to in writing, 15289177Speter * software distributed under the License is distributed on an 16289177Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17289177Speter * KIND, either express or implied. See the License for the 18289177Speter * specific language governing permissions and limitations 19289177Speter * under the License. 20289177Speter * ==================================================================== 21289177Speter */ 22289177Speter 23289177Speter#include "verify.h" 24289177Speter#include "fs_x.h" 25289177Speter#include "svn_time.h" 26289177Speter#include "private/svn_subr_private.h" 27289177Speter 28289177Speter#include "cached_data.h" 29289177Speter#include "rep-cache.h" 30362181Sdim#include "revprops.h" 31289177Speter#include "util.h" 32289177Speter#include "index.h" 33289177Speter 34289177Speter#include "../libsvn_fs/fs-loader.h" 35289177Speter 36289177Speter#include "svn_private_config.h" 37289177Speter 38289177Speter 39289177Speter/** Verifying. **/ 40289177Speter 41289177Speter/* Baton type expected by verify_walker(). The purpose is to limit the 42289177Speter * number of notifications sent. 43289177Speter */ 44289177Spetertypedef struct verify_walker_baton_t 45289177Speter{ 46289177Speter /* number of calls to verify_walker() since the last clean */ 47289177Speter int iteration_count; 48289177Speter 49289177Speter /* progress notification callback to invoke periodically (may be NULL) */ 50289177Speter svn_fs_progress_notify_func_t notify_func; 51289177Speter 52289177Speter /* baton to use with NOTIFY_FUNC */ 53289177Speter void *notify_baton; 54289177Speter 55289177Speter /* remember the last revision for which we called notify_func */ 56289177Speter svn_revnum_t last_notified_revision; 57289177Speter} verify_walker_baton_t; 58289177Speter 59289177Speter/* Used by svn_fs_x__verify(). 60289177Speter Implements svn_fs_x__walk_rep_reference().walker. */ 61289177Speterstatic svn_error_t * 62289177Speterverify_walker(svn_fs_x__representation_t *rep, 63289177Speter void *baton, 64289177Speter svn_fs_t *fs, 65289177Speter apr_pool_t *scratch_pool) 66289177Speter{ 67289177Speter verify_walker_baton_t *walker_baton = baton; 68289177Speter 69289177Speter /* notify and free resources periodically */ 70289177Speter if (walker_baton->iteration_count > 1000) 71289177Speter { 72289177Speter svn_revnum_t revision = svn_fs_x__get_revnum(rep->id.change_set); 73289177Speter if ( walker_baton->notify_func 74289177Speter && revision != walker_baton->last_notified_revision) 75289177Speter { 76289177Speter walker_baton->notify_func(revision, 77289177Speter walker_baton->notify_baton, 78289177Speter scratch_pool); 79289177Speter walker_baton->last_notified_revision = revision; 80289177Speter } 81289177Speter 82289177Speter walker_baton->iteration_count = 0; 83289177Speter } 84289177Speter 85289177Speter /* access the repo data */ 86289177Speter SVN_ERR(svn_fs_x__check_rep(rep, fs, scratch_pool)); 87289177Speter 88289177Speter /* update resource usage counters */ 89289177Speter walker_baton->iteration_count++; 90289177Speter 91289177Speter return SVN_NO_ERROR; 92289177Speter} 93289177Speter 94289177Speter/* Verify the rep cache DB's consistency with our rev / pack data. 95289177Speter * The function signature is similar to svn_fs_x__verify. 96289177Speter * The values of START and END have already been auto-selected and 97289177Speter * verified. 98289177Speter */ 99289177Speterstatic svn_error_t * 100289177Speterverify_rep_cache(svn_fs_t *fs, 101289177Speter svn_revnum_t start, 102289177Speter svn_revnum_t end, 103289177Speter svn_fs_progress_notify_func_t notify_func, 104289177Speter void *notify_baton, 105289177Speter svn_cancel_func_t cancel_func, 106289177Speter void *cancel_baton, 107289177Speter apr_pool_t *scratch_pool) 108289177Speter{ 109289177Speter svn_boolean_t exists; 110289177Speter 111289177Speter /* rep-cache verification. */ 112289177Speter SVN_ERR(svn_fs_x__exists_rep_cache(&exists, fs, scratch_pool)); 113289177Speter if (exists) 114289177Speter { 115289177Speter /* provide a baton to allow the reuse of open file handles between 116289177Speter iterations (saves 2/3 of OS level file operations). */ 117289177Speter verify_walker_baton_t *baton 118289177Speter = apr_pcalloc(scratch_pool, sizeof(*baton)); 119289177Speter 120289177Speter baton->last_notified_revision = SVN_INVALID_REVNUM; 121289177Speter baton->notify_func = notify_func; 122289177Speter baton->notify_baton = notify_baton; 123289177Speter 124289177Speter /* tell the user that we are now ready to do *something* */ 125289177Speter if (notify_func) 126289177Speter notify_func(SVN_INVALID_REVNUM, notify_baton, scratch_pool); 127289177Speter 128289177Speter /* Do not attempt to walk the rep-cache database if its file does 129289177Speter not exist, since doing so would create it --- which may confuse 130289177Speter the administrator. Don't take any lock. */ 131289177Speter SVN_ERR(svn_fs_x__walk_rep_reference(fs, start, end, 132289177Speter verify_walker, baton, 133289177Speter cancel_func, cancel_baton, 134289177Speter scratch_pool)); 135289177Speter } 136289177Speter 137289177Speter return SVN_NO_ERROR; 138289177Speter} 139289177Speter 140289177Speter/* Verify that the MD5 checksum of the data between offsets START and END 141289177Speter * in FILE matches the EXPECTED checksum. If there is a mismatch use the 142289177Speter * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC 143289177Speter * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ 144289177Speterstatic svn_error_t * 145362181Sdimverify_index_checksum(svn_fs_x__revision_file_t *file, 146289177Speter const char *name, 147362181Sdim svn_fs_x__index_info_t *index_info, 148289177Speter svn_cancel_func_t cancel_func, 149289177Speter void *cancel_baton, 150289177Speter apr_pool_t *scratch_pool) 151289177Speter{ 152289177Speter unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; 153362181Sdim apr_off_t size = index_info->end - index_info->start; 154289177Speter svn_checksum_t *actual; 155289177Speter svn_checksum_ctx_t *context 156289177Speter = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); 157289177Speter 158289177Speter /* Calculate the index checksum. */ 159362181Sdim SVN_ERR(svn_fs_x__rev_file_seek(file, NULL, index_info->start)); 160289177Speter while (size > 0) 161289177Speter { 162289177Speter apr_size_t to_read = size > sizeof(buffer) 163289177Speter ? sizeof(buffer) 164289177Speter : (apr_size_t)size; 165362181Sdim SVN_ERR(svn_fs_x__rev_file_read(file, buffer, to_read)); 166289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 167289177Speter size -= to_read; 168289177Speter 169289177Speter if (cancel_func) 170289177Speter SVN_ERR(cancel_func(cancel_baton)); 171289177Speter } 172289177Speter 173289177Speter SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); 174289177Speter 175289177Speter /* Verify that it matches the expected checksum. */ 176362181Sdim if (!svn_checksum_match(index_info->checksum, actual)) 177289177Speter { 178289177Speter const char *file_name; 179289177Speter 180362181Sdim SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); 181362181Sdim SVN_ERR(svn_checksum_mismatch_err(index_info->checksum, actual, 182362181Sdim scratch_pool, 183289177Speter _("%s checksum mismatch in file %s"), 184289177Speter name, file_name)); 185289177Speter } 186289177Speter 187289177Speter return SVN_NO_ERROR; 188289177Speter} 189289177Speter 190289177Speter/* Verify the MD5 checksums of the index data in the rev / pack file 191289177Speter * containing revision START in FS. If given, invoke CANCEL_FUNC with 192289177Speter * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary 193289177Speter * allocations. 194289177Speter */ 195289177Speterstatic svn_error_t * 196289177Speterverify_index_checksums(svn_fs_t *fs, 197289177Speter svn_revnum_t start, 198289177Speter svn_cancel_func_t cancel_func, 199289177Speter void *cancel_baton, 200289177Speter apr_pool_t *scratch_pool) 201289177Speter{ 202289177Speter svn_fs_x__revision_file_t *rev_file; 203362181Sdim svn_fs_x__index_info_t l2p_index_info; 204362181Sdim svn_fs_x__index_info_t p2l_index_info; 205289177Speter 206289177Speter /* Open the rev / pack file and read the footer */ 207362181Sdim SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 208362181Sdim SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); 209362181Sdim SVN_ERR(svn_fs_x__rev_file_p2l_info(&p2l_index_info, rev_file)); 210289177Speter 211289177Speter /* Verify the index contents against the checksum from the footer. */ 212362181Sdim SVN_ERR(verify_index_checksum(rev_file, "L2P index", &l2p_index_info, 213289177Speter cancel_func, cancel_baton, scratch_pool)); 214362181Sdim SVN_ERR(verify_index_checksum(rev_file, "P2L index", &p2l_index_info, 215289177Speter cancel_func, cancel_baton, scratch_pool)); 216289177Speter 217289177Speter /* Done. */ 218289177Speter SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 219289177Speter 220289177Speter return SVN_NO_ERROR; 221289177Speter} 222289177Speter 223289177Speter/* Verify that for all log-to-phys index entries for revisions START to 224289177Speter * START + COUNT-1 in FS there is a consistent entry in the phys-to-log 225289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 226289177Speter * intervals. Use SCRATCH_POOL for temporary allocations. 227289177Speter */ 228289177Speterstatic svn_error_t * 229289177Spetercompare_l2p_to_p2l_index(svn_fs_t *fs, 230289177Speter svn_revnum_t start, 231289177Speter svn_revnum_t count, 232289177Speter svn_cancel_func_t cancel_func, 233289177Speter void *cancel_baton, 234289177Speter apr_pool_t *scratch_pool) 235289177Speter{ 236289177Speter svn_revnum_t i; 237289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 238289177Speter apr_array_header_t *max_ids; 239289177Speter 240289177Speter /* common file access structure */ 241289177Speter svn_fs_x__revision_file_t *rev_file; 242362181Sdim SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 243289177Speter 244289177Speter /* determine the range of items to check for each revision */ 245289177Speter SVN_ERR(svn_fs_x__l2p_get_max_ids(&max_ids, fs, start, count, scratch_pool, 246289177Speter iterpool)); 247289177Speter 248289177Speter /* check all items in all revisions if the given range */ 249289177Speter for (i = 0; i < max_ids->nelts; ++i) 250289177Speter { 251289177Speter apr_uint64_t k; 252289177Speter apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t); 253289177Speter svn_revnum_t revision = start + i; 254289177Speter 255289177Speter for (k = 0; k < max_id; ++k) 256289177Speter { 257289177Speter apr_off_t offset; 258289177Speter apr_uint32_t sub_item; 259289177Speter svn_fs_x__id_t l2p_item; 260289177Speter svn_fs_x__id_t *p2l_item; 261289177Speter 262289177Speter l2p_item.change_set = svn_fs_x__change_set_by_rev(revision); 263289177Speter l2p_item.number = k; 264289177Speter 265289177Speter /* get L2P entry. Ignore unused entries. */ 266289177Speter SVN_ERR(svn_fs_x__item_offset(&offset, &sub_item, fs, rev_file, 267289177Speter &l2p_item, iterpool)); 268289177Speter if (offset == -1) 269289177Speter continue; 270289177Speter 271289177Speter /* find the corresponding P2L entry */ 272289177Speter SVN_ERR(svn_fs_x__p2l_item_lookup(&p2l_item, fs, rev_file, 273289177Speter revision, offset, sub_item, 274289177Speter iterpool, iterpool)); 275289177Speter 276289177Speter if (p2l_item == NULL) 277289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 278289177Speter NULL, 279289177Speter _("p2l index entry not found for " 280289177Speter "PHYS o%s:s%ld returned by " 281289177Speter "l2p index for LOG r%ld:i%ld"), 282289177Speter apr_off_t_toa(scratch_pool, offset), 283289177Speter (long)sub_item, revision, (long)k); 284289177Speter 285289177Speter if (!svn_fs_x__id_eq(&l2p_item, p2l_item)) 286289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 287289177Speter NULL, 288289177Speter _("p2l index info LOG r%ld:i%ld" 289289177Speter " does not match " 290289177Speter "l2p index for LOG r%ld:i%ld"), 291289177Speter svn_fs_x__get_revnum(p2l_item->change_set), 292289177Speter (long)p2l_item->number, revision, 293289177Speter (long)k); 294289177Speter 295289177Speter svn_pool_clear(iterpool); 296289177Speter } 297289177Speter 298289177Speter if (cancel_func) 299289177Speter SVN_ERR(cancel_func(cancel_baton)); 300289177Speter } 301289177Speter 302289177Speter svn_pool_destroy(iterpool); 303289177Speter 304289177Speter SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 305289177Speter 306289177Speter return SVN_NO_ERROR; 307289177Speter} 308289177Speter 309289177Speter/* Verify that for all phys-to-log index entries for revisions START to 310289177Speter * START + COUNT-1 in FS there is a consistent entry in the log-to-phys 311289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 312289177Speter * intervals. Use SCRATCH_POOL for temporary allocations. 313289177Speter * 314289177Speter * Please note that we can only check on pack / rev file granularity and 315289177Speter * must only be called for a single rev / pack file. 316289177Speter */ 317289177Speterstatic svn_error_t * 318289177Spetercompare_p2l_to_l2p_index(svn_fs_t *fs, 319289177Speter svn_revnum_t start, 320289177Speter svn_revnum_t count, 321289177Speter svn_cancel_func_t cancel_func, 322289177Speter void *cancel_baton, 323289177Speter apr_pool_t *scratch_pool) 324289177Speter{ 325289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 326289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 327289177Speter apr_pool_t *iterpool2 = svn_pool_create(scratch_pool); 328289177Speter apr_off_t max_offset; 329289177Speter apr_off_t offset = 0; 330289177Speter 331289177Speter /* common file access structure */ 332289177Speter svn_fs_x__revision_file_t *rev_file; 333362181Sdim SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 334289177Speter 335289177Speter /* get the size of the rev / pack file as covered by the P2L index */ 336289177Speter SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 337289177Speter scratch_pool)); 338289177Speter 339289177Speter /* for all offsets in the file, get the P2L index entries and check 340289177Speter them against the L2P index */ 341289177Speter for (offset = 0; offset < max_offset; ) 342289177Speter { 343289177Speter apr_array_header_t *entries; 344289177Speter svn_fs_x__p2l_entry_t *last_entry; 345289177Speter int i; 346289177Speter 347289177Speter svn_pool_clear(iterpool); 348289177Speter 349289177Speter /* get all entries for the current block */ 350289177Speter SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 351289177Speter offset, ffd->p2l_page_size, 352289177Speter iterpool, iterpool)); 353289177Speter if (entries->nelts == 0) 354289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 355289177Speter NULL, 356289177Speter _("p2l does not cover offset %s" 357289177Speter " for revision %ld"), 358289177Speter apr_off_t_toa(scratch_pool, offset), start); 359289177Speter 360289177Speter /* process all entries (and later continue with the next block) */ 361289177Speter last_entry 362289177Speter = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_x__p2l_entry_t); 363289177Speter offset = last_entry->offset + last_entry->size; 364289177Speter 365289177Speter for (i = 0; i < entries->nelts; ++i) 366289177Speter { 367289177Speter apr_uint32_t k; 368289177Speter svn_fs_x__p2l_entry_t *entry 369289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 370289177Speter 371289177Speter /* check all sub-items for consist entries in the L2P index */ 372289177Speter for (k = 0; k < entry->item_count; ++k) 373289177Speter { 374289177Speter apr_off_t l2p_offset; 375289177Speter apr_uint32_t sub_item; 376289177Speter svn_fs_x__id_t *p2l_item = &entry->items[k]; 377289177Speter svn_revnum_t revision 378289177Speter = svn_fs_x__get_revnum(p2l_item->change_set); 379289177Speter 380289177Speter svn_pool_clear(iterpool2); 381289177Speter SVN_ERR(svn_fs_x__item_offset(&l2p_offset, &sub_item, fs, 382289177Speter rev_file, p2l_item, iterpool2)); 383289177Speter 384289177Speter if (sub_item != k || l2p_offset != entry->offset) 385289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 386289177Speter NULL, 387289177Speter _("l2p index entry PHYS o%s:s%ld " 388289177Speter "does not match p2l index value " 389289177Speter "LOG r%ld:i%ld for PHYS o%s:s%ld"), 390289177Speter apr_off_t_toa(scratch_pool, 391289177Speter l2p_offset), 392289177Speter (long)sub_item, 393289177Speter revision, 394289177Speter (long)p2l_item->number, 395289177Speter apr_off_t_toa(scratch_pool, 396289177Speter entry->offset), 397289177Speter (long)k); 398289177Speter } 399289177Speter } 400289177Speter 401289177Speter if (cancel_func) 402289177Speter SVN_ERR(cancel_func(cancel_baton)); 403289177Speter } 404289177Speter 405289177Speter svn_pool_destroy(iterpool2); 406289177Speter svn_pool_destroy(iterpool); 407289177Speter 408289177Speter SVN_ERR(svn_fs_x__close_revision_file(rev_file)); 409289177Speter 410289177Speter return SVN_NO_ERROR; 411289177Speter} 412289177Speter 413289177Speter/* Items smaller than this can be read at once into a buffer and directly 414289177Speter * be checksummed. Larger items require stream processing. 415289177Speter * Must be a multiple of 8. */ 416289177Speter#define STREAM_THRESHOLD 4096 417289177Speter 418289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. SIZE must not 419289177Speter * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 420289177Speter */ 421289177Speterstatic svn_error_t * 422362181Sdimexpect_buffer_nul(svn_fs_x__revision_file_t *file, 423289177Speter apr_off_t size, 424289177Speter apr_pool_t *scratch_pool) 425289177Speter{ 426289177Speter union 427289177Speter { 428289177Speter unsigned char buffer[STREAM_THRESHOLD]; 429289177Speter apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)]; 430289177Speter } data; 431289177Speter 432289177Speter apr_size_t i; 433289177Speter SVN_ERR_ASSERT(size <= STREAM_THRESHOLD); 434289177Speter 435289177Speter /* read the whole data block; error out on failure */ 436289177Speter data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0; 437362181Sdim SVN_ERR(svn_fs_x__rev_file_read(file, data.buffer, size)); 438289177Speter 439289177Speter /* chunky check */ 440289177Speter for (i = 0; i < size / sizeof(apr_uint64_t); ++i) 441289177Speter if (data.chunks[i] != 0) 442289177Speter break; 443289177Speter 444289177Speter /* byte-wise check upon mismatch or at the end of the block */ 445289177Speter for (i *= sizeof(apr_uint64_t); i < size; ++i) 446289177Speter if (data.buffer[i] != 0) 447289177Speter { 448289177Speter const char *file_name; 449289177Speter apr_off_t offset; 450289177Speter 451362181Sdim SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); 452362181Sdim SVN_ERR(svn_fs_x__rev_file_offset(&offset, file)); 453289177Speter offset -= size - i; 454289177Speter 455289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 456289177Speter _("Empty section in file %s contains " 457289177Speter "non-NUL data at offset %s"), 458289177Speter file_name, 459289177Speter apr_off_t_toa(scratch_pool, offset)); 460289177Speter } 461289177Speter 462289177Speter return SVN_NO_ERROR; 463289177Speter} 464289177Speter 465289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. 466289177Speter * Use SCRATCH_POOL for temporary allocations. 467289177Speter */ 468289177Speterstatic svn_error_t * 469362181Sdimread_all_nul(svn_fs_x__revision_file_t *file, 470289177Speter apr_off_t size, 471289177Speter apr_pool_t *scratch_pool) 472289177Speter{ 473289177Speter for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD) 474289177Speter SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, scratch_pool)); 475289177Speter 476289177Speter if (size) 477289177Speter SVN_ERR(expect_buffer_nul(file, size, scratch_pool)); 478289177Speter 479289177Speter return SVN_NO_ERROR; 480289177Speter} 481289177Speter 482289177Speter/* Compare the ACTUAL checksum with the one expected by ENTRY. 483289177Speter * Return an error in case of mismatch. Use the name of FILE 484289177Speter * in error message. Allocate temporary data in SCRATCH_POOL. 485289177Speter */ 486289177Speterstatic svn_error_t * 487362181Sdimexpected_checksum(svn_fs_x__revision_file_t *file, 488289177Speter svn_fs_x__p2l_entry_t *entry, 489289177Speter apr_uint32_t actual, 490289177Speter apr_pool_t *scratch_pool) 491289177Speter{ 492289177Speter if (actual != entry->fnv1_checksum) 493289177Speter { 494289177Speter const char *file_name; 495289177Speter 496362181Sdim SVN_ERR(svn_fs_x__rev_file_name(&file_name, file, scratch_pool)); 497289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 498289177Speter _("Checksum mismatch in item at offset %s of " 499289177Speter "length %s bytes in file %s"), 500289177Speter apr_off_t_toa(scratch_pool, entry->offset), 501289177Speter apr_off_t_toa(scratch_pool, entry->size), 502289177Speter file_name); 503289177Speter } 504289177Speter 505289177Speter return SVN_NO_ERROR; 506289177Speter} 507289177Speter 508289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read 509289177Speter * from FILE will match ENTRY's expected checksum. SIZE must not 510289177Speter * exceed STREAM_THRESHOLD. Use SCRATCH_POOL for temporary allocations. 511289177Speter */ 512289177Speterstatic svn_error_t * 513362181Sdimexpected_buffered_checksum(svn_fs_x__revision_file_t *file, 514289177Speter svn_fs_x__p2l_entry_t *entry, 515289177Speter apr_pool_t *scratch_pool) 516289177Speter{ 517289177Speter unsigned char buffer[STREAM_THRESHOLD]; 518289177Speter SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD); 519289177Speter 520362181Sdim SVN_ERR(svn_fs_x__rev_file_read(file, buffer, (apr_size_t)entry->size)); 521289177Speter SVN_ERR(expected_checksum(file, entry, 522289177Speter svn__fnv1a_32x4(buffer, (apr_size_t)entry->size), 523289177Speter scratch_pool)); 524289177Speter 525289177Speter return SVN_NO_ERROR; 526289177Speter} 527289177Speter 528289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from 529289177Speter * FILE will match ENTRY's expected checksum. 530289177Speter * Use SCRATCH_POOL for temporary allocations. 531289177Speter */ 532289177Speterstatic svn_error_t * 533362181Sdimexpected_streamed_checksum(svn_fs_x__revision_file_t *file, 534289177Speter svn_fs_x__p2l_entry_t *entry, 535289177Speter apr_pool_t *scratch_pool) 536289177Speter{ 537289177Speter unsigned char buffer[STREAM_THRESHOLD]; 538289177Speter svn_checksum_t *checksum; 539289177Speter svn_checksum_ctx_t *context 540289177Speter = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, scratch_pool); 541289177Speter apr_off_t size = entry->size; 542289177Speter 543289177Speter while (size > 0) 544289177Speter { 545289177Speter apr_size_t to_read = size > sizeof(buffer) 546289177Speter ? sizeof(buffer) 547289177Speter : (apr_size_t)size; 548362181Sdim SVN_ERR(svn_fs_x__rev_file_read(file, buffer, to_read)); 549289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 550289177Speter size -= to_read; 551289177Speter } 552289177Speter 553289177Speter SVN_ERR(svn_checksum_final(&checksum, context, scratch_pool)); 554289177Speter SVN_ERR(expected_checksum(file, entry, 555289177Speter ntohl(*(const apr_uint32_t *)checksum->digest), 556289177Speter scratch_pool)); 557289177Speter 558289177Speter return SVN_NO_ERROR; 559289177Speter} 560289177Speter 561289177Speter/* Verify that for all phys-to-log index entries for revisions START to 562289177Speter * START + COUNT-1 in FS match the actual pack / rev file contents. 563289177Speter * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 564289177Speter * Use SCRATCH_POOL for temporary allocations. 565289177Speter * 566289177Speter * Please note that we can only check on pack / rev file granularity and 567289177Speter * must only be called for a single rev / pack file. 568289177Speter */ 569289177Speterstatic svn_error_t * 570289177Spetercompare_p2l_to_rev(svn_fs_t *fs, 571289177Speter svn_revnum_t start, 572289177Speter svn_revnum_t count, 573289177Speter svn_cancel_func_t cancel_func, 574289177Speter void *cancel_baton, 575289177Speter apr_pool_t *scratch_pool) 576289177Speter{ 577289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 578289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 579289177Speter apr_off_t max_offset; 580289177Speter apr_off_t offset = 0; 581289177Speter svn_fs_x__revision_file_t *rev_file; 582362181Sdim svn_fs_x__index_info_t l2p_index_info; 583289177Speter 584289177Speter /* open the pack / rev file that is covered by the p2l index */ 585362181Sdim SVN_ERR(svn_fs_x__rev_file_init(&rev_file, fs, start, scratch_pool)); 586289177Speter 587289177Speter /* check file size vs. range covered by index */ 588362181Sdim SVN_ERR(svn_fs_x__rev_file_l2p_info(&l2p_index_info, rev_file)); 589289177Speter SVN_ERR(svn_fs_x__p2l_get_max_offset(&max_offset, fs, rev_file, start, 590289177Speter scratch_pool)); 591289177Speter 592362181Sdim if (l2p_index_info.start != max_offset) 593289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, 594289177Speter _("File size of %s for revision r%ld does " 595289177Speter "not match p2l index size of %s"), 596289177Speter apr_off_t_toa(scratch_pool, 597362181Sdim l2p_index_info.start), 598289177Speter start, 599289177Speter apr_off_t_toa(scratch_pool, 600289177Speter max_offset)); 601289177Speter 602362181Sdim SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, 0)); 603289177Speter 604289177Speter /* for all offsets in the file, get the P2L index entries and check 605289177Speter them against the L2P index */ 606289177Speter for (offset = 0; offset < max_offset; ) 607289177Speter { 608289177Speter apr_array_header_t *entries; 609289177Speter int i; 610289177Speter 611289177Speter svn_pool_clear(iterpool); 612289177Speter 613289177Speter /* get all entries for the current block */ 614289177Speter SVN_ERR(svn_fs_x__p2l_index_lookup(&entries, fs, rev_file, start, 615289177Speter offset, ffd->p2l_page_size, 616289177Speter iterpool, iterpool)); 617289177Speter 618289177Speter /* The above might have moved the file pointer. 619289177Speter * Ensure we actually start reading at OFFSET. */ 620362181Sdim SVN_ERR(svn_fs_x__rev_file_seek(rev_file, NULL, offset)); 621289177Speter 622289177Speter /* process all entries (and later continue with the next block) */ 623289177Speter for (i = 0; i < entries->nelts; ++i) 624289177Speter { 625289177Speter svn_fs_x__p2l_entry_t *entry 626289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_x__p2l_entry_t); 627289177Speter 628289177Speter /* skip bits we previously checked */ 629289177Speter if (i == 0 && entry->offset < offset) 630289177Speter continue; 631289177Speter 632289177Speter /* skip zero-sized entries */ 633289177Speter if (entry->size == 0) 634289177Speter continue; 635289177Speter 636289177Speter /* p2l index must cover all rev / pack file offsets exactly once */ 637289177Speter if (entry->offset != offset) 638289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 639289177Speter NULL, 640289177Speter _("p2l index entry for revision r%ld" 641289177Speter " is non-contiguous between offsets " 642289177Speter " %s and %s"), 643289177Speter start, 644289177Speter apr_off_t_toa(scratch_pool, offset), 645289177Speter apr_off_t_toa(scratch_pool, 646289177Speter entry->offset)); 647289177Speter 648289177Speter /* empty sections must contain NUL bytes only */ 649289177Speter if (entry->type == SVN_FS_X__ITEM_TYPE_UNUSED) 650289177Speter { 651289177Speter /* skip filler entry at the end of the p2l index */ 652289177Speter if (entry->offset != max_offset) 653362181Sdim SVN_ERR(read_all_nul(rev_file, entry->size, iterpool)); 654289177Speter } 655289177Speter else 656289177Speter { 657289177Speter if (entry->size < STREAM_THRESHOLD) 658362181Sdim SVN_ERR(expected_buffered_checksum(rev_file, entry, 659289177Speter iterpool)); 660289177Speter else 661362181Sdim SVN_ERR(expected_streamed_checksum(rev_file, entry, 662289177Speter iterpool)); 663289177Speter } 664289177Speter 665289177Speter /* advance offset */ 666289177Speter offset += entry->size; 667289177Speter } 668289177Speter 669289177Speter if (cancel_func) 670289177Speter SVN_ERR(cancel_func(cancel_baton)); 671289177Speter } 672289177Speter 673289177Speter svn_pool_destroy(iterpool); 674289177Speter 675289177Speter return SVN_NO_ERROR; 676289177Speter} 677289177Speter 678289177Speter/* Verify that the revprops of the revisions START to END in FS can be 679289177Speter * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 680289177Speter * 681289177Speter * The values of START and END have already been auto-selected and 682289177Speter * verified. 683289177Speter */ 684289177Speterstatic svn_error_t * 685289177Speterverify_revprops(svn_fs_t *fs, 686289177Speter svn_revnum_t start, 687289177Speter svn_revnum_t end, 688289177Speter svn_cancel_func_t cancel_func, 689289177Speter void *cancel_baton, 690289177Speter apr_pool_t *scratch_pool) 691289177Speter{ 692289177Speter svn_revnum_t revision; 693289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 694289177Speter 695362181Sdim /* Invalidate the revprop generation once. 696362181Sdim * Use the cache inside the loop to speed up packed revprop access. */ 697362181Sdim svn_fs_x__invalidate_revprop_generation(fs); 698362181Sdim 699289177Speter for (revision = start; revision < end; ++revision) 700289177Speter { 701289177Speter svn_string_t *date; 702289177Speter apr_time_t timetemp; 703289177Speter 704289177Speter svn_pool_clear(iterpool); 705289177Speter 706289177Speter /* Access the svn:date revprop. 707289177Speter * This implies parsing all revprops for that revision. */ 708289177Speter SVN_ERR(svn_fs_x__revision_prop(&date, fs, revision, 709362181Sdim SVN_PROP_REVISION_DATE, FALSE, 710289177Speter iterpool, iterpool)); 711289177Speter 712289177Speter /* The time stamp is the only revprop that, if given, needs to 713289177Speter * have a valid content. */ 714289177Speter if (date) 715289177Speter SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool)); 716289177Speter 717289177Speter if (cancel_func) 718289177Speter SVN_ERR(cancel_func(cancel_baton)); 719289177Speter } 720289177Speter 721289177Speter svn_pool_destroy(iterpool); 722289177Speter 723289177Speter return SVN_NO_ERROR; 724289177Speter} 725289177Speter 726289177Speter/* Verify that on-disk representation has not been tempered with (in a way 727289177Speter * that leaves the repository in a corrupted state). This compares log-to- 728289177Speter * phys with phys-to-log indexes, verifies the low-level checksums and 729289177Speter * checks that all revprops are available. The function signature is 730289177Speter * similar to svn_fs_x__verify. 731289177Speter * 732289177Speter * The values of START and END have already been auto-selected and 733289177Speter * verified. 734289177Speter */ 735289177Speterstatic svn_error_t * 736289177Speterverify_metadata_consistency(svn_fs_t *fs, 737289177Speter svn_revnum_t start, 738289177Speter svn_revnum_t end, 739289177Speter svn_fs_progress_notify_func_t notify_func, 740289177Speter void *notify_baton, 741289177Speter svn_cancel_func_t cancel_func, 742289177Speter void *cancel_baton, 743289177Speter apr_pool_t *scratch_pool) 744289177Speter{ 745289177Speter svn_error_t *err; 746289177Speter svn_fs_x__data_t *ffd = fs->fsap_data; 747289177Speter svn_revnum_t revision, next_revision; 748289177Speter apr_pool_t *iterpool = svn_pool_create(scratch_pool); 749289177Speter 750289177Speter for (revision = start; revision <= end; revision = next_revision) 751289177Speter { 752289177Speter svn_revnum_t count = svn_fs_x__packed_base_rev(fs, revision); 753289177Speter svn_revnum_t pack_start = count; 754289177Speter svn_revnum_t pack_end = pack_start + svn_fs_x__pack_size(fs, revision); 755289177Speter 756289177Speter svn_pool_clear(iterpool); 757289177Speter 758289177Speter if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) 759289177Speter notify_func(pack_start, notify_baton, iterpool); 760289177Speter 761289177Speter /* Check for external corruption to the indexes. */ 762289177Speter err = verify_index_checksums(fs, pack_start, cancel_func, 763289177Speter cancel_baton, iterpool); 764289177Speter 765289177Speter /* two-way index check */ 766289177Speter if (!err) 767289177Speter err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, 768289177Speter cancel_func, cancel_baton, iterpool); 769289177Speter if (!err) 770289177Speter err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, 771289177Speter cancel_func, cancel_baton, iterpool); 772289177Speter 773289177Speter /* verify in-index checksums and types vs. actual rev / pack files */ 774289177Speter if (!err) 775289177Speter err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, 776289177Speter cancel_func, cancel_baton, iterpool); 777289177Speter 778289177Speter /* ensure that revprops are available and accessible */ 779289177Speter if (!err) 780289177Speter err = verify_revprops(fs, pack_start, pack_end, 781289177Speter cancel_func, cancel_baton, iterpool); 782289177Speter 783289177Speter /* concurrent packing is one of the reasons why verification may fail. 784289177Speter Make sure, we operate on up-to-date information. */ 785289177Speter if (err) 786362181Sdim { 787362181Sdim svn_error_t *err2 788362181Sdim = svn_fs_x__read_min_unpacked_rev(&ffd->min_unpacked_rev, 789362181Sdim fs, scratch_pool); 790289177Speter 791362181Sdim /* Be careful to not leak ERR. */ 792362181Sdim if (err2) 793362181Sdim return svn_error_trace(svn_error_compose_create(err, err2)); 794362181Sdim } 795362181Sdim 796289177Speter /* retry the whole shard if it got packed in the meantime */ 797289177Speter if (err && count != svn_fs_x__pack_size(fs, revision)) 798289177Speter { 799289177Speter svn_error_clear(err); 800289177Speter 801289177Speter /* We could simply assign revision here but the code below is 802289177Speter more intuitive to maintainers. */ 803289177Speter next_revision = svn_fs_x__packed_base_rev(fs, revision); 804289177Speter } 805289177Speter else 806289177Speter { 807289177Speter SVN_ERR(err); 808289177Speter next_revision = pack_end; 809289177Speter } 810289177Speter } 811289177Speter 812289177Speter svn_pool_destroy(iterpool); 813289177Speter 814289177Speter return SVN_NO_ERROR; 815289177Speter} 816289177Speter 817289177Spetersvn_error_t * 818289177Spetersvn_fs_x__verify(svn_fs_t *fs, 819289177Speter svn_revnum_t start, 820289177Speter svn_revnum_t end, 821289177Speter svn_fs_progress_notify_func_t notify_func, 822289177Speter void *notify_baton, 823289177Speter svn_cancel_func_t cancel_func, 824289177Speter void *cancel_baton, 825289177Speter apr_pool_t *scratch_pool) 826289177Speter{ 827289177Speter /* Input validation. */ 828289177Speter if (! SVN_IS_VALID_REVNUM(start)) 829289177Speter start = 0; 830289177Speter if (! SVN_IS_VALID_REVNUM(end)) 831362181Sdim { 832362181Sdim SVN_ERR(svn_fs_x__youngest_rev(&end, fs, scratch_pool)); 833362181Sdim } 834362181Sdim 835289177Speter SVN_ERR(svn_fs_x__ensure_revision_exists(start, fs, scratch_pool)); 836289177Speter SVN_ERR(svn_fs_x__ensure_revision_exists(end, fs, scratch_pool)); 837289177Speter 838289177Speter /* log/phys index consistency. We need to check them first to make 839289177Speter sure we can access the rev / pack files in format7. */ 840289177Speter SVN_ERR(verify_metadata_consistency(fs, start, end, 841289177Speter notify_func, notify_baton, 842289177Speter cancel_func, cancel_baton, 843289177Speter scratch_pool)); 844289177Speter 845289177Speter /* rep cache consistency */ 846289177Speter SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton, 847289177Speter cancel_func, cancel_baton, scratch_pool)); 848289177Speter 849289177Speter return SVN_NO_ERROR; 850289177Speter} 851