1289177Speter/* verify.c --- verification of FSFS filesystems 2289177Speter * 3289177Speter * ==================================================================== 4289177Speter * Licensed to the Apache Software Foundation (ASF) under one 5289177Speter * or more contributor license agreements. See the NOTICE file 6289177Speter * distributed with this work for additional information 7289177Speter * regarding copyright ownership. The ASF licenses this file 8289177Speter * to you under the Apache License, Version 2.0 (the 9289177Speter * "License"); you may not use this file except in compliance 10289177Speter * with the License. You may obtain a copy of the License at 11289177Speter * 12289177Speter * http://www.apache.org/licenses/LICENSE-2.0 13289177Speter * 14289177Speter * Unless required by applicable law or agreed to in writing, 15289177Speter * software distributed under the License is distributed on an 16289177Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17289177Speter * KIND, either express or implied. See the License for the 18289177Speter * specific language governing permissions and limitations 19289177Speter * under the License. 20289177Speter * ==================================================================== 21289177Speter */ 22289177Speter 23289177Speter#include "svn_sorts.h" 24289177Speter#include "svn_checksum.h" 25289177Speter#include "svn_time.h" 26289177Speter#include "private/svn_subr_private.h" 27289177Speter 28289177Speter#include "verify.h" 29289177Speter#include "fs_fs.h" 30289177Speter 31289177Speter#include "cached_data.h" 32289177Speter#include "rep-cache.h" 33289177Speter#include "util.h" 34289177Speter#include "index.h" 35289177Speter 36289177Speter#include "../libsvn_fs/fs-loader.h" 37289177Speter 38289177Speter#include "svn_private_config.h" 39289177Speter 40289177Speter 41289177Speter/** Verifying. **/ 42289177Speter 43289177Speter/* Baton type expected by verify_walker(). The purpose is to reuse open 44289177Speter * rev / pack file handles between calls. Its contents need to be cleaned 45289177Speter * periodically to limit resource usage. 46289177Speter */ 47289177Spetertypedef struct verify_walker_baton_t 48289177Speter{ 49289177Speter /* number of calls to verify_walker() since the last clean */ 50289177Speter int iteration_count; 51289177Speter 52289177Speter /* number of files opened since the last clean */ 53289177Speter int file_count; 54289177Speter 55289177Speter /* progress notification callback to invoke periodically (may be NULL) */ 56289177Speter svn_fs_progress_notify_func_t notify_func; 57289177Speter 58289177Speter /* baton to use with NOTIFY_FUNC */ 59289177Speter void *notify_baton; 60289177Speter 61289177Speter /* remember the last revision for which we called notify_func */ 62289177Speter svn_revnum_t last_notified_revision; 63289177Speter 64289177Speter /* cached hint for successive calls to svn_fs_fs__check_rep() */ 65289177Speter void *hint; 66289177Speter 67289177Speter /* pool to use for the file handles etc. */ 68289177Speter apr_pool_t *pool; 69289177Speter} verify_walker_baton_t; 70289177Speter 71289177Speter/* Used by svn_fs_fs__verify(). 72289177Speter Implements svn_fs_fs__walk_rep_reference().walker. */ 73289177Speterstatic svn_error_t * 74289177Speterverify_walker(representation_t *rep, 75289177Speter void *baton, 76289177Speter svn_fs_t *fs, 77289177Speter apr_pool_t *scratch_pool) 78289177Speter{ 79289177Speter verify_walker_baton_t *walker_baton = baton; 80289177Speter void *previous_hint; 81289177Speter 82289177Speter /* notify and free resources periodically */ 83289177Speter if ( walker_baton->iteration_count > 1000 84289177Speter || walker_baton->file_count > 16) 85289177Speter { 86289177Speter if ( walker_baton->notify_func 87289177Speter && rep->revision != walker_baton->last_notified_revision) 88289177Speter { 89289177Speter walker_baton->notify_func(rep->revision, 90289177Speter walker_baton->notify_baton, 91289177Speter scratch_pool); 92289177Speter walker_baton->last_notified_revision = rep->revision; 93289177Speter } 94289177Speter 95289177Speter svn_pool_clear(walker_baton->pool); 96289177Speter 97289177Speter walker_baton->iteration_count = 0; 98289177Speter walker_baton->file_count = 0; 99289177Speter walker_baton->hint = NULL; 100289177Speter } 101289177Speter 102289177Speter /* access the repo data */ 103289177Speter previous_hint = walker_baton->hint; 104289177Speter SVN_ERR(svn_fs_fs__check_rep(rep, fs, &walker_baton->hint, 105289177Speter walker_baton->pool)); 106289177Speter 107289177Speter /* update resource usage counters */ 108289177Speter walker_baton->iteration_count++; 109289177Speter if (previous_hint != walker_baton->hint) 110289177Speter walker_baton->file_count++; 111289177Speter 112289177Speter return SVN_NO_ERROR; 113289177Speter} 114289177Speter 115289177Speter/* Verify the rep cache DB's consistency with our rev / pack data. 116289177Speter * The function signature is similar to svn_fs_fs__verify. 117289177Speter * The values of START and END have already been auto-selected and 118289177Speter * verified. 119289177Speter */ 120289177Speterstatic svn_error_t * 121289177Speterverify_rep_cache(svn_fs_t *fs, 122289177Speter svn_revnum_t start, 123289177Speter svn_revnum_t end, 124289177Speter svn_fs_progress_notify_func_t notify_func, 125289177Speter void *notify_baton, 126289177Speter svn_cancel_func_t cancel_func, 127289177Speter void *cancel_baton, 128289177Speter apr_pool_t *pool) 129289177Speter{ 130289177Speter svn_boolean_t exists; 131289177Speter 132289177Speter /* rep-cache verification. */ 133289177Speter SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool)); 134289177Speter if (exists) 135289177Speter { 136289177Speter /* provide a baton to allow the reuse of open file handles between 137289177Speter iterations (saves 2/3 of OS level file operations). */ 138289177Speter verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton)); 139289177Speter baton->pool = svn_pool_create(pool); 140289177Speter baton->last_notified_revision = SVN_INVALID_REVNUM; 141289177Speter baton->notify_func = notify_func; 142289177Speter baton->notify_baton = notify_baton; 143289177Speter 144289177Speter /* tell the user that we are now ready to do *something* */ 145289177Speter if (notify_func) 146289177Speter notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool); 147289177Speter 148289177Speter /* Do not attempt to walk the rep-cache database if its file does 149289177Speter not exist, since doing so would create it --- which may confuse 150289177Speter the administrator. Don't take any lock. */ 151289177Speter SVN_ERR(svn_fs_fs__walk_rep_reference(fs, start, end, 152289177Speter verify_walker, baton, 153289177Speter cancel_func, cancel_baton, 154289177Speter pool)); 155289177Speter 156289177Speter /* walker resource cleanup */ 157289177Speter svn_pool_destroy(baton->pool); 158289177Speter } 159289177Speter 160289177Speter return SVN_NO_ERROR; 161289177Speter} 162289177Speter 163289177Speter/* Verify that the MD5 checksum of the data between offsets START and END 164289177Speter * in FILE matches the EXPECTED checksum. If there is a mismatch use the 165289177Speter * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC 166289177Speter * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ 167289177Speterstatic svn_error_t * 168289177Speterverify_index_checksum(apr_file_t *file, 169289177Speter const char *name, 170289177Speter apr_off_t start, 171289177Speter apr_off_t end, 172289177Speter svn_checksum_t *expected, 173289177Speter svn_cancel_func_t cancel_func, 174289177Speter void *cancel_baton, 175289177Speter apr_pool_t *scratch_pool) 176289177Speter{ 177289177Speter unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; 178289177Speter apr_off_t size = end - start; 179289177Speter svn_checksum_t *actual; 180289177Speter svn_checksum_ctx_t *context 181289177Speter = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); 182289177Speter 183289177Speter /* Calculate the index checksum. */ 184289177Speter SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool)); 185289177Speter while (size > 0) 186289177Speter { 187289177Speter apr_size_t to_read = size > sizeof(buffer) 188289177Speter ? sizeof(buffer) 189289177Speter : (apr_size_t)size; 190289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 191289177Speter scratch_pool)); 192289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 193289177Speter size -= to_read; 194289177Speter 195289177Speter if (cancel_func) 196289177Speter SVN_ERR(cancel_func(cancel_baton)); 197289177Speter } 198289177Speter 199289177Speter SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); 200289177Speter 201289177Speter /* Verify that it matches the expected checksum. */ 202289177Speter if (!svn_checksum_match(expected, actual)) 203289177Speter { 204289177Speter const char *file_name; 205289177Speter 206289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 207289177Speter SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, 208289177Speter _("%s checksum mismatch in file %s"), 209289177Speter name, file_name)); 210289177Speter } 211289177Speter 212289177Speter return SVN_NO_ERROR; 213289177Speter} 214289177Speter 215289177Speter/* Verify the MD5 checksums of the index data in the rev / pack file 216289177Speter * containing revision START in FS. If given, invoke CANCEL_FUNC with 217289177Speter * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary 218289177Speter * allocations. 219289177Speter */ 220289177Speterstatic svn_error_t * 221289177Speterverify_index_checksums(svn_fs_t *fs, 222289177Speter svn_revnum_t start, 223289177Speter svn_cancel_func_t cancel_func, 224289177Speter void *cancel_baton, 225289177Speter apr_pool_t *scratch_pool) 226289177Speter{ 227289177Speter svn_fs_fs__revision_file_t *rev_file; 228289177Speter 229289177Speter /* Open the rev / pack file and read the footer */ 230289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, 231289177Speter scratch_pool, scratch_pool)); 232289177Speter SVN_ERR(svn_fs_fs__auto_read_footer(rev_file)); 233289177Speter 234289177Speter /* Verify the index contents against the checksum from the footer. */ 235289177Speter SVN_ERR(verify_index_checksum(rev_file->file, "L2P index", 236289177Speter rev_file->l2p_offset, rev_file->p2l_offset, 237289177Speter rev_file->l2p_checksum, 238289177Speter cancel_func, cancel_baton, scratch_pool)); 239289177Speter SVN_ERR(verify_index_checksum(rev_file->file, "P2L index", 240289177Speter rev_file->p2l_offset, rev_file->footer_offset, 241289177Speter rev_file->p2l_checksum, 242289177Speter cancel_func, cancel_baton, scratch_pool)); 243289177Speter 244289177Speter /* Done. */ 245289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 246289177Speter 247289177Speter return SVN_NO_ERROR; 248289177Speter} 249289177Speter 250289177Speter/* Verify that for all log-to-phys index entries for revisions START to 251289177Speter * START + COUNT-1 in FS there is a consistent entry in the phys-to-log 252289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 253289177Speter * intervals. Use POOL for allocations. 254289177Speter */ 255289177Speterstatic svn_error_t * 256289177Spetercompare_l2p_to_p2l_index(svn_fs_t *fs, 257289177Speter svn_revnum_t start, 258289177Speter svn_revnum_t count, 259289177Speter svn_cancel_func_t cancel_func, 260289177Speter void *cancel_baton, 261289177Speter apr_pool_t *pool) 262289177Speter{ 263289177Speter svn_revnum_t i; 264289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 265289177Speter apr_array_header_t *max_ids; 266289177Speter 267289177Speter /* common file access structure */ 268289177Speter svn_fs_fs__revision_file_t *rev_file; 269289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool, 270289177Speter iterpool)); 271289177Speter 272289177Speter /* determine the range of items to check for each revision */ 273289177Speter SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool, 274289177Speter iterpool)); 275289177Speter 276289177Speter /* check all items in all revisions if the given range */ 277289177Speter for (i = 0; i < max_ids->nelts; ++i) 278289177Speter { 279289177Speter apr_uint64_t k; 280289177Speter apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t); 281289177Speter svn_revnum_t revision = start + i; 282289177Speter 283289177Speter for (k = 0; k < max_id; ++k) 284289177Speter { 285289177Speter apr_off_t offset; 286289177Speter svn_fs_fs__p2l_entry_t *p2l_entry; 287289177Speter svn_pool_clear(iterpool); 288289177Speter 289289177Speter /* get L2P entry. Ignore unused entries. */ 290289177Speter SVN_ERR(svn_fs_fs__item_offset(&offset, fs, rev_file, revision, 291289177Speter NULL, k, iterpool)); 292289177Speter if (offset == -1) 293289177Speter continue; 294289177Speter 295289177Speter /* find the corresponding P2L entry */ 296289177Speter SVN_ERR(svn_fs_fs__p2l_entry_lookup(&p2l_entry, fs, rev_file, 297289177Speter revision, offset, iterpool, 298289177Speter iterpool)); 299289177Speter 300289177Speter if (p2l_entry == NULL) 301289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 302289177Speter NULL, 303289177Speter _("p2l index entry not found for " 304289177Speter "PHYS %s returned by " 305289177Speter "l2p index for LOG r%ld:i%ld"), 306289177Speter apr_off_t_toa(pool, offset), 307289177Speter revision, (long)k); 308289177Speter 309289177Speter if ( p2l_entry->item.number != k 310289177Speter || p2l_entry->item.revision != revision) 311289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 312289177Speter NULL, 313289177Speter _("p2l index info LOG r%ld:i%ld" 314289177Speter " does not match " 315289177Speter "l2p index for LOG r%ld:i%ld"), 316289177Speter p2l_entry->item.revision, 317289177Speter (long)p2l_entry->item.number, 318289177Speter revision, (long)k); 319289177Speter } 320289177Speter 321289177Speter if (cancel_func) 322289177Speter SVN_ERR(cancel_func(cancel_baton)); 323289177Speter } 324289177Speter 325289177Speter svn_pool_destroy(iterpool); 326289177Speter 327289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 328289177Speter 329289177Speter return SVN_NO_ERROR; 330289177Speter} 331289177Speter 332289177Speter/* Verify that for all phys-to-log index entries for revisions START to 333289177Speter * START + COUNT-1 in FS there is a consistent entry in the log-to-phys 334289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 335289177Speter * intervals. Use POOL for allocations. 336289177Speter * 337289177Speter * Please note that we can only check on pack / rev file granularity and 338289177Speter * must only be called for a single rev / pack file. 339289177Speter */ 340289177Speterstatic svn_error_t * 341289177Spetercompare_p2l_to_l2p_index(svn_fs_t *fs, 342289177Speter svn_revnum_t start, 343289177Speter svn_revnum_t count, 344289177Speter svn_cancel_func_t cancel_func, 345289177Speter void *cancel_baton, 346289177Speter apr_pool_t *pool) 347289177Speter{ 348289177Speter fs_fs_data_t *ffd = fs->fsap_data; 349289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 350289177Speter apr_off_t max_offset; 351289177Speter apr_off_t offset = 0; 352289177Speter 353289177Speter /* common file access structure */ 354289177Speter svn_fs_fs__revision_file_t *rev_file; 355289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool, 356289177Speter iterpool)); 357289177Speter 358289177Speter /* get the size of the rev / pack file as covered by the P2L index */ 359289177Speter SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, rev_file, start, 360289177Speter pool)); 361289177Speter 362289177Speter /* for all offsets in the file, get the P2L index entries and check 363289177Speter them against the L2P index */ 364289177Speter for (offset = 0; offset < max_offset; ) 365289177Speter { 366289177Speter apr_array_header_t *entries; 367289177Speter svn_fs_fs__p2l_entry_t *last_entry; 368289177Speter int i; 369289177Speter 370289177Speter svn_pool_clear(iterpool); 371289177Speter 372289177Speter /* get all entries for the current block */ 373289177Speter SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start, 374289177Speter offset, ffd->p2l_page_size, 375289177Speter iterpool, iterpool)); 376289177Speter if (entries->nelts == 0) 377289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 378289177Speter NULL, 379289177Speter _("p2l does not cover offset %s" 380289177Speter " for revision %ld"), 381289177Speter apr_off_t_toa(pool, offset), start); 382289177Speter 383289177Speter /* process all entries (and later continue with the next block) */ 384289177Speter last_entry 385289177Speter = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_fs__p2l_entry_t); 386289177Speter offset = last_entry->offset + last_entry->size; 387289177Speter 388289177Speter for (i = 0; i < entries->nelts; ++i) 389289177Speter { 390289177Speter svn_fs_fs__p2l_entry_t *entry 391289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t); 392289177Speter 393289177Speter /* check all sub-items for consist entries in the L2P index */ 394289177Speter if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED) 395289177Speter { 396289177Speter /* There is no L2P entry for unused rev file sections. 397289177Speter * And its P2L index data is hardly ever used. But we 398289177Speter * should still check whether someone tempered with it. */ 399289177Speter if ( entry->item.revision != SVN_INVALID_REVNUM 400289177Speter && ( entry->item.revision < start 401289177Speter || entry->item.revision >= start + count)) 402289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 403289177Speter NULL, 404289177Speter _("Empty P2L entry for PHYS %s " 405289177Speter "refers to revision %ld outside " 406289177Speter "the rev / pack file (%ld-%ld)"), 407289177Speter apr_off_t_toa(pool, entry->offset), 408289177Speter entry->item.revision, 409289177Speter start, start + count - 1); 410289177Speter } 411289177Speter else 412289177Speter { 413289177Speter apr_off_t l2p_offset; 414289177Speter SVN_ERR(svn_fs_fs__item_offset(&l2p_offset, fs, rev_file, 415289177Speter entry->item.revision, NULL, 416289177Speter entry->item.number, iterpool)); 417289177Speter 418289177Speter if (l2p_offset != entry->offset) 419289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 420289177Speter NULL, 421289177Speter _("l2p index entry PHYS %s" 422289177Speter "does not match p2l index value " 423289177Speter "LOG r%ld:i%ld for PHYS %s"), 424289177Speter apr_off_t_toa(pool, l2p_offset), 425289177Speter entry->item.revision, 426289177Speter (long)entry->item.number, 427289177Speter apr_off_t_toa(pool, entry->offset)); 428289177Speter } 429289177Speter } 430289177Speter 431289177Speter if (cancel_func) 432289177Speter SVN_ERR(cancel_func(cancel_baton)); 433289177Speter } 434289177Speter 435289177Speter svn_pool_destroy(iterpool); 436289177Speter 437289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 438289177Speter 439289177Speter return SVN_NO_ERROR; 440289177Speter} 441289177Speter 442289177Speter/* Items smaller than this can be read at once into a buffer and directly 443289177Speter * be checksummed. Larger items require stream processing. 444289177Speter * Must be a multiple of 8. */ 445289177Speter#define STREAM_THRESHOLD 4096 446289177Speter 447289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. 448289177Speter * SIZE must not exceed STREAM_THRESHOLD. Use POOL for allocations. 449289177Speter */ 450289177Speterstatic svn_error_t * 451289177Speterexpect_buffer_nul(apr_file_t *file, 452289177Speter apr_off_t size, 453289177Speter apr_pool_t *pool) 454289177Speter{ 455289177Speter union 456289177Speter { 457289177Speter unsigned char buffer[STREAM_THRESHOLD]; 458289177Speter apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)]; 459289177Speter } data; 460289177Speter 461289177Speter apr_size_t i; 462289177Speter SVN_ERR_ASSERT(size <= STREAM_THRESHOLD); 463289177Speter 464289177Speter /* read the whole data block; error out on failure */ 465289177Speter data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0; 466289177Speter SVN_ERR(svn_io_file_read_full2(file, data.buffer, size, NULL, NULL, pool)); 467289177Speter 468289177Speter /* chunky check */ 469289177Speter for (i = 0; i < size / sizeof(apr_uint64_t); ++i) 470289177Speter if (data.chunks[i] != 0) 471289177Speter break; 472289177Speter 473289177Speter /* byte-wise check upon mismatch or at the end of the block */ 474289177Speter for (i *= sizeof(apr_uint64_t); i < size; ++i) 475289177Speter if (data.buffer[i] != 0) 476289177Speter { 477289177Speter const char *file_name; 478289177Speter apr_off_t offset; 479289177Speter 480289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, pool)); 481289177Speter SVN_ERR(svn_fs_fs__get_file_offset(&offset, file, pool)); 482289177Speter offset -= size - i; 483289177Speter 484289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 485289177Speter _("Empty section in file %s contains " 486289177Speter "non-NUL data at offset %s"), 487289177Speter file_name, apr_off_t_toa(pool, offset)); 488289177Speter } 489289177Speter 490289177Speter return SVN_NO_ERROR; 491289177Speter} 492289177Speter 493289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. 494289177Speter * Use POOL for allocations. 495289177Speter */ 496289177Speterstatic svn_error_t * 497289177Speterread_all_nul(apr_file_t *file, 498289177Speter apr_off_t size, 499289177Speter apr_pool_t *pool) 500289177Speter{ 501289177Speter for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD) 502289177Speter SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, pool)); 503289177Speter 504289177Speter if (size) 505289177Speter SVN_ERR(expect_buffer_nul(file, size, pool)); 506289177Speter 507289177Speter return SVN_NO_ERROR; 508289177Speter} 509289177Speter 510289177Speter/* Compare the ACTUAL checksum with the one expected by ENTRY. 511289177Speter * Return an error in case of mismatch. Use the name of FILE 512289177Speter * in error message. Allocate data in POOL. 513289177Speter */ 514289177Speterstatic svn_error_t * 515289177Speterexpected_checksum(apr_file_t *file, 516289177Speter svn_fs_fs__p2l_entry_t *entry, 517289177Speter apr_uint32_t actual, 518289177Speter apr_pool_t *pool) 519289177Speter{ 520289177Speter if (actual != entry->fnv1_checksum) 521289177Speter { 522289177Speter const char *file_name; 523289177Speter 524289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, pool)); 525289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 526289177Speter _("Checksum mismatch in item at offset %s of " 527289177Speter "length %s bytes in file %s"), 528289177Speter apr_off_t_toa(pool, entry->offset), 529289177Speter apr_off_t_toa(pool, entry->size), file_name); 530289177Speter } 531289177Speter 532289177Speter return SVN_NO_ERROR; 533289177Speter} 534289177Speter 535289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read 536289177Speter * from FILE will match ENTRY's expected checksum. SIZE must not 537289177Speter * exceed STREAM_THRESHOLD. Use POOL for allocations. 538289177Speter */ 539289177Speterstatic svn_error_t * 540289177Speterexpected_buffered_checksum(apr_file_t *file, 541289177Speter svn_fs_fs__p2l_entry_t *entry, 542289177Speter apr_pool_t *pool) 543289177Speter{ 544289177Speter unsigned char buffer[STREAM_THRESHOLD]; 545289177Speter SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD); 546289177Speter 547289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, (apr_size_t)entry->size, 548289177Speter NULL, NULL, pool)); 549289177Speter SVN_ERR(expected_checksum(file, entry, 550289177Speter svn__fnv1a_32x4(buffer, (apr_size_t)entry->size), 551289177Speter pool)); 552289177Speter 553289177Speter return SVN_NO_ERROR; 554289177Speter} 555289177Speter 556289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from 557289177Speter * FILE will match ENTRY's expected checksum. Use POOL for allocations. 558289177Speter */ 559289177Speterstatic svn_error_t * 560289177Speterexpected_streamed_checksum(apr_file_t *file, 561289177Speter svn_fs_fs__p2l_entry_t *entry, 562289177Speter apr_pool_t *pool) 563289177Speter{ 564289177Speter unsigned char buffer[STREAM_THRESHOLD]; 565289177Speter svn_checksum_t *checksum; 566289177Speter svn_checksum_ctx_t *context 567289177Speter = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool); 568289177Speter apr_off_t size = entry->size; 569289177Speter 570289177Speter while (size > 0) 571289177Speter { 572289177Speter apr_size_t to_read = size > sizeof(buffer) 573289177Speter ? sizeof(buffer) 574289177Speter : (apr_size_t)size; 575289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 576289177Speter pool)); 577289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 578289177Speter size -= to_read; 579289177Speter } 580289177Speter 581289177Speter SVN_ERR(svn_checksum_final(&checksum, context, pool)); 582289177Speter SVN_ERR(expected_checksum(file, entry, 583289177Speter ntohl(*(const apr_uint32_t *)checksum->digest), 584289177Speter pool)); 585289177Speter 586289177Speter return SVN_NO_ERROR; 587289177Speter} 588289177Speter 589289177Speter/* Verify that for all phys-to-log index entries for revisions START to 590289177Speter * START + COUNT-1 in FS match the actual pack / rev file contents. 591289177Speter * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 592289177Speter * Use POOL for allocations. 593289177Speter * 594289177Speter * Please note that we can only check on pack / rev file granularity and 595289177Speter * must only be called for a single rev / pack file. 596289177Speter */ 597289177Speterstatic svn_error_t * 598289177Spetercompare_p2l_to_rev(svn_fs_t *fs, 599289177Speter svn_revnum_t start, 600289177Speter svn_revnum_t count, 601289177Speter svn_cancel_func_t cancel_func, 602289177Speter void *cancel_baton, 603289177Speter apr_pool_t *pool) 604289177Speter{ 605289177Speter fs_fs_data_t *ffd = fs->fsap_data; 606289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 607289177Speter apr_off_t max_offset; 608289177Speter apr_off_t offset = 0; 609289177Speter svn_fs_fs__revision_file_t *rev_file; 610289177Speter 611289177Speter /* open the pack / rev file that is covered by the p2l index */ 612289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool, 613289177Speter iterpool)); 614289177Speter 615289177Speter /* check file size vs. range covered by index */ 616289177Speter SVN_ERR(svn_fs_fs__auto_read_footer(rev_file)); 617289177Speter SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, rev_file, start, 618289177Speter pool)); 619289177Speter 620289177Speter if (rev_file->l2p_offset != max_offset) 621289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, 622289177Speter _("File size of %s for revision r%ld does " 623289177Speter "not match p2l index size of %s"), 624289177Speter apr_off_t_toa(pool, rev_file->l2p_offset), start, 625289177Speter apr_off_t_toa(pool, max_offset)); 626289177Speter 627289177Speter SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0, 628289177Speter pool)); 629289177Speter 630289177Speter /* for all offsets in the file, get the P2L index entries and check 631289177Speter them against the L2P index */ 632289177Speter for (offset = 0; offset < max_offset; ) 633289177Speter { 634289177Speter apr_array_header_t *entries; 635289177Speter int i; 636289177Speter 637289177Speter svn_pool_clear(iterpool); 638289177Speter 639289177Speter /* get all entries for the current block */ 640289177Speter SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start, 641289177Speter offset, ffd->p2l_page_size, 642289177Speter iterpool, iterpool)); 643289177Speter 644289177Speter /* The above might have moved the file pointer. 645289177Speter * Ensure we actually start reading at OFFSET. */ 646289177Speter SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, 647289177Speter NULL, offset, iterpool)); 648289177Speter 649289177Speter /* process all entries (and later continue with the next block) */ 650289177Speter for (i = 0; i < entries->nelts; ++i) 651289177Speter { 652289177Speter svn_fs_fs__p2l_entry_t *entry 653289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t); 654289177Speter 655289177Speter /* skip bits we previously checked */ 656289177Speter if (i == 0 && entry->offset < offset) 657289177Speter continue; 658289177Speter 659289177Speter /* skip zero-sized entries */ 660289177Speter if (entry->size == 0) 661289177Speter continue; 662289177Speter 663289177Speter /* p2l index must cover all rev / pack file offsets exactly once */ 664289177Speter if (entry->offset != offset) 665289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 666289177Speter NULL, 667289177Speter _("p2l index entry for revision r%ld" 668289177Speter " is non-contiguous between offsets " 669289177Speter " %s and %s"), 670289177Speter start, 671289177Speter apr_off_t_toa(pool, offset), 672289177Speter apr_off_t_toa(pool, entry->offset)); 673289177Speter 674289177Speter /* empty sections must contain NUL bytes only */ 675289177Speter if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED) 676289177Speter { 677289177Speter /* skip filler entry at the end of the p2l index */ 678289177Speter if (entry->offset != max_offset) 679289177Speter SVN_ERR(read_all_nul(rev_file->file, entry->size, pool)); 680289177Speter } 681289177Speter else 682289177Speter { 683289177Speter if (entry->size < STREAM_THRESHOLD) 684289177Speter SVN_ERR(expected_buffered_checksum(rev_file->file, entry, 685289177Speter pool)); 686289177Speter else 687289177Speter SVN_ERR(expected_streamed_checksum(rev_file->file, entry, 688289177Speter pool)); 689289177Speter } 690289177Speter 691289177Speter /* advance offset */ 692289177Speter offset += entry->size; 693289177Speter } 694289177Speter 695289177Speter if (cancel_func) 696289177Speter SVN_ERR(cancel_func(cancel_baton)); 697289177Speter } 698289177Speter 699289177Speter svn_pool_destroy(iterpool); 700289177Speter 701289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 702289177Speter 703289177Speter return SVN_NO_ERROR; 704289177Speter} 705289177Speter 706289177Speter/* Verify that the revprops of the revisions START to END in FS can be 707289177Speter * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 708289177Speter * 709289177Speter * The values of START and END have already been auto-selected and 710289177Speter * verified. 711289177Speter */ 712289177Speterstatic svn_error_t * 713289177Speterverify_revprops(svn_fs_t *fs, 714289177Speter svn_revnum_t start, 715289177Speter svn_revnum_t end, 716289177Speter svn_cancel_func_t cancel_func, 717289177Speter void *cancel_baton, 718289177Speter apr_pool_t *pool) 719289177Speter{ 720289177Speter svn_revnum_t revision; 721289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 722289177Speter 723289177Speter for (revision = start; revision < end; ++revision) 724289177Speter { 725289177Speter svn_string_t *date; 726289177Speter apr_time_t timetemp; 727289177Speter 728289177Speter svn_pool_clear(iterpool); 729289177Speter 730289177Speter /* Access the svn:date revprop. 731289177Speter * This implies parsing all revprops for that revision. */ 732289177Speter SVN_ERR(svn_fs_fs__revision_prop(&date, fs, revision, 733289177Speter SVN_PROP_REVISION_DATE, iterpool)); 734289177Speter 735289177Speter /* The time stamp is the only revprop that, if given, needs to 736289177Speter * have a valid content. */ 737289177Speter if (date) 738289177Speter SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool)); 739289177Speter 740289177Speter if (cancel_func) 741289177Speter SVN_ERR(cancel_func(cancel_baton)); 742289177Speter } 743289177Speter 744289177Speter svn_pool_destroy(iterpool); 745289177Speter 746289177Speter return SVN_NO_ERROR; 747289177Speter} 748289177Speter 749289177Speterstatic svn_revnum_t 750289177Speterpack_size(svn_fs_t *fs, svn_revnum_t rev) 751289177Speter{ 752289177Speter fs_fs_data_t *ffd = fs->fsap_data; 753289177Speter 754289177Speter return rev < ffd->min_unpacked_rev ? ffd->max_files_per_dir : 1; 755289177Speter} 756289177Speter 757289177Speter/* Verify that on-disk representation has not been tempered with (in a way 758289177Speter * that leaves the repository in a corrupted state). This compares log-to- 759289177Speter * phys with phys-to-log indexes, verifies the low-level checksums and 760289177Speter * checks that all revprops are available. The function signature is 761289177Speter * similar to svn_fs_fs__verify. 762289177Speter * 763289177Speter * The values of START and END have already been auto-selected and 764289177Speter * verified. You may call this for format7 or higher repos. 765289177Speter */ 766289177Speterstatic svn_error_t * 767289177Speterverify_f7_metadata_consistency(svn_fs_t *fs, 768289177Speter svn_revnum_t start, 769289177Speter svn_revnum_t end, 770289177Speter svn_fs_progress_notify_func_t notify_func, 771289177Speter void *notify_baton, 772289177Speter svn_cancel_func_t cancel_func, 773289177Speter void *cancel_baton, 774289177Speter apr_pool_t *pool) 775289177Speter{ 776289177Speter fs_fs_data_t *ffd = fs->fsap_data; 777289177Speter svn_revnum_t revision, next_revision; 778289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 779289177Speter 780289177Speter for (revision = start; revision <= end; revision = next_revision) 781289177Speter { 782289177Speter svn_error_t *err = SVN_NO_ERROR; 783289177Speter 784289177Speter svn_revnum_t count = pack_size(fs, revision); 785289177Speter svn_revnum_t pack_start = svn_fs_fs__packed_base_rev(fs, revision); 786289177Speter svn_revnum_t pack_end = pack_start + count; 787289177Speter 788289177Speter svn_pool_clear(iterpool); 789289177Speter 790289177Speter if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) 791289177Speter notify_func(pack_start, notify_baton, iterpool); 792289177Speter 793289177Speter /* Check for external corruption to the indexes. */ 794289177Speter err = verify_index_checksums(fs, pack_start, cancel_func, 795289177Speter cancel_baton, iterpool); 796289177Speter 797289177Speter /* two-way index check */ 798289177Speter if (!err) 799289177Speter err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, 800289177Speter cancel_func, cancel_baton, iterpool); 801289177Speter if (!err) 802289177Speter err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, 803289177Speter cancel_func, cancel_baton, iterpool); 804289177Speter 805289177Speter /* verify in-index checksums and types vs. actual rev / pack files */ 806289177Speter if (!err) 807289177Speter err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, 808289177Speter cancel_func, cancel_baton, iterpool); 809289177Speter 810289177Speter /* ensure that revprops are available and accessible */ 811289177Speter if (!err) 812289177Speter err = verify_revprops(fs, pack_start, pack_end, 813289177Speter cancel_func, cancel_baton, iterpool); 814289177Speter 815289177Speter /* concurrent packing is one of the reasons why verification may fail. 816289177Speter Make sure, we operate on up-to-date information. */ 817289177Speter if (err) 818289177Speter { 819289177Speter svn_error_t *err2 820289177Speter = svn_fs_fs__read_min_unpacked_rev(&ffd->min_unpacked_rev, 821289177Speter fs, pool); 822289177Speter 823289177Speter /* Be careful to not leak ERR. */ 824289177Speter if (err2) 825289177Speter return svn_error_trace(svn_error_compose_create(err, err2)); 826289177Speter } 827289177Speter 828289177Speter /* retry the whole shard if it got packed in the meantime */ 829289177Speter if (err && count != pack_size(fs, revision)) 830289177Speter { 831289177Speter svn_error_clear(err); 832289177Speter 833289177Speter /* We could simply assign revision here but the code below is 834289177Speter more intuitive to maintainers. */ 835289177Speter next_revision = svn_fs_fs__packed_base_rev(fs, revision); 836289177Speter } 837289177Speter else 838289177Speter { 839289177Speter SVN_ERR(err); 840289177Speter next_revision = pack_end; 841289177Speter } 842289177Speter } 843289177Speter 844289177Speter svn_pool_destroy(iterpool); 845289177Speter 846289177Speter return SVN_NO_ERROR; 847289177Speter} 848289177Speter 849289177Spetersvn_error_t * 850289177Spetersvn_fs_fs__verify(svn_fs_t *fs, 851289177Speter svn_revnum_t start, 852289177Speter svn_revnum_t end, 853289177Speter svn_fs_progress_notify_func_t notify_func, 854289177Speter void *notify_baton, 855289177Speter svn_cancel_func_t cancel_func, 856289177Speter void *cancel_baton, 857289177Speter apr_pool_t *pool) 858289177Speter{ 859289177Speter fs_fs_data_t *ffd = fs->fsap_data; 860289177Speter svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ 861289177Speter 862289177Speter /* Input validation. */ 863289177Speter if (! SVN_IS_VALID_REVNUM(start)) 864289177Speter start = 0; 865289177Speter if (! SVN_IS_VALID_REVNUM(end)) 866289177Speter end = youngest; 867289177Speter SVN_ERR(svn_fs_fs__ensure_revision_exists(start, fs, pool)); 868289177Speter SVN_ERR(svn_fs_fs__ensure_revision_exists(end, fs, pool)); 869289177Speter 870289177Speter /* log/phys index consistency. We need to check them first to make 871289177Speter sure we can access the rev / pack files in format7. */ 872289177Speter if (svn_fs_fs__use_log_addressing(fs)) 873289177Speter SVN_ERR(verify_f7_metadata_consistency(fs, start, end, 874289177Speter notify_func, notify_baton, 875289177Speter cancel_func, cancel_baton, pool)); 876289177Speter 877289177Speter /* rep cache consistency */ 878289177Speter if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) 879289177Speter SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton, 880289177Speter cancel_func, cancel_baton, pool)); 881289177Speter 882289177Speter return SVN_NO_ERROR; 883289177Speter} 884