1289177Speter/* verify.c --- verification of FSFS filesystems 2289177Speter * 3289177Speter * ==================================================================== 4289177Speter * Licensed to the Apache Software Foundation (ASF) under one 5289177Speter * or more contributor license agreements. See the NOTICE file 6289177Speter * distributed with this work for additional information 7289177Speter * regarding copyright ownership. The ASF licenses this file 8289177Speter * to you under the Apache License, Version 2.0 (the 9289177Speter * "License"); you may not use this file except in compliance 10289177Speter * with the License. You may obtain a copy of the License at 11289177Speter * 12289177Speter * http://www.apache.org/licenses/LICENSE-2.0 13289177Speter * 14289177Speter * Unless required by applicable law or agreed to in writing, 15289177Speter * software distributed under the License is distributed on an 16289177Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17289177Speter * KIND, either express or implied. See the License for the 18289177Speter * specific language governing permissions and limitations 19289177Speter * under the License. 20289177Speter * ==================================================================== 21289177Speter */ 22289177Speter 23289177Speter#include "svn_sorts.h" 24289177Speter#include "svn_checksum.h" 25289177Speter#include "svn_time.h" 26289177Speter#include "private/svn_subr_private.h" 27289177Speter 28289177Speter#include "verify.h" 29289177Speter#include "fs_fs.h" 30289177Speter 31289177Speter#include "cached_data.h" 32289177Speter#include "rep-cache.h" 33362181Sdim#include "revprops.h" 34289177Speter#include "util.h" 35289177Speter#include "index.h" 36289177Speter 37289177Speter#include "../libsvn_fs/fs-loader.h" 38289177Speter 39289177Speter#include "svn_private_config.h" 40289177Speter 41289177Speter 42289177Speter/** Verifying. **/ 43289177Speter 44289177Speter/* Baton type expected by verify_walker(). The purpose is to reuse open 45289177Speter * rev / pack file handles between calls. Its contents need to be cleaned 46289177Speter * periodically to limit resource usage. 47289177Speter */ 48289177Spetertypedef struct verify_walker_baton_t 49289177Speter{ 50289177Speter /* number of calls to verify_walker() since the last clean */ 51289177Speter int iteration_count; 52289177Speter 53289177Speter /* number of files opened since the last clean */ 54289177Speter int file_count; 55289177Speter 56289177Speter /* progress notification callback to invoke periodically (may be NULL) */ 57289177Speter svn_fs_progress_notify_func_t notify_func; 58289177Speter 59289177Speter /* baton to use with NOTIFY_FUNC */ 60289177Speter void *notify_baton; 61289177Speter 62289177Speter /* remember the last revision for which we called notify_func */ 63289177Speter svn_revnum_t last_notified_revision; 64289177Speter 65289177Speter /* cached hint for successive calls to svn_fs_fs__check_rep() */ 66289177Speter void *hint; 67289177Speter 68289177Speter /* pool to use for the file handles etc. */ 69289177Speter apr_pool_t *pool; 70289177Speter} verify_walker_baton_t; 71289177Speter 72289177Speter/* Used by svn_fs_fs__verify(). 73289177Speter Implements svn_fs_fs__walk_rep_reference().walker. */ 74289177Speterstatic svn_error_t * 75289177Speterverify_walker(representation_t *rep, 76289177Speter void *baton, 77289177Speter svn_fs_t *fs, 78289177Speter apr_pool_t *scratch_pool) 79289177Speter{ 80289177Speter verify_walker_baton_t *walker_baton = baton; 81289177Speter void *previous_hint; 82289177Speter 83289177Speter /* notify and free resources periodically */ 84289177Speter if ( walker_baton->iteration_count > 1000 85289177Speter || walker_baton->file_count > 16) 86289177Speter { 87289177Speter if ( walker_baton->notify_func 88289177Speter && rep->revision != walker_baton->last_notified_revision) 89289177Speter { 90289177Speter walker_baton->notify_func(rep->revision, 91289177Speter walker_baton->notify_baton, 92289177Speter scratch_pool); 93289177Speter walker_baton->last_notified_revision = rep->revision; 94289177Speter } 95289177Speter 96289177Speter svn_pool_clear(walker_baton->pool); 97289177Speter 98289177Speter walker_baton->iteration_count = 0; 99289177Speter walker_baton->file_count = 0; 100289177Speter walker_baton->hint = NULL; 101289177Speter } 102289177Speter 103289177Speter /* access the repo data */ 104289177Speter previous_hint = walker_baton->hint; 105289177Speter SVN_ERR(svn_fs_fs__check_rep(rep, fs, &walker_baton->hint, 106289177Speter walker_baton->pool)); 107289177Speter 108289177Speter /* update resource usage counters */ 109289177Speter walker_baton->iteration_count++; 110289177Speter if (previous_hint != walker_baton->hint) 111289177Speter walker_baton->file_count++; 112289177Speter 113289177Speter return SVN_NO_ERROR; 114289177Speter} 115289177Speter 116289177Speter/* Verify the rep cache DB's consistency with our rev / pack data. 117289177Speter * The function signature is similar to svn_fs_fs__verify. 118289177Speter * The values of START and END have already been auto-selected and 119289177Speter * verified. 120289177Speter */ 121289177Speterstatic svn_error_t * 122289177Speterverify_rep_cache(svn_fs_t *fs, 123289177Speter svn_revnum_t start, 124289177Speter svn_revnum_t end, 125289177Speter svn_fs_progress_notify_func_t notify_func, 126289177Speter void *notify_baton, 127289177Speter svn_cancel_func_t cancel_func, 128289177Speter void *cancel_baton, 129289177Speter apr_pool_t *pool) 130289177Speter{ 131289177Speter svn_boolean_t exists; 132289177Speter 133289177Speter /* rep-cache verification. */ 134289177Speter SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool)); 135289177Speter if (exists) 136289177Speter { 137289177Speter /* provide a baton to allow the reuse of open file handles between 138289177Speter iterations (saves 2/3 of OS level file operations). */ 139289177Speter verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton)); 140289177Speter baton->pool = svn_pool_create(pool); 141289177Speter baton->last_notified_revision = SVN_INVALID_REVNUM; 142289177Speter baton->notify_func = notify_func; 143289177Speter baton->notify_baton = notify_baton; 144289177Speter 145289177Speter /* tell the user that we are now ready to do *something* */ 146289177Speter if (notify_func) 147289177Speter notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool); 148289177Speter 149289177Speter /* Do not attempt to walk the rep-cache database if its file does 150289177Speter not exist, since doing so would create it --- which may confuse 151289177Speter the administrator. Don't take any lock. */ 152289177Speter SVN_ERR(svn_fs_fs__walk_rep_reference(fs, start, end, 153289177Speter verify_walker, baton, 154289177Speter cancel_func, cancel_baton, 155289177Speter pool)); 156289177Speter 157289177Speter /* walker resource cleanup */ 158289177Speter svn_pool_destroy(baton->pool); 159289177Speter } 160289177Speter 161289177Speter return SVN_NO_ERROR; 162289177Speter} 163289177Speter 164289177Speter/* Verify that the MD5 checksum of the data between offsets START and END 165289177Speter * in FILE matches the EXPECTED checksum. If there is a mismatch use the 166289177Speter * indedx NAME in the error message. Supports cancellation with CANCEL_FUNC 167289177Speter * and CANCEL_BATON. SCRATCH_POOL is for temporary allocations. */ 168289177Speterstatic svn_error_t * 169289177Speterverify_index_checksum(apr_file_t *file, 170289177Speter const char *name, 171289177Speter apr_off_t start, 172289177Speter apr_off_t end, 173289177Speter svn_checksum_t *expected, 174289177Speter svn_cancel_func_t cancel_func, 175289177Speter void *cancel_baton, 176289177Speter apr_pool_t *scratch_pool) 177289177Speter{ 178289177Speter unsigned char buffer[SVN__STREAM_CHUNK_SIZE]; 179289177Speter apr_off_t size = end - start; 180289177Speter svn_checksum_t *actual; 181289177Speter svn_checksum_ctx_t *context 182289177Speter = svn_checksum_ctx_create(svn_checksum_md5, scratch_pool); 183289177Speter 184289177Speter /* Calculate the index checksum. */ 185289177Speter SVN_ERR(svn_io_file_seek(file, APR_SET, &start, scratch_pool)); 186289177Speter while (size > 0) 187289177Speter { 188289177Speter apr_size_t to_read = size > sizeof(buffer) 189289177Speter ? sizeof(buffer) 190289177Speter : (apr_size_t)size; 191289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 192289177Speter scratch_pool)); 193289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 194289177Speter size -= to_read; 195289177Speter 196289177Speter if (cancel_func) 197289177Speter SVN_ERR(cancel_func(cancel_baton)); 198289177Speter } 199289177Speter 200289177Speter SVN_ERR(svn_checksum_final(&actual, context, scratch_pool)); 201289177Speter 202289177Speter /* Verify that it matches the expected checksum. */ 203289177Speter if (!svn_checksum_match(expected, actual)) 204289177Speter { 205289177Speter const char *file_name; 206289177Speter 207289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, scratch_pool)); 208289177Speter SVN_ERR(svn_checksum_mismatch_err(expected, actual, scratch_pool, 209289177Speter _("%s checksum mismatch in file %s"), 210289177Speter name, file_name)); 211289177Speter } 212289177Speter 213289177Speter return SVN_NO_ERROR; 214289177Speter} 215289177Speter 216289177Speter/* Verify the MD5 checksums of the index data in the rev / pack file 217289177Speter * containing revision START in FS. If given, invoke CANCEL_FUNC with 218289177Speter * CANCEL_BATON at regular intervals. Use SCRATCH_POOL for temporary 219289177Speter * allocations. 220289177Speter */ 221289177Speterstatic svn_error_t * 222289177Speterverify_index_checksums(svn_fs_t *fs, 223289177Speter svn_revnum_t start, 224289177Speter svn_cancel_func_t cancel_func, 225289177Speter void *cancel_baton, 226289177Speter apr_pool_t *scratch_pool) 227289177Speter{ 228289177Speter svn_fs_fs__revision_file_t *rev_file; 229289177Speter 230289177Speter /* Open the rev / pack file and read the footer */ 231289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, 232289177Speter scratch_pool, scratch_pool)); 233289177Speter SVN_ERR(svn_fs_fs__auto_read_footer(rev_file)); 234289177Speter 235289177Speter /* Verify the index contents against the checksum from the footer. */ 236289177Speter SVN_ERR(verify_index_checksum(rev_file->file, "L2P index", 237289177Speter rev_file->l2p_offset, rev_file->p2l_offset, 238289177Speter rev_file->l2p_checksum, 239289177Speter cancel_func, cancel_baton, scratch_pool)); 240289177Speter SVN_ERR(verify_index_checksum(rev_file->file, "P2L index", 241289177Speter rev_file->p2l_offset, rev_file->footer_offset, 242289177Speter rev_file->p2l_checksum, 243289177Speter cancel_func, cancel_baton, scratch_pool)); 244289177Speter 245289177Speter /* Done. */ 246289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 247289177Speter 248289177Speter return SVN_NO_ERROR; 249289177Speter} 250289177Speter 251289177Speter/* Verify that for all log-to-phys index entries for revisions START to 252289177Speter * START + COUNT-1 in FS there is a consistent entry in the phys-to-log 253289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 254289177Speter * intervals. Use POOL for allocations. 255289177Speter */ 256289177Speterstatic svn_error_t * 257289177Spetercompare_l2p_to_p2l_index(svn_fs_t *fs, 258289177Speter svn_revnum_t start, 259289177Speter svn_revnum_t count, 260289177Speter svn_cancel_func_t cancel_func, 261289177Speter void *cancel_baton, 262289177Speter apr_pool_t *pool) 263289177Speter{ 264289177Speter svn_revnum_t i; 265289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 266289177Speter apr_array_header_t *max_ids; 267289177Speter 268289177Speter /* common file access structure */ 269289177Speter svn_fs_fs__revision_file_t *rev_file; 270289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool, 271289177Speter iterpool)); 272289177Speter 273289177Speter /* determine the range of items to check for each revision */ 274289177Speter SVN_ERR(svn_fs_fs__l2p_get_max_ids(&max_ids, fs, start, count, pool, 275289177Speter iterpool)); 276289177Speter 277289177Speter /* check all items in all revisions if the given range */ 278289177Speter for (i = 0; i < max_ids->nelts; ++i) 279289177Speter { 280289177Speter apr_uint64_t k; 281289177Speter apr_uint64_t max_id = APR_ARRAY_IDX(max_ids, i, apr_uint64_t); 282289177Speter svn_revnum_t revision = start + i; 283289177Speter 284289177Speter for (k = 0; k < max_id; ++k) 285289177Speter { 286289177Speter apr_off_t offset; 287289177Speter svn_fs_fs__p2l_entry_t *p2l_entry; 288289177Speter svn_pool_clear(iterpool); 289289177Speter 290289177Speter /* get L2P entry. Ignore unused entries. */ 291289177Speter SVN_ERR(svn_fs_fs__item_offset(&offset, fs, rev_file, revision, 292289177Speter NULL, k, iterpool)); 293289177Speter if (offset == -1) 294289177Speter continue; 295289177Speter 296289177Speter /* find the corresponding P2L entry */ 297289177Speter SVN_ERR(svn_fs_fs__p2l_entry_lookup(&p2l_entry, fs, rev_file, 298289177Speter revision, offset, iterpool, 299289177Speter iterpool)); 300289177Speter 301289177Speter if (p2l_entry == NULL) 302289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 303289177Speter NULL, 304289177Speter _("p2l index entry not found for " 305289177Speter "PHYS %s returned by " 306289177Speter "l2p index for LOG r%ld:i%ld"), 307289177Speter apr_off_t_toa(pool, offset), 308289177Speter revision, (long)k); 309289177Speter 310289177Speter if ( p2l_entry->item.number != k 311289177Speter || p2l_entry->item.revision != revision) 312289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 313289177Speter NULL, 314289177Speter _("p2l index info LOG r%ld:i%ld" 315289177Speter " does not match " 316289177Speter "l2p index for LOG r%ld:i%ld"), 317289177Speter p2l_entry->item.revision, 318289177Speter (long)p2l_entry->item.number, 319289177Speter revision, (long)k); 320289177Speter } 321289177Speter 322289177Speter if (cancel_func) 323289177Speter SVN_ERR(cancel_func(cancel_baton)); 324289177Speter } 325289177Speter 326289177Speter svn_pool_destroy(iterpool); 327289177Speter 328289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 329289177Speter 330289177Speter return SVN_NO_ERROR; 331289177Speter} 332289177Speter 333289177Speter/* Verify that for all phys-to-log index entries for revisions START to 334289177Speter * START + COUNT-1 in FS there is a consistent entry in the log-to-phys 335289177Speter * index. If given, invoke CANCEL_FUNC with CANCEL_BATON at regular 336289177Speter * intervals. Use POOL for allocations. 337289177Speter * 338289177Speter * Please note that we can only check on pack / rev file granularity and 339289177Speter * must only be called for a single rev / pack file. 340289177Speter */ 341289177Speterstatic svn_error_t * 342289177Spetercompare_p2l_to_l2p_index(svn_fs_t *fs, 343289177Speter svn_revnum_t start, 344289177Speter svn_revnum_t count, 345289177Speter svn_cancel_func_t cancel_func, 346289177Speter void *cancel_baton, 347289177Speter apr_pool_t *pool) 348289177Speter{ 349289177Speter fs_fs_data_t *ffd = fs->fsap_data; 350289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 351289177Speter apr_off_t max_offset; 352289177Speter apr_off_t offset = 0; 353289177Speter 354289177Speter /* common file access structure */ 355289177Speter svn_fs_fs__revision_file_t *rev_file; 356289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool, 357289177Speter iterpool)); 358289177Speter 359289177Speter /* get the size of the rev / pack file as covered by the P2L index */ 360289177Speter SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, rev_file, start, 361289177Speter pool)); 362289177Speter 363289177Speter /* for all offsets in the file, get the P2L index entries and check 364289177Speter them against the L2P index */ 365289177Speter for (offset = 0; offset < max_offset; ) 366289177Speter { 367289177Speter apr_array_header_t *entries; 368289177Speter svn_fs_fs__p2l_entry_t *last_entry; 369289177Speter int i; 370289177Speter 371289177Speter svn_pool_clear(iterpool); 372289177Speter 373289177Speter /* get all entries for the current block */ 374289177Speter SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start, 375289177Speter offset, ffd->p2l_page_size, 376289177Speter iterpool, iterpool)); 377289177Speter if (entries->nelts == 0) 378289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 379289177Speter NULL, 380289177Speter _("p2l does not cover offset %s" 381289177Speter " for revision %ld"), 382289177Speter apr_off_t_toa(pool, offset), start); 383289177Speter 384289177Speter /* process all entries (and later continue with the next block) */ 385289177Speter last_entry 386289177Speter = &APR_ARRAY_IDX(entries, entries->nelts-1, svn_fs_fs__p2l_entry_t); 387289177Speter offset = last_entry->offset + last_entry->size; 388289177Speter 389289177Speter for (i = 0; i < entries->nelts; ++i) 390289177Speter { 391289177Speter svn_fs_fs__p2l_entry_t *entry 392289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t); 393289177Speter 394289177Speter /* check all sub-items for consist entries in the L2P index */ 395289177Speter if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED) 396289177Speter { 397289177Speter /* There is no L2P entry for unused rev file sections. 398289177Speter * And its P2L index data is hardly ever used. But we 399289177Speter * should still check whether someone tempered with it. */ 400289177Speter if ( entry->item.revision != SVN_INVALID_REVNUM 401289177Speter && ( entry->item.revision < start 402289177Speter || entry->item.revision >= start + count)) 403289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 404289177Speter NULL, 405289177Speter _("Empty P2L entry for PHYS %s " 406289177Speter "refers to revision %ld outside " 407289177Speter "the rev / pack file (%ld-%ld)"), 408289177Speter apr_off_t_toa(pool, entry->offset), 409289177Speter entry->item.revision, 410289177Speter start, start + count - 1); 411289177Speter } 412289177Speter else 413289177Speter { 414289177Speter apr_off_t l2p_offset; 415289177Speter SVN_ERR(svn_fs_fs__item_offset(&l2p_offset, fs, rev_file, 416289177Speter entry->item.revision, NULL, 417289177Speter entry->item.number, iterpool)); 418289177Speter 419289177Speter if (l2p_offset != entry->offset) 420289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 421289177Speter NULL, 422289177Speter _("l2p index entry PHYS %s" 423289177Speter "does not match p2l index value " 424289177Speter "LOG r%ld:i%ld for PHYS %s"), 425289177Speter apr_off_t_toa(pool, l2p_offset), 426289177Speter entry->item.revision, 427289177Speter (long)entry->item.number, 428289177Speter apr_off_t_toa(pool, entry->offset)); 429289177Speter } 430289177Speter } 431289177Speter 432289177Speter if (cancel_func) 433289177Speter SVN_ERR(cancel_func(cancel_baton)); 434289177Speter } 435289177Speter 436289177Speter svn_pool_destroy(iterpool); 437289177Speter 438289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 439289177Speter 440289177Speter return SVN_NO_ERROR; 441289177Speter} 442289177Speter 443289177Speter/* Items smaller than this can be read at once into a buffer and directly 444289177Speter * be checksummed. Larger items require stream processing. 445289177Speter * Must be a multiple of 8. */ 446289177Speter#define STREAM_THRESHOLD 4096 447289177Speter 448289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. 449289177Speter * SIZE must not exceed STREAM_THRESHOLD. Use POOL for allocations. 450289177Speter */ 451289177Speterstatic svn_error_t * 452289177Speterexpect_buffer_nul(apr_file_t *file, 453289177Speter apr_off_t size, 454289177Speter apr_pool_t *pool) 455289177Speter{ 456289177Speter union 457289177Speter { 458289177Speter unsigned char buffer[STREAM_THRESHOLD]; 459289177Speter apr_uint64_t chunks[STREAM_THRESHOLD / sizeof(apr_uint64_t)]; 460289177Speter } data; 461289177Speter 462289177Speter apr_size_t i; 463289177Speter SVN_ERR_ASSERT(size <= STREAM_THRESHOLD); 464289177Speter 465289177Speter /* read the whole data block; error out on failure */ 466289177Speter data.chunks[(size - 1)/ sizeof(apr_uint64_t)] = 0; 467362181Sdim SVN_ERR(svn_io_file_read_full2(file, data.buffer, (apr_size_t)size, NULL, 468362181Sdim NULL, pool)); 469289177Speter 470289177Speter /* chunky check */ 471289177Speter for (i = 0; i < size / sizeof(apr_uint64_t); ++i) 472289177Speter if (data.chunks[i] != 0) 473289177Speter break; 474289177Speter 475289177Speter /* byte-wise check upon mismatch or at the end of the block */ 476289177Speter for (i *= sizeof(apr_uint64_t); i < size; ++i) 477289177Speter if (data.buffer[i] != 0) 478289177Speter { 479289177Speter const char *file_name; 480289177Speter apr_off_t offset; 481289177Speter 482289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, pool)); 483362181Sdim SVN_ERR(svn_io_file_get_offset(&offset, file, pool)); 484289177Speter offset -= size - i; 485289177Speter 486289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 487289177Speter _("Empty section in file %s contains " 488289177Speter "non-NUL data at offset %s"), 489289177Speter file_name, apr_off_t_toa(pool, offset)); 490289177Speter } 491289177Speter 492289177Speter return SVN_NO_ERROR; 493289177Speter} 494289177Speter 495289177Speter/* Verify that the next SIZE bytes read from FILE are NUL. 496289177Speter * Use POOL for allocations. 497289177Speter */ 498289177Speterstatic svn_error_t * 499289177Speterread_all_nul(apr_file_t *file, 500289177Speter apr_off_t size, 501289177Speter apr_pool_t *pool) 502289177Speter{ 503289177Speter for (; size >= STREAM_THRESHOLD; size -= STREAM_THRESHOLD) 504289177Speter SVN_ERR(expect_buffer_nul(file, STREAM_THRESHOLD, pool)); 505289177Speter 506289177Speter if (size) 507289177Speter SVN_ERR(expect_buffer_nul(file, size, pool)); 508289177Speter 509289177Speter return SVN_NO_ERROR; 510289177Speter} 511289177Speter 512289177Speter/* Compare the ACTUAL checksum with the one expected by ENTRY. 513289177Speter * Return an error in case of mismatch. Use the name of FILE 514289177Speter * in error message. Allocate data in POOL. 515289177Speter */ 516289177Speterstatic svn_error_t * 517289177Speterexpected_checksum(apr_file_t *file, 518289177Speter svn_fs_fs__p2l_entry_t *entry, 519289177Speter apr_uint32_t actual, 520289177Speter apr_pool_t *pool) 521289177Speter{ 522289177Speter if (actual != entry->fnv1_checksum) 523289177Speter { 524289177Speter const char *file_name; 525289177Speter 526289177Speter SVN_ERR(svn_io_file_name_get(&file_name, file, pool)); 527289177Speter return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 528289177Speter _("Checksum mismatch in item at offset %s of " 529289177Speter "length %s bytes in file %s"), 530289177Speter apr_off_t_toa(pool, entry->offset), 531289177Speter apr_off_t_toa(pool, entry->size), file_name); 532289177Speter } 533289177Speter 534289177Speter return SVN_NO_ERROR; 535289177Speter} 536289177Speter 537289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read 538289177Speter * from FILE will match ENTRY's expected checksum. SIZE must not 539289177Speter * exceed STREAM_THRESHOLD. Use POOL for allocations. 540289177Speter */ 541289177Speterstatic svn_error_t * 542289177Speterexpected_buffered_checksum(apr_file_t *file, 543289177Speter svn_fs_fs__p2l_entry_t *entry, 544289177Speter apr_pool_t *pool) 545289177Speter{ 546289177Speter unsigned char buffer[STREAM_THRESHOLD]; 547289177Speter SVN_ERR_ASSERT(entry->size <= STREAM_THRESHOLD); 548289177Speter 549289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, (apr_size_t)entry->size, 550289177Speter NULL, NULL, pool)); 551289177Speter SVN_ERR(expected_checksum(file, entry, 552289177Speter svn__fnv1a_32x4(buffer, (apr_size_t)entry->size), 553289177Speter pool)); 554289177Speter 555289177Speter return SVN_NO_ERROR; 556289177Speter} 557289177Speter 558289177Speter/* Verify that the FNV checksum over the next ENTRY->SIZE bytes read from 559289177Speter * FILE will match ENTRY's expected checksum. Use POOL for allocations. 560289177Speter */ 561289177Speterstatic svn_error_t * 562289177Speterexpected_streamed_checksum(apr_file_t *file, 563289177Speter svn_fs_fs__p2l_entry_t *entry, 564289177Speter apr_pool_t *pool) 565289177Speter{ 566289177Speter unsigned char buffer[STREAM_THRESHOLD]; 567289177Speter svn_checksum_t *checksum; 568289177Speter svn_checksum_ctx_t *context 569289177Speter = svn_checksum_ctx_create(svn_checksum_fnv1a_32x4, pool); 570289177Speter apr_off_t size = entry->size; 571289177Speter 572289177Speter while (size > 0) 573289177Speter { 574289177Speter apr_size_t to_read = size > sizeof(buffer) 575289177Speter ? sizeof(buffer) 576289177Speter : (apr_size_t)size; 577289177Speter SVN_ERR(svn_io_file_read_full2(file, buffer, to_read, NULL, NULL, 578289177Speter pool)); 579289177Speter SVN_ERR(svn_checksum_update(context, buffer, to_read)); 580289177Speter size -= to_read; 581289177Speter } 582289177Speter 583289177Speter SVN_ERR(svn_checksum_final(&checksum, context, pool)); 584289177Speter SVN_ERR(expected_checksum(file, entry, 585289177Speter ntohl(*(const apr_uint32_t *)checksum->digest), 586289177Speter pool)); 587289177Speter 588289177Speter return SVN_NO_ERROR; 589289177Speter} 590289177Speter 591289177Speter/* Verify that for all phys-to-log index entries for revisions START to 592289177Speter * START + COUNT-1 in FS match the actual pack / rev file contents. 593289177Speter * If given, invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 594289177Speter * Use POOL for allocations. 595289177Speter * 596289177Speter * Please note that we can only check on pack / rev file granularity and 597289177Speter * must only be called for a single rev / pack file. 598289177Speter */ 599289177Speterstatic svn_error_t * 600289177Spetercompare_p2l_to_rev(svn_fs_t *fs, 601289177Speter svn_revnum_t start, 602289177Speter svn_revnum_t count, 603289177Speter svn_cancel_func_t cancel_func, 604289177Speter void *cancel_baton, 605289177Speter apr_pool_t *pool) 606289177Speter{ 607289177Speter fs_fs_data_t *ffd = fs->fsap_data; 608289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 609289177Speter apr_off_t max_offset; 610289177Speter apr_off_t offset = 0; 611289177Speter svn_fs_fs__revision_file_t *rev_file; 612289177Speter 613289177Speter /* open the pack / rev file that is covered by the p2l index */ 614289177Speter SVN_ERR(svn_fs_fs__open_pack_or_rev_file(&rev_file, fs, start, pool, 615289177Speter iterpool)); 616289177Speter 617289177Speter /* check file size vs. range covered by index */ 618289177Speter SVN_ERR(svn_fs_fs__auto_read_footer(rev_file)); 619289177Speter SVN_ERR(svn_fs_fs__p2l_get_max_offset(&max_offset, fs, rev_file, start, 620289177Speter pool)); 621289177Speter 622289177Speter if (rev_file->l2p_offset != max_offset) 623289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, NULL, 624289177Speter _("File size of %s for revision r%ld does " 625289177Speter "not match p2l index size of %s"), 626289177Speter apr_off_t_toa(pool, rev_file->l2p_offset), start, 627289177Speter apr_off_t_toa(pool, max_offset)); 628289177Speter 629289177Speter SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, NULL, 0, 630289177Speter pool)); 631289177Speter 632289177Speter /* for all offsets in the file, get the P2L index entries and check 633289177Speter them against the L2P index */ 634289177Speter for (offset = 0; offset < max_offset; ) 635289177Speter { 636289177Speter apr_array_header_t *entries; 637289177Speter int i; 638289177Speter 639289177Speter svn_pool_clear(iterpool); 640289177Speter 641289177Speter /* get all entries for the current block */ 642289177Speter SVN_ERR(svn_fs_fs__p2l_index_lookup(&entries, fs, rev_file, start, 643289177Speter offset, ffd->p2l_page_size, 644289177Speter iterpool, iterpool)); 645289177Speter 646289177Speter /* The above might have moved the file pointer. 647289177Speter * Ensure we actually start reading at OFFSET. */ 648289177Speter SVN_ERR(svn_io_file_aligned_seek(rev_file->file, ffd->block_size, 649289177Speter NULL, offset, iterpool)); 650289177Speter 651289177Speter /* process all entries (and later continue with the next block) */ 652289177Speter for (i = 0; i < entries->nelts; ++i) 653289177Speter { 654289177Speter svn_fs_fs__p2l_entry_t *entry 655289177Speter = &APR_ARRAY_IDX(entries, i, svn_fs_fs__p2l_entry_t); 656289177Speter 657289177Speter /* skip bits we previously checked */ 658289177Speter if (i == 0 && entry->offset < offset) 659289177Speter continue; 660289177Speter 661289177Speter /* skip zero-sized entries */ 662289177Speter if (entry->size == 0) 663289177Speter continue; 664289177Speter 665289177Speter /* p2l index must cover all rev / pack file offsets exactly once */ 666289177Speter if (entry->offset != offset) 667289177Speter return svn_error_createf(SVN_ERR_FS_INDEX_INCONSISTENT, 668289177Speter NULL, 669289177Speter _("p2l index entry for revision r%ld" 670289177Speter " is non-contiguous between offsets " 671289177Speter " %s and %s"), 672289177Speter start, 673289177Speter apr_off_t_toa(pool, offset), 674289177Speter apr_off_t_toa(pool, entry->offset)); 675289177Speter 676362181Sdim /* Check type <-> item dependencies. */ 677362181Sdim 678362181Sdim /* Entry types must be within the valid range. */ 679362181Sdim if (entry->type >= SVN_FS_FS__ITEM_TYPE_ANY_REP) 680362181Sdim return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 681362181Sdim NULL, 682362181Sdim _("p2l index entry for revision r%ld" 683362181Sdim " at offset %s contains invalid item" 684362181Sdim " type %u"), 685362181Sdim start, 686362181Sdim apr_off_t_toa(pool, offset), 687362181Sdim (unsigned int)entry->type); 688362181Sdim 689362181Sdim /* There can be only one changes entry and that has a fixed type 690362181Sdim * and item number. Its presence and parse-ability will be checked 691362181Sdim * during later stages of the verification process. */ 692362181Sdim if ( (entry->type == SVN_FS_FS__ITEM_TYPE_CHANGES) 693362181Sdim != (entry->item.number == SVN_FS_FS__ITEM_INDEX_CHANGES)) 694362181Sdim return svn_error_createf(SVN_ERR_FS_INDEX_CORRUPTION, 695362181Sdim NULL, 696362181Sdim _("p2l index entry for changes in" 697362181Sdim " revision r%ld is item" 698362181Sdim " %"APR_UINT64_T_FMT 699362181Sdim " of type %u at offset %s"), 700362181Sdim entry->item.revision, 701362181Sdim entry->item.number, 702362181Sdim (unsigned int)entry->type, 703362181Sdim apr_off_t_toa(pool, offset)); 704362181Sdim 705362181Sdim /* Check contents. */ 706289177Speter if (entry->type == SVN_FS_FS__ITEM_TYPE_UNUSED) 707289177Speter { 708362181Sdim /* Empty sections must contain NUL bytes only. 709362181Sdim * Beware of the filler at the end of the p2l index. */ 710289177Speter if (entry->offset != max_offset) 711289177Speter SVN_ERR(read_all_nul(rev_file->file, entry->size, pool)); 712289177Speter } 713289177Speter else 714289177Speter { 715362181Sdim /* Generic contents check against checksum. */ 716289177Speter if (entry->size < STREAM_THRESHOLD) 717289177Speter SVN_ERR(expected_buffered_checksum(rev_file->file, entry, 718289177Speter pool)); 719289177Speter else 720289177Speter SVN_ERR(expected_streamed_checksum(rev_file->file, entry, 721289177Speter pool)); 722289177Speter } 723289177Speter 724289177Speter /* advance offset */ 725289177Speter offset += entry->size; 726289177Speter } 727289177Speter 728289177Speter if (cancel_func) 729289177Speter SVN_ERR(cancel_func(cancel_baton)); 730289177Speter } 731289177Speter 732289177Speter svn_pool_destroy(iterpool); 733289177Speter 734289177Speter SVN_ERR(svn_fs_fs__close_revision_file(rev_file)); 735289177Speter 736289177Speter return SVN_NO_ERROR; 737289177Speter} 738289177Speter 739289177Speter/* Verify that the revprops of the revisions START to END in FS can be 740289177Speter * accessed. Invoke CANCEL_FUNC with CANCEL_BATON at regular intervals. 741289177Speter * 742289177Speter * The values of START and END have already been auto-selected and 743289177Speter * verified. 744289177Speter */ 745289177Speterstatic svn_error_t * 746289177Speterverify_revprops(svn_fs_t *fs, 747289177Speter svn_revnum_t start, 748289177Speter svn_revnum_t end, 749289177Speter svn_cancel_func_t cancel_func, 750289177Speter void *cancel_baton, 751289177Speter apr_pool_t *pool) 752289177Speter{ 753289177Speter svn_revnum_t revision; 754289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 755289177Speter 756362181Sdim /* Invalidate the revprop cache once. 757362181Sdim * Use the cache inside the loop to speed up packed revprop access. */ 758362181Sdim svn_fs_fs__reset_revprop_cache(fs); 759362181Sdim 760289177Speter for (revision = start; revision < end; ++revision) 761289177Speter { 762289177Speter svn_string_t *date; 763289177Speter apr_time_t timetemp; 764289177Speter 765289177Speter svn_pool_clear(iterpool); 766289177Speter 767289177Speter /* Access the svn:date revprop. 768289177Speter * This implies parsing all revprops for that revision. */ 769289177Speter SVN_ERR(svn_fs_fs__revision_prop(&date, fs, revision, 770362181Sdim SVN_PROP_REVISION_DATE, FALSE, 771362181Sdim iterpool, iterpool)); 772289177Speter 773289177Speter /* The time stamp is the only revprop that, if given, needs to 774289177Speter * have a valid content. */ 775289177Speter if (date) 776289177Speter SVN_ERR(svn_time_from_cstring(&timetemp, date->data, iterpool)); 777289177Speter 778289177Speter if (cancel_func) 779289177Speter SVN_ERR(cancel_func(cancel_baton)); 780289177Speter } 781289177Speter 782289177Speter svn_pool_destroy(iterpool); 783289177Speter 784289177Speter return SVN_NO_ERROR; 785289177Speter} 786289177Speter 787289177Speterstatic svn_revnum_t 788289177Speterpack_size(svn_fs_t *fs, svn_revnum_t rev) 789289177Speter{ 790289177Speter fs_fs_data_t *ffd = fs->fsap_data; 791289177Speter 792289177Speter return rev < ffd->min_unpacked_rev ? ffd->max_files_per_dir : 1; 793289177Speter} 794289177Speter 795289177Speter/* Verify that on-disk representation has not been tempered with (in a way 796289177Speter * that leaves the repository in a corrupted state). This compares log-to- 797289177Speter * phys with phys-to-log indexes, verifies the low-level checksums and 798289177Speter * checks that all revprops are available. The function signature is 799289177Speter * similar to svn_fs_fs__verify. 800289177Speter * 801289177Speter * The values of START and END have already been auto-selected and 802289177Speter * verified. You may call this for format7 or higher repos. 803289177Speter */ 804289177Speterstatic svn_error_t * 805289177Speterverify_f7_metadata_consistency(svn_fs_t *fs, 806289177Speter svn_revnum_t start, 807289177Speter svn_revnum_t end, 808289177Speter svn_fs_progress_notify_func_t notify_func, 809289177Speter void *notify_baton, 810289177Speter svn_cancel_func_t cancel_func, 811289177Speter void *cancel_baton, 812289177Speter apr_pool_t *pool) 813289177Speter{ 814289177Speter fs_fs_data_t *ffd = fs->fsap_data; 815289177Speter svn_revnum_t revision, next_revision; 816289177Speter apr_pool_t *iterpool = svn_pool_create(pool); 817289177Speter 818289177Speter for (revision = start; revision <= end; revision = next_revision) 819289177Speter { 820289177Speter svn_error_t *err = SVN_NO_ERROR; 821289177Speter 822289177Speter svn_revnum_t count = pack_size(fs, revision); 823289177Speter svn_revnum_t pack_start = svn_fs_fs__packed_base_rev(fs, revision); 824289177Speter svn_revnum_t pack_end = pack_start + count; 825289177Speter 826289177Speter svn_pool_clear(iterpool); 827289177Speter 828289177Speter if (notify_func && (pack_start % ffd->max_files_per_dir == 0)) 829289177Speter notify_func(pack_start, notify_baton, iterpool); 830289177Speter 831289177Speter /* Check for external corruption to the indexes. */ 832289177Speter err = verify_index_checksums(fs, pack_start, cancel_func, 833289177Speter cancel_baton, iterpool); 834289177Speter 835289177Speter /* two-way index check */ 836289177Speter if (!err) 837289177Speter err = compare_l2p_to_p2l_index(fs, pack_start, pack_end - pack_start, 838289177Speter cancel_func, cancel_baton, iterpool); 839289177Speter if (!err) 840289177Speter err = compare_p2l_to_l2p_index(fs, pack_start, pack_end - pack_start, 841289177Speter cancel_func, cancel_baton, iterpool); 842289177Speter 843289177Speter /* verify in-index checksums and types vs. actual rev / pack files */ 844289177Speter if (!err) 845289177Speter err = compare_p2l_to_rev(fs, pack_start, pack_end - pack_start, 846289177Speter cancel_func, cancel_baton, iterpool); 847289177Speter 848289177Speter /* ensure that revprops are available and accessible */ 849289177Speter if (!err) 850289177Speter err = verify_revprops(fs, pack_start, pack_end, 851289177Speter cancel_func, cancel_baton, iterpool); 852289177Speter 853289177Speter /* concurrent packing is one of the reasons why verification may fail. 854289177Speter Make sure, we operate on up-to-date information. */ 855289177Speter if (err) 856289177Speter { 857289177Speter svn_error_t *err2 858289177Speter = svn_fs_fs__read_min_unpacked_rev(&ffd->min_unpacked_rev, 859289177Speter fs, pool); 860289177Speter 861289177Speter /* Be careful to not leak ERR. */ 862289177Speter if (err2) 863289177Speter return svn_error_trace(svn_error_compose_create(err, err2)); 864289177Speter } 865289177Speter 866289177Speter /* retry the whole shard if it got packed in the meantime */ 867289177Speter if (err && count != pack_size(fs, revision)) 868289177Speter { 869289177Speter svn_error_clear(err); 870289177Speter 871289177Speter /* We could simply assign revision here but the code below is 872289177Speter more intuitive to maintainers. */ 873289177Speter next_revision = svn_fs_fs__packed_base_rev(fs, revision); 874289177Speter } 875289177Speter else 876289177Speter { 877289177Speter SVN_ERR(err); 878289177Speter next_revision = pack_end; 879289177Speter } 880289177Speter } 881289177Speter 882289177Speter svn_pool_destroy(iterpool); 883289177Speter 884289177Speter return SVN_NO_ERROR; 885289177Speter} 886289177Speter 887289177Spetersvn_error_t * 888289177Spetersvn_fs_fs__verify(svn_fs_t *fs, 889289177Speter svn_revnum_t start, 890289177Speter svn_revnum_t end, 891289177Speter svn_fs_progress_notify_func_t notify_func, 892289177Speter void *notify_baton, 893289177Speter svn_cancel_func_t cancel_func, 894289177Speter void *cancel_baton, 895289177Speter apr_pool_t *pool) 896289177Speter{ 897289177Speter fs_fs_data_t *ffd = fs->fsap_data; 898289177Speter 899289177Speter /* Input validation. */ 900289177Speter if (! SVN_IS_VALID_REVNUM(start)) 901289177Speter start = 0; 902289177Speter if (! SVN_IS_VALID_REVNUM(end)) 903362181Sdim { 904362181Sdim SVN_ERR(svn_fs_fs__youngest_rev(&end, fs, pool)); 905362181Sdim } 906362181Sdim 907289177Speter SVN_ERR(svn_fs_fs__ensure_revision_exists(start, fs, pool)); 908289177Speter SVN_ERR(svn_fs_fs__ensure_revision_exists(end, fs, pool)); 909289177Speter 910289177Speter /* log/phys index consistency. We need to check them first to make 911289177Speter sure we can access the rev / pack files in format7. */ 912289177Speter if (svn_fs_fs__use_log_addressing(fs)) 913289177Speter SVN_ERR(verify_f7_metadata_consistency(fs, start, end, 914289177Speter notify_func, notify_baton, 915289177Speter cancel_func, cancel_baton, pool)); 916289177Speter 917289177Speter /* rep cache consistency */ 918289177Speter if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) 919289177Speter SVN_ERR(verify_rep_cache(fs, start, end, notify_func, notify_baton, 920289177Speter cancel_func, cancel_baton, pool)); 921289177Speter 922289177Speter return SVN_NO_ERROR; 923289177Speter} 924