1/* fs_fs.c --- filesystem operations specific to fs_fs 2 * 3 * ==================================================================== 4 * Licensed to the Apache Software Foundation (ASF) under one 5 * or more contributor license agreements. See the NOTICE file 6 * distributed with this work for additional information 7 * regarding copyright ownership. The ASF licenses this file 8 * to you under the Apache License, Version 2.0 (the 9 * "License"); you may not use this file except in compliance 10 * with the License. You may obtain a copy of the License at 11 * 12 * http://www.apache.org/licenses/LICENSE-2.0 13 * 14 * Unless required by applicable law or agreed to in writing, 15 * software distributed under the License is distributed on an 16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 * KIND, either express or implied. See the License for the 18 * specific language governing permissions and limitations 19 * under the License. 20 * ==================================================================== 21 */ 22 23#include <stdlib.h> 24#include <stdio.h> 25#include <string.h> 26#include <ctype.h> 27#include <assert.h> 28#include <errno.h> 29 30#include <apr_general.h> 31#include <apr_pools.h> 32#include <apr_file_io.h> 33#include <apr_uuid.h> 34#include <apr_lib.h> 35#include <apr_md5.h> 36#include <apr_sha1.h> 37#include <apr_strings.h> 38#include <apr_thread_mutex.h> 39 40#include "svn_pools.h" 41#include "svn_fs.h" 42#include "svn_dirent_uri.h" 43#include "svn_path.h" 44#include "svn_hash.h" 45#include "svn_props.h" 46#include "svn_sorts.h" 47#include "svn_string.h" 48#include "svn_time.h" 49#include "svn_mergeinfo.h" 50#include "svn_config.h" 51#include "svn_ctype.h" 52#include "svn_version.h" 53 54#include "fs.h" 55#include "tree.h" 56#include "lock.h" 57#include "key-gen.h" 58#include "fs_fs.h" 59#include "id.h" 60#include "rep-cache.h" 61#include "temp_serializer.h" 62 63#include "private/svn_string_private.h" 64#include "private/svn_fs_util.h" 65#include "private/svn_subr_private.h" 66#include "private/svn_delta_private.h" 67#include "../libsvn_fs/fs-loader.h" 68 69#include "svn_private_config.h" 70#include "temp_serializer.h" 71 72/* An arbitrary maximum path length, so clients can't run us out of memory 73 * by giving us arbitrarily large paths. */ 74#define FSFS_MAX_PATH_LEN 4096 75 76/* The default maximum number of files per directory to store in the 77 rev and revprops directory. The number below is somewhat arbitrary, 78 and can be overridden by defining the macro while compiling; the 79 figure of 1000 is reasonable for VFAT filesystems, which are by far 80 the worst performers in this area. */ 81#ifndef SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR 82#define SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR 1000 83#endif 84 85/* Begin deltification after a node history exceeded this this limit. 86 Useful values are 4 to 64 with 16 being a good compromise between 87 computational overhead and repository size savings. 88 Should be a power of 2. 89 Values < 2 will result in standard skip-delta behavior. */ 90#define SVN_FS_FS_MAX_LINEAR_DELTIFICATION 16 91 92/* Finding a deltification base takes operations proportional to the 93 number of changes being skipped. To prevent exploding runtime 94 during commits, limit the deltification range to this value. 95 Should be a power of 2 minus one. 96 Values < 1 disable deltification. */ 97#define SVN_FS_FS_MAX_DELTIFICATION_WALK 1023 98 99/* Give writing processes 10 seconds to replace an existing revprop 100 file with a new one. After that time, we assume that the writing 101 process got aborted and that we have re-read revprops. */ 102#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) 103 104/* The following are names of atomics that will be used to communicate 105 * revprop updates across all processes on this machine. */ 106#define ATOMIC_REVPROP_GENERATION "rev-prop-generation" 107#define ATOMIC_REVPROP_TIMEOUT "rev-prop-timeout" 108#define ATOMIC_REVPROP_NAMESPACE "rev-prop-atomics" 109 110/* Following are defines that specify the textual elements of the 111 native filesystem directories and revision files. */ 112 113/* Headers used to describe node-revision in the revision file. */ 114#define HEADER_ID "id" 115#define HEADER_TYPE "type" 116#define HEADER_COUNT "count" 117#define HEADER_PROPS "props" 118#define HEADER_TEXT "text" 119#define HEADER_CPATH "cpath" 120#define HEADER_PRED "pred" 121#define HEADER_COPYFROM "copyfrom" 122#define HEADER_COPYROOT "copyroot" 123#define HEADER_FRESHTXNRT "is-fresh-txn-root" 124#define HEADER_MINFO_HERE "minfo-here" 125#define HEADER_MINFO_CNT "minfo-cnt" 126 127/* Kinds that a change can be. */ 128#define ACTION_MODIFY "modify" 129#define ACTION_ADD "add" 130#define ACTION_DELETE "delete" 131#define ACTION_REPLACE "replace" 132#define ACTION_RESET "reset" 133 134/* True and False flags. */ 135#define FLAG_TRUE "true" 136#define FLAG_FALSE "false" 137 138/* Kinds that a node-rev can be. */ 139#define KIND_FILE "file" 140#define KIND_DIR "dir" 141 142/* Kinds of representation. */ 143#define REP_PLAIN "PLAIN" 144#define REP_DELTA "DELTA" 145 146/* Notes: 147 148To avoid opening and closing the rev-files all the time, it would 149probably be advantageous to keep each rev-file open for the 150lifetime of the transaction object. I'll leave that as a later 151optimization for now. 152 153I didn't keep track of pool lifetimes at all in this code. There 154are likely some errors because of that. 155 156*/ 157 158/* The vtable associated with an open transaction object. */ 159static txn_vtable_t txn_vtable = { 160 svn_fs_fs__commit_txn, 161 svn_fs_fs__abort_txn, 162 svn_fs_fs__txn_prop, 163 svn_fs_fs__txn_proplist, 164 svn_fs_fs__change_txn_prop, 165 svn_fs_fs__txn_root, 166 svn_fs_fs__change_txn_props 167}; 168 169/* Declarations. */ 170 171static svn_error_t * 172read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev, 173 const char *path, 174 apr_pool_t *pool); 175 176static svn_error_t * 177update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool); 178 179static svn_error_t * 180get_youngest(svn_revnum_t *youngest_p, const char *fs_path, apr_pool_t *pool); 181 182static svn_error_t * 183verify_walker(representation_t *rep, 184 void *baton, 185 svn_fs_t *fs, 186 apr_pool_t *scratch_pool); 187 188/* Pathname helper functions */ 189 190/* Return TRUE is REV is packed in FS, FALSE otherwise. */ 191static svn_boolean_t 192is_packed_rev(svn_fs_t *fs, svn_revnum_t rev) 193{ 194 fs_fs_data_t *ffd = fs->fsap_data; 195 196 return (rev < ffd->min_unpacked_rev); 197} 198 199/* Return TRUE is REV is packed in FS, FALSE otherwise. */ 200static svn_boolean_t 201is_packed_revprop(svn_fs_t *fs, svn_revnum_t rev) 202{ 203 fs_fs_data_t *ffd = fs->fsap_data; 204 205 /* rev 0 will not be packed */ 206 return (rev < ffd->min_unpacked_rev) 207 && (rev != 0) 208 && (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT); 209} 210 211static const char * 212path_format(svn_fs_t *fs, apr_pool_t *pool) 213{ 214 return svn_dirent_join(fs->path, PATH_FORMAT, pool); 215} 216 217static APR_INLINE const char * 218path_uuid(svn_fs_t *fs, apr_pool_t *pool) 219{ 220 return svn_dirent_join(fs->path, PATH_UUID, pool); 221} 222 223const char * 224svn_fs_fs__path_current(svn_fs_t *fs, apr_pool_t *pool) 225{ 226 return svn_dirent_join(fs->path, PATH_CURRENT, pool); 227} 228 229static APR_INLINE const char * 230path_txn_current(svn_fs_t *fs, apr_pool_t *pool) 231{ 232 return svn_dirent_join(fs->path, PATH_TXN_CURRENT, pool); 233} 234 235static APR_INLINE const char * 236path_txn_current_lock(svn_fs_t *fs, apr_pool_t *pool) 237{ 238 return svn_dirent_join(fs->path, PATH_TXN_CURRENT_LOCK, pool); 239} 240 241static APR_INLINE const char * 242path_lock(svn_fs_t *fs, apr_pool_t *pool) 243{ 244 return svn_dirent_join(fs->path, PATH_LOCK_FILE, pool); 245} 246 247static const char * 248path_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) 249{ 250 return svn_dirent_join(fs->path, PATH_REVPROP_GENERATION, pool); 251} 252 253static const char * 254path_rev_packed(svn_fs_t *fs, svn_revnum_t rev, const char *kind, 255 apr_pool_t *pool) 256{ 257 fs_fs_data_t *ffd = fs->fsap_data; 258 259 assert(ffd->max_files_per_dir); 260 assert(is_packed_rev(fs, rev)); 261 262 return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR, 263 apr_psprintf(pool, 264 "%ld" PATH_EXT_PACKED_SHARD, 265 rev / ffd->max_files_per_dir), 266 kind, NULL); 267} 268 269static const char * 270path_rev_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) 271{ 272 fs_fs_data_t *ffd = fs->fsap_data; 273 274 assert(ffd->max_files_per_dir); 275 return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR, 276 apr_psprintf(pool, "%ld", 277 rev / ffd->max_files_per_dir), 278 NULL); 279} 280 281static const char * 282path_rev(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) 283{ 284 fs_fs_data_t *ffd = fs->fsap_data; 285 286 assert(! is_packed_rev(fs, rev)); 287 288 if (ffd->max_files_per_dir) 289 { 290 return svn_dirent_join(path_rev_shard(fs, rev, pool), 291 apr_psprintf(pool, "%ld", rev), 292 pool); 293 } 294 295 return svn_dirent_join_many(pool, fs->path, PATH_REVS_DIR, 296 apr_psprintf(pool, "%ld", rev), NULL); 297} 298 299svn_error_t * 300svn_fs_fs__path_rev_absolute(const char **path, 301 svn_fs_t *fs, 302 svn_revnum_t rev, 303 apr_pool_t *pool) 304{ 305 fs_fs_data_t *ffd = fs->fsap_data; 306 307 if (ffd->format < SVN_FS_FS__MIN_PACKED_FORMAT 308 || ! is_packed_rev(fs, rev)) 309 { 310 *path = path_rev(fs, rev, pool); 311 } 312 else 313 { 314 *path = path_rev_packed(fs, rev, PATH_PACKED, pool); 315 } 316 317 return SVN_NO_ERROR; 318} 319 320static const char * 321path_revprops_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) 322{ 323 fs_fs_data_t *ffd = fs->fsap_data; 324 325 assert(ffd->max_files_per_dir); 326 return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR, 327 apr_psprintf(pool, "%ld", 328 rev / ffd->max_files_per_dir), 329 NULL); 330} 331 332static const char * 333path_revprops_pack_shard(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) 334{ 335 fs_fs_data_t *ffd = fs->fsap_data; 336 337 assert(ffd->max_files_per_dir); 338 return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR, 339 apr_psprintf(pool, "%ld" PATH_EXT_PACKED_SHARD, 340 rev / ffd->max_files_per_dir), 341 NULL); 342} 343 344static const char * 345path_revprops(svn_fs_t *fs, svn_revnum_t rev, apr_pool_t *pool) 346{ 347 fs_fs_data_t *ffd = fs->fsap_data; 348 349 if (ffd->max_files_per_dir) 350 { 351 return svn_dirent_join(path_revprops_shard(fs, rev, pool), 352 apr_psprintf(pool, "%ld", rev), 353 pool); 354 } 355 356 return svn_dirent_join_many(pool, fs->path, PATH_REVPROPS_DIR, 357 apr_psprintf(pool, "%ld", rev), NULL); 358} 359 360static APR_INLINE const char * 361path_txn_dir(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) 362{ 363 SVN_ERR_ASSERT_NO_RETURN(txn_id != NULL); 364 return svn_dirent_join_many(pool, fs->path, PATH_TXNS_DIR, 365 apr_pstrcat(pool, txn_id, PATH_EXT_TXN, 366 (char *)NULL), 367 NULL); 368} 369 370/* Return the name of the sha1->rep mapping file in transaction TXN_ID 371 * within FS for the given SHA1 checksum. Use POOL for allocations. 372 */ 373static APR_INLINE const char * 374path_txn_sha1(svn_fs_t *fs, const char *txn_id, svn_checksum_t *sha1, 375 apr_pool_t *pool) 376{ 377 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), 378 svn_checksum_to_cstring(sha1, pool), 379 pool); 380} 381 382static APR_INLINE const char * 383path_txn_changes(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) 384{ 385 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_CHANGES, pool); 386} 387 388static APR_INLINE const char * 389path_txn_props(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) 390{ 391 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_TXN_PROPS, pool); 392} 393 394static APR_INLINE const char * 395path_txn_next_ids(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) 396{ 397 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_NEXT_IDS, pool); 398} 399 400static APR_INLINE const char * 401path_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool) 402{ 403 return svn_dirent_join(fs->path, PATH_MIN_UNPACKED_REV, pool); 404} 405 406 407static APR_INLINE const char * 408path_txn_proto_rev(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) 409{ 410 fs_fs_data_t *ffd = fs->fsap_data; 411 if (ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) 412 return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR, 413 apr_pstrcat(pool, txn_id, PATH_EXT_REV, 414 (char *)NULL), 415 NULL); 416 else 417 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_REV, pool); 418} 419 420static APR_INLINE const char * 421path_txn_proto_rev_lock(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) 422{ 423 fs_fs_data_t *ffd = fs->fsap_data; 424 if (ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) 425 return svn_dirent_join_many(pool, fs->path, PATH_TXN_PROTOS_DIR, 426 apr_pstrcat(pool, txn_id, PATH_EXT_REV_LOCK, 427 (char *)NULL), 428 NULL); 429 else 430 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), PATH_REV_LOCK, 431 pool); 432} 433 434static const char * 435path_txn_node_rev(svn_fs_t *fs, const svn_fs_id_t *id, apr_pool_t *pool) 436{ 437 const char *txn_id = svn_fs_fs__id_txn_id(id); 438 const char *node_id = svn_fs_fs__id_node_id(id); 439 const char *copy_id = svn_fs_fs__id_copy_id(id); 440 const char *name = apr_psprintf(pool, PATH_PREFIX_NODE "%s.%s", 441 node_id, copy_id); 442 443 return svn_dirent_join(path_txn_dir(fs, txn_id, pool), name, pool); 444} 445 446static APR_INLINE const char * 447path_txn_node_props(svn_fs_t *fs, const svn_fs_id_t *id, apr_pool_t *pool) 448{ 449 return apr_pstrcat(pool, path_txn_node_rev(fs, id, pool), PATH_EXT_PROPS, 450 (char *)NULL); 451} 452 453static APR_INLINE const char * 454path_txn_node_children(svn_fs_t *fs, const svn_fs_id_t *id, apr_pool_t *pool) 455{ 456 return apr_pstrcat(pool, path_txn_node_rev(fs, id, pool), 457 PATH_EXT_CHILDREN, (char *)NULL); 458} 459 460static APR_INLINE const char * 461path_node_origin(svn_fs_t *fs, const char *node_id, apr_pool_t *pool) 462{ 463 size_t len = strlen(node_id); 464 const char *node_id_minus_last_char = 465 (len == 1) ? "0" : apr_pstrmemdup(pool, node_id, len - 1); 466 return svn_dirent_join_many(pool, fs->path, PATH_NODE_ORIGINS_DIR, 467 node_id_minus_last_char, NULL); 468} 469 470static APR_INLINE const char * 471path_and_offset_of(apr_file_t *file, apr_pool_t *pool) 472{ 473 const char *path; 474 apr_off_t offset = 0; 475 476 if (apr_file_name_get(&path, file) != APR_SUCCESS) 477 path = "(unknown)"; 478 479 if (apr_file_seek(file, APR_CUR, &offset) != APR_SUCCESS) 480 offset = -1; 481 482 return apr_psprintf(pool, "%s:%" APR_OFF_T_FMT, path, offset); 483} 484 485 486 487/* Functions for working with shared transaction data. */ 488 489/* Return the transaction object for transaction TXN_ID from the 490 transaction list of filesystem FS (which must already be locked via the 491 txn_list_lock mutex). If the transaction does not exist in the list, 492 then create a new transaction object and return it (if CREATE_NEW is 493 true) or return NULL (otherwise). */ 494static fs_fs_shared_txn_data_t * 495get_shared_txn(svn_fs_t *fs, const char *txn_id, svn_boolean_t create_new) 496{ 497 fs_fs_data_t *ffd = fs->fsap_data; 498 fs_fs_shared_data_t *ffsd = ffd->shared; 499 fs_fs_shared_txn_data_t *txn; 500 501 for (txn = ffsd->txns; txn; txn = txn->next) 502 if (strcmp(txn->txn_id, txn_id) == 0) 503 break; 504 505 if (txn || !create_new) 506 return txn; 507 508 /* Use the transaction object from the (single-object) freelist, 509 if one is available, or otherwise create a new object. */ 510 if (ffsd->free_txn) 511 { 512 txn = ffsd->free_txn; 513 ffsd->free_txn = NULL; 514 } 515 else 516 { 517 apr_pool_t *subpool = svn_pool_create(ffsd->common_pool); 518 txn = apr_palloc(subpool, sizeof(*txn)); 519 txn->pool = subpool; 520 } 521 522 assert(strlen(txn_id) < sizeof(txn->txn_id)); 523 apr_cpystrn(txn->txn_id, txn_id, sizeof(txn->txn_id)); 524 txn->being_written = FALSE; 525 526 /* Link this transaction into the head of the list. We will typically 527 be dealing with only one active transaction at a time, so it makes 528 sense for searches through the transaction list to look at the 529 newest transactions first. */ 530 txn->next = ffsd->txns; 531 ffsd->txns = txn; 532 533 return txn; 534} 535 536/* Free the transaction object for transaction TXN_ID, and remove it 537 from the transaction list of filesystem FS (which must already be 538 locked via the txn_list_lock mutex). Do nothing if the transaction 539 does not exist. */ 540static void 541free_shared_txn(svn_fs_t *fs, const char *txn_id) 542{ 543 fs_fs_data_t *ffd = fs->fsap_data; 544 fs_fs_shared_data_t *ffsd = ffd->shared; 545 fs_fs_shared_txn_data_t *txn, *prev = NULL; 546 547 for (txn = ffsd->txns; txn; prev = txn, txn = txn->next) 548 if (strcmp(txn->txn_id, txn_id) == 0) 549 break; 550 551 if (!txn) 552 return; 553 554 if (prev) 555 prev->next = txn->next; 556 else 557 ffsd->txns = txn->next; 558 559 /* As we typically will be dealing with one transaction after another, 560 we will maintain a single-object free list so that we can hopefully 561 keep reusing the same transaction object. */ 562 if (!ffsd->free_txn) 563 ffsd->free_txn = txn; 564 else 565 svn_pool_destroy(txn->pool); 566} 567 568 569/* Obtain a lock on the transaction list of filesystem FS, call BODY 570 with FS, BATON, and POOL, and then unlock the transaction list. 571 Return what BODY returned. */ 572static svn_error_t * 573with_txnlist_lock(svn_fs_t *fs, 574 svn_error_t *(*body)(svn_fs_t *fs, 575 const void *baton, 576 apr_pool_t *pool), 577 const void *baton, 578 apr_pool_t *pool) 579{ 580 fs_fs_data_t *ffd = fs->fsap_data; 581 fs_fs_shared_data_t *ffsd = ffd->shared; 582 583 SVN_MUTEX__WITH_LOCK(ffsd->txn_list_lock, 584 body(fs, baton, pool)); 585 586 return SVN_NO_ERROR; 587} 588 589 590/* Get a lock on empty file LOCK_FILENAME, creating it in POOL. */ 591static svn_error_t * 592get_lock_on_filesystem(const char *lock_filename, 593 apr_pool_t *pool) 594{ 595 svn_error_t *err = svn_io_file_lock2(lock_filename, TRUE, FALSE, pool); 596 597 if (err && APR_STATUS_IS_ENOENT(err->apr_err)) 598 { 599 /* No lock file? No big deal; these are just empty files 600 anyway. Create it and try again. */ 601 svn_error_clear(err); 602 err = NULL; 603 604 SVN_ERR(svn_io_file_create(lock_filename, "", pool)); 605 SVN_ERR(svn_io_file_lock2(lock_filename, TRUE, FALSE, pool)); 606 } 607 608 return svn_error_trace(err); 609} 610 611/* Reset the HAS_WRITE_LOCK member in the FFD given as BATON_VOID. 612 When registered with the pool holding the lock on the lock file, 613 this makes sure the flag gets reset just before we release the lock. */ 614static apr_status_t 615reset_lock_flag(void *baton_void) 616{ 617 fs_fs_data_t *ffd = baton_void; 618 ffd->has_write_lock = FALSE; 619 return APR_SUCCESS; 620} 621 622/* Obtain a write lock on the file LOCK_FILENAME (protecting with 623 LOCK_MUTEX if APR is threaded) in a subpool of POOL, call BODY with 624 BATON and that subpool, destroy the subpool (releasing the write 625 lock) and return what BODY returned. If IS_GLOBAL_LOCK is set, 626 set the HAS_WRITE_LOCK flag while we keep the write lock. */ 627static svn_error_t * 628with_some_lock_file(svn_fs_t *fs, 629 svn_error_t *(*body)(void *baton, 630 apr_pool_t *pool), 631 void *baton, 632 const char *lock_filename, 633 svn_boolean_t is_global_lock, 634 apr_pool_t *pool) 635{ 636 apr_pool_t *subpool = svn_pool_create(pool); 637 svn_error_t *err = get_lock_on_filesystem(lock_filename, subpool); 638 639 if (!err) 640 { 641 fs_fs_data_t *ffd = fs->fsap_data; 642 643 if (is_global_lock) 644 { 645 /* set the "got the lock" flag and register reset function */ 646 apr_pool_cleanup_register(subpool, 647 ffd, 648 reset_lock_flag, 649 apr_pool_cleanup_null); 650 ffd->has_write_lock = TRUE; 651 } 652 653 /* nobody else will modify the repo state 654 => read HEAD & pack info once */ 655 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 656 SVN_ERR(update_min_unpacked_rev(fs, pool)); 657 SVN_ERR(get_youngest(&ffd->youngest_rev_cache, fs->path, 658 pool)); 659 err = body(baton, subpool); 660 } 661 662 svn_pool_destroy(subpool); 663 664 return svn_error_trace(err); 665} 666 667svn_error_t * 668svn_fs_fs__with_write_lock(svn_fs_t *fs, 669 svn_error_t *(*body)(void *baton, 670 apr_pool_t *pool), 671 void *baton, 672 apr_pool_t *pool) 673{ 674 fs_fs_data_t *ffd = fs->fsap_data; 675 fs_fs_shared_data_t *ffsd = ffd->shared; 676 677 SVN_MUTEX__WITH_LOCK(ffsd->fs_write_lock, 678 with_some_lock_file(fs, body, baton, 679 path_lock(fs, pool), 680 TRUE, 681 pool)); 682 683 return SVN_NO_ERROR; 684} 685 686/* Run BODY (with BATON and POOL) while the txn-current file 687 of FS is locked. */ 688static svn_error_t * 689with_txn_current_lock(svn_fs_t *fs, 690 svn_error_t *(*body)(void *baton, 691 apr_pool_t *pool), 692 void *baton, 693 apr_pool_t *pool) 694{ 695 fs_fs_data_t *ffd = fs->fsap_data; 696 fs_fs_shared_data_t *ffsd = ffd->shared; 697 698 SVN_MUTEX__WITH_LOCK(ffsd->txn_current_lock, 699 with_some_lock_file(fs, body, baton, 700 path_txn_current_lock(fs, pool), 701 FALSE, 702 pool)); 703 704 return SVN_NO_ERROR; 705} 706 707/* A structure used by unlock_proto_rev() and unlock_proto_rev_body(), 708 which see. */ 709struct unlock_proto_rev_baton 710{ 711 const char *txn_id; 712 void *lockcookie; 713}; 714 715/* Callback used in the implementation of unlock_proto_rev(). */ 716static svn_error_t * 717unlock_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) 718{ 719 const struct unlock_proto_rev_baton *b = baton; 720 const char *txn_id = b->txn_id; 721 apr_file_t *lockfile = b->lockcookie; 722 fs_fs_shared_txn_data_t *txn = get_shared_txn(fs, txn_id, FALSE); 723 apr_status_t apr_err; 724 725 if (!txn) 726 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 727 _("Can't unlock unknown transaction '%s'"), 728 txn_id); 729 if (!txn->being_written) 730 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 731 _("Can't unlock nonlocked transaction '%s'"), 732 txn_id); 733 734 apr_err = apr_file_unlock(lockfile); 735 if (apr_err) 736 return svn_error_wrap_apr 737 (apr_err, 738 _("Can't unlock prototype revision lockfile for transaction '%s'"), 739 txn_id); 740 apr_err = apr_file_close(lockfile); 741 if (apr_err) 742 return svn_error_wrap_apr 743 (apr_err, 744 _("Can't close prototype revision lockfile for transaction '%s'"), 745 txn_id); 746 747 txn->being_written = FALSE; 748 749 return SVN_NO_ERROR; 750} 751 752/* Unlock the prototype revision file for transaction TXN_ID in filesystem 753 FS using cookie LOCKCOOKIE. The original prototype revision file must 754 have been closed _before_ calling this function. 755 756 Perform temporary allocations in POOL. */ 757static svn_error_t * 758unlock_proto_rev(svn_fs_t *fs, const char *txn_id, void *lockcookie, 759 apr_pool_t *pool) 760{ 761 struct unlock_proto_rev_baton b; 762 763 b.txn_id = txn_id; 764 b.lockcookie = lockcookie; 765 return with_txnlist_lock(fs, unlock_proto_rev_body, &b, pool); 766} 767 768/* Same as unlock_proto_rev(), but requires that the transaction list 769 lock is already held. */ 770static svn_error_t * 771unlock_proto_rev_list_locked(svn_fs_t *fs, const char *txn_id, 772 void *lockcookie, 773 apr_pool_t *pool) 774{ 775 struct unlock_proto_rev_baton b; 776 777 b.txn_id = txn_id; 778 b.lockcookie = lockcookie; 779 return unlock_proto_rev_body(fs, &b, pool); 780} 781 782/* A structure used by get_writable_proto_rev() and 783 get_writable_proto_rev_body(), which see. */ 784struct get_writable_proto_rev_baton 785{ 786 apr_file_t **file; 787 void **lockcookie; 788 const char *txn_id; 789}; 790 791/* Callback used in the implementation of get_writable_proto_rev(). */ 792static svn_error_t * 793get_writable_proto_rev_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) 794{ 795 const struct get_writable_proto_rev_baton *b = baton; 796 apr_file_t **file = b->file; 797 void **lockcookie = b->lockcookie; 798 const char *txn_id = b->txn_id; 799 svn_error_t *err; 800 fs_fs_shared_txn_data_t *txn = get_shared_txn(fs, txn_id, TRUE); 801 802 /* First, ensure that no thread in this process (including this one) 803 is currently writing to this transaction's proto-rev file. */ 804 if (txn->being_written) 805 return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL, 806 _("Cannot write to the prototype revision file " 807 "of transaction '%s' because a previous " 808 "representation is currently being written by " 809 "this process"), 810 txn_id); 811 812 813 /* We know that no thread in this process is writing to the proto-rev 814 file, and by extension, that no thread in this process is holding a 815 lock on the prototype revision lock file. It is therefore safe 816 for us to attempt to lock this file, to see if any other process 817 is holding a lock. */ 818 819 { 820 apr_file_t *lockfile; 821 apr_status_t apr_err; 822 const char *lockfile_path = path_txn_proto_rev_lock(fs, txn_id, pool); 823 824 /* Open the proto-rev lockfile, creating it if necessary, as it may 825 not exist if the transaction dates from before the lockfiles were 826 introduced. 827 828 ### We'd also like to use something like svn_io_file_lock2(), but 829 that forces us to create a subpool just to be able to unlock 830 the file, which seems a waste. */ 831 SVN_ERR(svn_io_file_open(&lockfile, lockfile_path, 832 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); 833 834 apr_err = apr_file_lock(lockfile, 835 APR_FLOCK_EXCLUSIVE | APR_FLOCK_NONBLOCK); 836 if (apr_err) 837 { 838 svn_error_clear(svn_io_file_close(lockfile, pool)); 839 840 if (APR_STATUS_IS_EAGAIN(apr_err)) 841 return svn_error_createf(SVN_ERR_FS_REP_BEING_WRITTEN, NULL, 842 _("Cannot write to the prototype revision " 843 "file of transaction '%s' because a " 844 "previous representation is currently " 845 "being written by another process"), 846 txn_id); 847 848 return svn_error_wrap_apr(apr_err, 849 _("Can't get exclusive lock on file '%s'"), 850 svn_dirent_local_style(lockfile_path, pool)); 851 } 852 853 *lockcookie = lockfile; 854 } 855 856 /* We've successfully locked the transaction; mark it as such. */ 857 txn->being_written = TRUE; 858 859 860 /* Now open the prototype revision file and seek to the end. */ 861 err = svn_io_file_open(file, path_txn_proto_rev(fs, txn_id, pool), 862 APR_WRITE | APR_BUFFERED, APR_OS_DEFAULT, pool); 863 864 /* You might expect that we could dispense with the following seek 865 and achieve the same thing by opening the file using APR_APPEND. 866 Unfortunately, APR's buffered file implementation unconditionally 867 places its initial file pointer at the start of the file (even for 868 files opened with APR_APPEND), so we need this seek to reconcile 869 the APR file pointer to the OS file pointer (since we need to be 870 able to read the current file position later). */ 871 if (!err) 872 { 873 apr_off_t offset = 0; 874 err = svn_io_file_seek(*file, APR_END, &offset, pool); 875 } 876 877 if (err) 878 { 879 err = svn_error_compose_create( 880 err, 881 unlock_proto_rev_list_locked(fs, txn_id, *lockcookie, pool)); 882 883 *lockcookie = NULL; 884 } 885 886 return svn_error_trace(err); 887} 888 889/* Get a handle to the prototype revision file for transaction TXN_ID in 890 filesystem FS, and lock it for writing. Return FILE, a file handle 891 positioned at the end of the file, and LOCKCOOKIE, a cookie that 892 should be passed to unlock_proto_rev() to unlock the file once FILE 893 has been closed. 894 895 If the prototype revision file is already locked, return error 896 SVN_ERR_FS_REP_BEING_WRITTEN. 897 898 Perform all allocations in POOL. */ 899static svn_error_t * 900get_writable_proto_rev(apr_file_t **file, 901 void **lockcookie, 902 svn_fs_t *fs, const char *txn_id, 903 apr_pool_t *pool) 904{ 905 struct get_writable_proto_rev_baton b; 906 907 b.file = file; 908 b.lockcookie = lockcookie; 909 b.txn_id = txn_id; 910 911 return with_txnlist_lock(fs, get_writable_proto_rev_body, &b, pool); 912} 913 914/* Callback used in the implementation of purge_shared_txn(). */ 915static svn_error_t * 916purge_shared_txn_body(svn_fs_t *fs, const void *baton, apr_pool_t *pool) 917{ 918 const char *txn_id = baton; 919 920 free_shared_txn(fs, txn_id); 921 svn_fs_fs__reset_txn_caches(fs); 922 923 return SVN_NO_ERROR; 924} 925 926/* Purge the shared data for transaction TXN_ID in filesystem FS. 927 Perform all allocations in POOL. */ 928static svn_error_t * 929purge_shared_txn(svn_fs_t *fs, const char *txn_id, apr_pool_t *pool) 930{ 931 return with_txnlist_lock(fs, purge_shared_txn_body, txn_id, pool); 932} 933 934 935 936/* Fetch the current offset of FILE into *OFFSET_P. */ 937static svn_error_t * 938get_file_offset(apr_off_t *offset_p, apr_file_t *file, apr_pool_t *pool) 939{ 940 apr_off_t offset; 941 942 /* Note that, for buffered files, one (possibly surprising) side-effect 943 of this call is to flush any unwritten data to disk. */ 944 offset = 0; 945 SVN_ERR(svn_io_file_seek(file, APR_CUR, &offset, pool)); 946 *offset_p = offset; 947 948 return SVN_NO_ERROR; 949} 950 951 952/* Check that BUF, a nul-terminated buffer of text from file PATH, 953 contains only digits at OFFSET and beyond, raising an error if not. 954 TITLE contains a user-visible description of the file, usually the 955 short file name. 956 957 Uses POOL for temporary allocation. */ 958static svn_error_t * 959check_file_buffer_numeric(const char *buf, apr_off_t offset, 960 const char *path, const char *title, 961 apr_pool_t *pool) 962{ 963 const char *p; 964 965 for (p = buf + offset; *p; p++) 966 if (!svn_ctype_isdigit(*p)) 967 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, 968 _("%s file '%s' contains unexpected non-digit '%c' within '%s'"), 969 title, svn_dirent_local_style(path, pool), *p, buf); 970 971 return SVN_NO_ERROR; 972} 973 974/* Check that BUF, a nul-terminated buffer of text from format file PATH, 975 contains only digits at OFFSET and beyond, raising an error if not. 976 977 Uses POOL for temporary allocation. */ 978static svn_error_t * 979check_format_file_buffer_numeric(const char *buf, apr_off_t offset, 980 const char *path, apr_pool_t *pool) 981{ 982 return check_file_buffer_numeric(buf, offset, path, "Format", pool); 983} 984 985/* Read the format number and maximum number of files per directory 986 from PATH and return them in *PFORMAT and *MAX_FILES_PER_DIR 987 respectively. 988 989 *MAX_FILES_PER_DIR is obtained from the 'layout' format option, and 990 will be set to zero if a linear scheme should be used. 991 992 Use POOL for temporary allocation. */ 993static svn_error_t * 994read_format(int *pformat, int *max_files_per_dir, 995 const char *path, apr_pool_t *pool) 996{ 997 svn_error_t *err; 998 svn_stream_t *stream; 999 svn_stringbuf_t *content; 1000 svn_stringbuf_t *buf; 1001 svn_boolean_t eos = FALSE; 1002 1003 err = svn_stringbuf_from_file2(&content, path, pool); 1004 if (err && APR_STATUS_IS_ENOENT(err->apr_err)) 1005 { 1006 /* Treat an absent format file as format 1. Do not try to 1007 create the format file on the fly, because the repository 1008 might be read-only for us, or this might be a read-only 1009 operation, and the spirit of FSFS is to make no changes 1010 whatseover in read-only operations. See thread starting at 1011 http://subversion.tigris.org/servlets/ReadMsg?list=dev&msgNo=97600 1012 for more. */ 1013 svn_error_clear(err); 1014 *pformat = 1; 1015 *max_files_per_dir = 0; 1016 1017 return SVN_NO_ERROR; 1018 } 1019 SVN_ERR(err); 1020 1021 stream = svn_stream_from_stringbuf(content, pool); 1022 SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); 1023 if (buf->len == 0 && eos) 1024 { 1025 /* Return a more useful error message. */ 1026 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, 1027 _("Can't read first line of format file '%s'"), 1028 svn_dirent_local_style(path, pool)); 1029 } 1030 1031 /* Check that the first line contains only digits. */ 1032 SVN_ERR(check_format_file_buffer_numeric(buf->data, 0, path, pool)); 1033 SVN_ERR(svn_cstring_atoi(pformat, buf->data)); 1034 1035 /* Set the default values for anything that can be set via an option. */ 1036 *max_files_per_dir = 0; 1037 1038 /* Read any options. */ 1039 while (!eos) 1040 { 1041 SVN_ERR(svn_stream_readline(stream, &buf, "\n", &eos, pool)); 1042 if (buf->len == 0) 1043 break; 1044 1045 if (*pformat >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT && 1046 strncmp(buf->data, "layout ", 7) == 0) 1047 { 1048 if (strcmp(buf->data + 7, "linear") == 0) 1049 { 1050 *max_files_per_dir = 0; 1051 continue; 1052 } 1053 1054 if (strncmp(buf->data + 7, "sharded ", 8) == 0) 1055 { 1056 /* Check that the argument is numeric. */ 1057 SVN_ERR(check_format_file_buffer_numeric(buf->data, 15, path, pool)); 1058 SVN_ERR(svn_cstring_atoi(max_files_per_dir, buf->data + 15)); 1059 continue; 1060 } 1061 } 1062 1063 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT, NULL, 1064 _("'%s' contains invalid filesystem format option '%s'"), 1065 svn_dirent_local_style(path, pool), buf->data); 1066 } 1067 1068 return SVN_NO_ERROR; 1069} 1070 1071/* Write the format number and maximum number of files per directory 1072 to a new format file in PATH, possibly expecting to overwrite a 1073 previously existing file. 1074 1075 Use POOL for temporary allocation. */ 1076static svn_error_t * 1077write_format(const char *path, int format, int max_files_per_dir, 1078 svn_boolean_t overwrite, apr_pool_t *pool) 1079{ 1080 svn_stringbuf_t *sb; 1081 1082 SVN_ERR_ASSERT(1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER); 1083 1084 sb = svn_stringbuf_createf(pool, "%d\n", format); 1085 1086 if (format >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT) 1087 { 1088 if (max_files_per_dir) 1089 svn_stringbuf_appendcstr(sb, apr_psprintf(pool, "layout sharded %d\n", 1090 max_files_per_dir)); 1091 else 1092 svn_stringbuf_appendcstr(sb, "layout linear\n"); 1093 } 1094 1095 /* svn_io_write_version_file() does a load of magic to allow it to 1096 replace version files that already exist. We only need to do 1097 that when we're allowed to overwrite an existing file. */ 1098 if (! overwrite) 1099 { 1100 /* Create the file */ 1101 SVN_ERR(svn_io_file_create(path, sb->data, pool)); 1102 } 1103 else 1104 { 1105 const char *path_tmp; 1106 1107 SVN_ERR(svn_io_write_unique(&path_tmp, 1108 svn_dirent_dirname(path, pool), 1109 sb->data, sb->len, 1110 svn_io_file_del_none, pool)); 1111 1112 /* rename the temp file as the real destination */ 1113 SVN_ERR(svn_io_file_rename(path_tmp, path, pool)); 1114 } 1115 1116 /* And set the perms to make it read only */ 1117 return svn_io_set_file_read_only(path, FALSE, pool); 1118} 1119 1120/* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format 1121 number is not the same as a format number supported by this 1122 Subversion. */ 1123static svn_error_t * 1124check_format(int format) 1125{ 1126 /* Blacklist. These formats may be either younger or older than 1127 SVN_FS_FS__FORMAT_NUMBER, but we don't support them. */ 1128 if (format == SVN_FS_FS__PACKED_REVPROP_SQLITE_DEV_FORMAT) 1129 return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, 1130 _("Found format '%d', only created by " 1131 "unreleased dev builds; see " 1132 "http://subversion.apache.org" 1133 "/docs/release-notes/1.7#revprop-packing"), 1134 format); 1135 1136 /* We support all formats from 1-current simultaneously */ 1137 if (1 <= format && format <= SVN_FS_FS__FORMAT_NUMBER) 1138 return SVN_NO_ERROR; 1139 1140 return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT, NULL, 1141 _("Expected FS format between '1' and '%d'; found format '%d'"), 1142 SVN_FS_FS__FORMAT_NUMBER, format); 1143} 1144 1145svn_boolean_t 1146svn_fs_fs__fs_supports_mergeinfo(svn_fs_t *fs) 1147{ 1148 fs_fs_data_t *ffd = fs->fsap_data; 1149 return ffd->format >= SVN_FS_FS__MIN_MERGEINFO_FORMAT; 1150} 1151 1152/* Read the configuration information of the file system at FS_PATH 1153 * and set the respective values in FFD. Use POOL for allocations. 1154 */ 1155static svn_error_t * 1156read_config(fs_fs_data_t *ffd, 1157 const char *fs_path, 1158 apr_pool_t *pool) 1159{ 1160 SVN_ERR(svn_config_read3(&ffd->config, 1161 svn_dirent_join(fs_path, PATH_CONFIG, pool), 1162 FALSE, FALSE, FALSE, pool)); 1163 1164 /* Initialize ffd->rep_sharing_allowed. */ 1165 if (ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) 1166 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->rep_sharing_allowed, 1167 CONFIG_SECTION_REP_SHARING, 1168 CONFIG_OPTION_ENABLE_REP_SHARING, TRUE)); 1169 else 1170 ffd->rep_sharing_allowed = FALSE; 1171 1172 /* Initialize deltification settings in ffd. */ 1173 if (ffd->format >= SVN_FS_FS__MIN_DELTIFICATION_FORMAT) 1174 { 1175 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_directories, 1176 CONFIG_SECTION_DELTIFICATION, 1177 CONFIG_OPTION_ENABLE_DIR_DELTIFICATION, 1178 FALSE)); 1179 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->deltify_properties, 1180 CONFIG_SECTION_DELTIFICATION, 1181 CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION, 1182 FALSE)); 1183 SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_deltification_walk, 1184 CONFIG_SECTION_DELTIFICATION, 1185 CONFIG_OPTION_MAX_DELTIFICATION_WALK, 1186 SVN_FS_FS_MAX_DELTIFICATION_WALK)); 1187 SVN_ERR(svn_config_get_int64(ffd->config, &ffd->max_linear_deltification, 1188 CONFIG_SECTION_DELTIFICATION, 1189 CONFIG_OPTION_MAX_LINEAR_DELTIFICATION, 1190 SVN_FS_FS_MAX_LINEAR_DELTIFICATION)); 1191 } 1192 else 1193 { 1194 ffd->deltify_directories = FALSE; 1195 ffd->deltify_properties = FALSE; 1196 ffd->max_deltification_walk = SVN_FS_FS_MAX_DELTIFICATION_WALK; 1197 ffd->max_linear_deltification = SVN_FS_FS_MAX_LINEAR_DELTIFICATION; 1198 } 1199 1200 /* Initialize revprop packing settings in ffd. */ 1201 if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 1202 { 1203 SVN_ERR(svn_config_get_bool(ffd->config, &ffd->compress_packed_revprops, 1204 CONFIG_SECTION_PACKED_REVPROPS, 1205 CONFIG_OPTION_COMPRESS_PACKED_REVPROPS, 1206 FALSE)); 1207 SVN_ERR(svn_config_get_int64(ffd->config, &ffd->revprop_pack_size, 1208 CONFIG_SECTION_PACKED_REVPROPS, 1209 CONFIG_OPTION_REVPROP_PACK_SIZE, 1210 ffd->compress_packed_revprops 1211 ? 0x100 1212 : 0x40)); 1213 1214 ffd->revprop_pack_size *= 1024; 1215 } 1216 else 1217 { 1218 ffd->revprop_pack_size = 0x10000; 1219 ffd->compress_packed_revprops = FALSE; 1220 } 1221 1222 return SVN_NO_ERROR; 1223} 1224 1225static svn_error_t * 1226write_config(svn_fs_t *fs, 1227 apr_pool_t *pool) 1228{ 1229#define NL APR_EOL_STR 1230 static const char * const fsfs_conf_contents = 1231"### This file controls the configuration of the FSFS filesystem." NL 1232"" NL 1233"[" SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS "]" NL 1234"### These options name memcached servers used to cache internal FSFS" NL 1235"### data. See http://www.danga.com/memcached/ for more information on" NL 1236"### memcached. To use memcached with FSFS, run one or more memcached" NL 1237"### servers, and specify each of them as an option like so:" NL 1238"# first-server = 127.0.0.1:11211" NL 1239"# remote-memcached = mymemcached.corp.example.com:11212" NL 1240"### The option name is ignored; the value is of the form HOST:PORT." NL 1241"### memcached servers can be shared between multiple repositories;" NL 1242"### however, if you do this, you *must* ensure that repositories have" NL 1243"### distinct UUIDs and paths, or else cached data from one repository" NL 1244"### might be used by another accidentally. Note also that memcached has" NL 1245"### no authentication for reads or writes, so you must ensure that your" NL 1246"### memcached servers are only accessible by trusted users." NL 1247"" NL 1248"[" CONFIG_SECTION_CACHES "]" NL 1249"### When a cache-related error occurs, normally Subversion ignores it" NL 1250"### and continues, logging an error if the server is appropriately" NL 1251"### configured (and ignoring it with file:// access). To make" NL 1252"### Subversion never ignore cache errors, uncomment this line." NL 1253"# " CONFIG_OPTION_FAIL_STOP " = true" NL 1254"" NL 1255"[" CONFIG_SECTION_REP_SHARING "]" NL 1256"### To conserve space, the filesystem can optionally avoid storing" NL 1257"### duplicate representations. This comes at a slight cost in" NL 1258"### performance, as maintaining a database of shared representations can" NL 1259"### increase commit times. The space savings are dependent upon the size" NL 1260"### of the repository, the number of objects it contains and the amount of" NL 1261"### duplication between them, usually a function of the branching and" NL 1262"### merging process." NL 1263"###" NL 1264"### The following parameter enables rep-sharing in the repository. It can" NL 1265"### be switched on and off at will, but for best space-saving results" NL 1266"### should be enabled consistently over the life of the repository." NL 1267"### 'svnadmin verify' will check the rep-cache regardless of this setting." NL 1268"### rep-sharing is enabled by default." NL 1269"# " CONFIG_OPTION_ENABLE_REP_SHARING " = true" NL 1270"" NL 1271"[" CONFIG_SECTION_DELTIFICATION "]" NL 1272"### To conserve space, the filesystem stores data as differences against" NL 1273"### existing representations. This comes at a slight cost in performance," NL 1274"### as calculating differences can increase commit times. Reading data" NL 1275"### will also create higher CPU load and the data will be fragmented." NL 1276"### Since deltification tends to save significant amounts of disk space," NL 1277"### the overall I/O load can actually be lower." NL 1278"###" NL 1279"### The options in this section allow for tuning the deltification" NL 1280"### strategy. Their effects on data size and server performance may vary" NL 1281"### from one repository to another. Versions prior to 1.8 will ignore" NL 1282"### this section." NL 1283"###" NL 1284"### The following parameter enables deltification for directories. It can" NL 1285"### be switched on and off at will, but for best space-saving results" NL 1286"### should be enabled consistently over the life of the repository." NL 1287"### Repositories containing large directories will benefit greatly." NL 1288"### In rarely read repositories, the I/O overhead may be significant as" NL 1289"### cache hit rates will most likely be low" NL 1290"### directory deltification is disabled by default." NL 1291"# " CONFIG_OPTION_ENABLE_DIR_DELTIFICATION " = false" NL 1292"###" NL 1293"### The following parameter enables deltification for properties on files" NL 1294"### and directories. Overall, this is a minor tuning option but can save" NL 1295"### some disk space if you merge frequently or frequently change node" NL 1296"### properties. You should not activate this if rep-sharing has been" NL 1297"### disabled because this may result in a net increase in repository size." NL 1298"### property deltification is disabled by default." NL 1299"# " CONFIG_OPTION_ENABLE_PROPS_DELTIFICATION " = false" NL 1300"###" NL 1301"### During commit, the server may need to walk the whole change history of" NL 1302"### of a given node to find a suitable deltification base. This linear" NL 1303"### process can impact commit times, svnadmin load and similar operations." NL 1304"### This setting limits the depth of the deltification history. If the" NL 1305"### threshold has been reached, the node will be stored as fulltext and a" NL 1306"### new deltification history begins." NL 1307"### Note, this is unrelated to svn log." NL 1308"### Very large values rarely provide significant additional savings but" NL 1309"### can impact performance greatly - in particular if directory" NL 1310"### deltification has been activated. Very small values may be useful in" NL 1311"### repositories that are dominated by large, changing binaries." NL 1312"### Should be a power of two minus 1. A value of 0 will effectively" NL 1313"### disable deltification." NL 1314"### For 1.8, the default value is 1023; earlier versions have no limit." NL 1315"# " CONFIG_OPTION_MAX_DELTIFICATION_WALK " = 1023" NL 1316"###" NL 1317"### The skip-delta scheme used by FSFS tends to repeatably store redundant" NL 1318"### delta information where a simple delta against the latest version is" NL 1319"### often smaller. By default, 1.8+ will therefore use skip deltas only" NL 1320"### after the linear chain of deltas has grown beyond the threshold" NL 1321"### specified by this setting." NL 1322"### Values up to 64 can result in some reduction in repository size for" NL 1323"### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL 1324"### numbers can reduce those costs at the cost of more disk space. For" NL 1325"### rarely read repositories or those containing larger binaries, this may" NL 1326"### present a better trade-off." NL 1327"### Should be a power of two. A value of 1 or smaller will cause the" NL 1328"### exclusive use of skip-deltas (as in pre-1.8)." NL 1329"### For 1.8, the default value is 16; earlier versions use 1." NL 1330"# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION " = 16" NL 1331"" NL 1332"[" CONFIG_SECTION_PACKED_REVPROPS "]" NL 1333"### This parameter controls the size (in kBytes) of packed revprop files." NL 1334"### Revprops of consecutive revisions will be concatenated into a single" NL 1335"### file up to but not exceeding the threshold given here. However, each" NL 1336"### pack file may be much smaller and revprops of a single revision may be" NL 1337"### much larger than the limit set here. The threshold will be applied" NL 1338"### before optional compression takes place." NL 1339"### Large values will reduce disk space usage at the expense of increased" NL 1340"### latency and CPU usage reading and changing individual revprops. They" NL 1341"### become an advantage when revprop caching has been enabled because a" NL 1342"### lot of data can be read in one go. Values smaller than 4 kByte will" NL 1343"### not improve latency any further and quickly render revprop packing" NL 1344"### ineffective." NL 1345"### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL 1346"### pack files and 256 kBytes when compression has been enabled." NL 1347"# " CONFIG_OPTION_REVPROP_PACK_SIZE " = 64" NL 1348"###" NL 1349"### To save disk space, packed revprop files may be compressed. Standard" NL 1350"### revprops tend to allow for very effective compression. Reading and" NL 1351"### even more so writing, become significantly more CPU intensive. With" NL 1352"### revprop caching enabled, the overhead can be offset by reduced I/O" NL 1353"### unless you often modify revprops after packing." NL 1354"### Compressing packed revprops is disabled by default." NL 1355"# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS " = false" NL 1356; 1357#undef NL 1358 return svn_io_file_create(svn_dirent_join(fs->path, PATH_CONFIG, pool), 1359 fsfs_conf_contents, pool); 1360} 1361 1362static svn_error_t * 1363read_min_unpacked_rev(svn_revnum_t *min_unpacked_rev, 1364 const char *path, 1365 apr_pool_t *pool) 1366{ 1367 char buf[80]; 1368 apr_file_t *file; 1369 apr_size_t len; 1370 1371 SVN_ERR(svn_io_file_open(&file, path, APR_READ | APR_BUFFERED, 1372 APR_OS_DEFAULT, pool)); 1373 len = sizeof(buf); 1374 SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); 1375 SVN_ERR(svn_io_file_close(file, pool)); 1376 1377 *min_unpacked_rev = SVN_STR_TO_REV(buf); 1378 return SVN_NO_ERROR; 1379} 1380 1381static svn_error_t * 1382update_min_unpacked_rev(svn_fs_t *fs, apr_pool_t *pool) 1383{ 1384 fs_fs_data_t *ffd = fs->fsap_data; 1385 1386 SVN_ERR_ASSERT(ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT); 1387 1388 return read_min_unpacked_rev(&ffd->min_unpacked_rev, 1389 path_min_unpacked_rev(fs, pool), 1390 pool); 1391} 1392 1393svn_error_t * 1394svn_fs_fs__open(svn_fs_t *fs, const char *path, apr_pool_t *pool) 1395{ 1396 fs_fs_data_t *ffd = fs->fsap_data; 1397 apr_file_t *uuid_file; 1398 int format, max_files_per_dir; 1399 char buf[APR_UUID_FORMATTED_LENGTH + 2]; 1400 apr_size_t limit; 1401 1402 fs->path = apr_pstrdup(fs->pool, path); 1403 1404 /* Read the FS format number. */ 1405 SVN_ERR(read_format(&format, &max_files_per_dir, 1406 path_format(fs, pool), pool)); 1407 SVN_ERR(check_format(format)); 1408 1409 /* Now we've got a format number no matter what. */ 1410 ffd->format = format; 1411 ffd->max_files_per_dir = max_files_per_dir; 1412 1413 /* Read in and cache the repository uuid. */ 1414 SVN_ERR(svn_io_file_open(&uuid_file, path_uuid(fs, pool), 1415 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); 1416 1417 limit = sizeof(buf); 1418 SVN_ERR(svn_io_read_length_line(uuid_file, buf, &limit, pool)); 1419 fs->uuid = apr_pstrdup(fs->pool, buf); 1420 1421 SVN_ERR(svn_io_file_close(uuid_file, pool)); 1422 1423 /* Read the min unpacked revision. */ 1424 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 1425 SVN_ERR(update_min_unpacked_rev(fs, pool)); 1426 1427 /* Read the configuration file. */ 1428 SVN_ERR(read_config(ffd, fs->path, pool)); 1429 1430 return get_youngest(&(ffd->youngest_rev_cache), path, pool); 1431} 1432 1433/* Wrapper around svn_io_file_create which ignores EEXIST. */ 1434static svn_error_t * 1435create_file_ignore_eexist(const char *file, 1436 const char *contents, 1437 apr_pool_t *pool) 1438{ 1439 svn_error_t *err = svn_io_file_create(file, contents, pool); 1440 if (err && APR_STATUS_IS_EEXIST(err->apr_err)) 1441 { 1442 svn_error_clear(err); 1443 err = SVN_NO_ERROR; 1444 } 1445 return svn_error_trace(err); 1446} 1447 1448/* forward declarations */ 1449 1450static svn_error_t * 1451pack_revprops_shard(const char *pack_file_dir, 1452 const char *shard_path, 1453 apr_int64_t shard, 1454 int max_files_per_dir, 1455 apr_off_t max_pack_size, 1456 int compression_level, 1457 svn_cancel_func_t cancel_func, 1458 void *cancel_baton, 1459 apr_pool_t *scratch_pool); 1460 1461static svn_error_t * 1462delete_revprops_shard(const char *shard_path, 1463 apr_int64_t shard, 1464 int max_files_per_dir, 1465 svn_cancel_func_t cancel_func, 1466 void *cancel_baton, 1467 apr_pool_t *scratch_pool); 1468 1469/* In the filesystem FS, pack all revprop shards up to min_unpacked_rev. 1470 * 1471 * NOTE: Keep the old non-packed shards around until after the format bump. 1472 * Otherwise, re-running upgrade will drop the packed revprop shard but 1473 * have no unpacked data anymore. Call upgrade_cleanup_pack_revprops after 1474 * the bump. 1475 * 1476 * Use SCRATCH_POOL for temporary allocations. 1477 */ 1478static svn_error_t * 1479upgrade_pack_revprops(svn_fs_t *fs, 1480 apr_pool_t *scratch_pool) 1481{ 1482 fs_fs_data_t *ffd = fs->fsap_data; 1483 const char *revprops_shard_path; 1484 const char *revprops_pack_file_dir; 1485 apr_int64_t shard; 1486 apr_int64_t first_unpacked_shard 1487 = ffd->min_unpacked_rev / ffd->max_files_per_dir; 1488 1489 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1490 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 1491 scratch_pool); 1492 int compression_level = ffd->compress_packed_revprops 1493 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 1494 : SVN_DELTA_COMPRESSION_LEVEL_NONE; 1495 1496 /* first, pack all revprops shards to match the packed revision shards */ 1497 for (shard = 0; shard < first_unpacked_shard; ++shard) 1498 { 1499 revprops_pack_file_dir = svn_dirent_join(revsprops_dir, 1500 apr_psprintf(iterpool, 1501 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, 1502 shard), 1503 iterpool); 1504 revprops_shard_path = svn_dirent_join(revsprops_dir, 1505 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 1506 iterpool); 1507 1508 SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path, 1509 shard, ffd->max_files_per_dir, 1510 (int)(0.9 * ffd->revprop_pack_size), 1511 compression_level, 1512 NULL, NULL, iterpool)); 1513 svn_pool_clear(iterpool); 1514 } 1515 1516 svn_pool_destroy(iterpool); 1517 1518 return SVN_NO_ERROR; 1519} 1520 1521/* In the filesystem FS, remove all non-packed revprop shards up to 1522 * min_unpacked_rev. Use SCRATCH_POOL for temporary allocations. 1523 * See upgrade_pack_revprops for more info. 1524 */ 1525static svn_error_t * 1526upgrade_cleanup_pack_revprops(svn_fs_t *fs, 1527 apr_pool_t *scratch_pool) 1528{ 1529 fs_fs_data_t *ffd = fs->fsap_data; 1530 const char *revprops_shard_path; 1531 apr_int64_t shard; 1532 apr_int64_t first_unpacked_shard 1533 = ffd->min_unpacked_rev / ffd->max_files_per_dir; 1534 1535 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1536 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 1537 scratch_pool); 1538 1539 /* delete the non-packed revprops shards afterwards */ 1540 for (shard = 0; shard < first_unpacked_shard; ++shard) 1541 { 1542 revprops_shard_path = svn_dirent_join(revsprops_dir, 1543 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 1544 iterpool); 1545 SVN_ERR(delete_revprops_shard(revprops_shard_path, 1546 shard, ffd->max_files_per_dir, 1547 NULL, NULL, iterpool)); 1548 svn_pool_clear(iterpool); 1549 } 1550 1551 svn_pool_destroy(iterpool); 1552 1553 return SVN_NO_ERROR; 1554} 1555 1556static svn_error_t * 1557upgrade_body(void *baton, apr_pool_t *pool) 1558{ 1559 svn_fs_t *fs = baton; 1560 int format, max_files_per_dir; 1561 const char *format_path = path_format(fs, pool); 1562 svn_node_kind_t kind; 1563 svn_boolean_t needs_revprop_shard_cleanup = FALSE; 1564 1565 /* Read the FS format number and max-files-per-dir setting. */ 1566 SVN_ERR(read_format(&format, &max_files_per_dir, format_path, pool)); 1567 SVN_ERR(check_format(format)); 1568 1569 /* If the config file does not exist, create one. */ 1570 SVN_ERR(svn_io_check_path(svn_dirent_join(fs->path, PATH_CONFIG, pool), 1571 &kind, pool)); 1572 switch (kind) 1573 { 1574 case svn_node_none: 1575 SVN_ERR(write_config(fs, pool)); 1576 break; 1577 case svn_node_file: 1578 break; 1579 default: 1580 return svn_error_createf(SVN_ERR_FS_GENERAL, NULL, 1581 _("'%s' is not a regular file." 1582 " Please move it out of " 1583 "the way and try again"), 1584 svn_dirent_join(fs->path, PATH_CONFIG, pool)); 1585 } 1586 1587 /* If we're already up-to-date, there's nothing else to be done here. */ 1588 if (format == SVN_FS_FS__FORMAT_NUMBER) 1589 return SVN_NO_ERROR; 1590 1591 /* If our filesystem predates the existance of the 'txn-current 1592 file', make that file and its corresponding lock file. */ 1593 if (format < SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) 1594 { 1595 SVN_ERR(create_file_ignore_eexist(path_txn_current(fs, pool), "0\n", 1596 pool)); 1597 SVN_ERR(create_file_ignore_eexist(path_txn_current_lock(fs, pool), "", 1598 pool)); 1599 } 1600 1601 /* If our filesystem predates the existance of the 'txn-protorevs' 1602 dir, make that directory. */ 1603 if (format < SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) 1604 { 1605 /* We don't use path_txn_proto_rev() here because it expects 1606 we've already bumped our format. */ 1607 SVN_ERR(svn_io_make_dir_recursively( 1608 svn_dirent_join(fs->path, PATH_TXN_PROTOS_DIR, pool), pool)); 1609 } 1610 1611 /* If our filesystem is new enough, write the min unpacked rev file. */ 1612 if (format < SVN_FS_FS__MIN_PACKED_FORMAT) 1613 SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool)); 1614 1615 /* If the file system supports revision packing but not revprop packing 1616 *and* the FS has been sharded, pack the revprops up to the point that 1617 revision data has been packed. However, keep the non-packed revprop 1618 files around until after the format bump */ 1619 if ( format >= SVN_FS_FS__MIN_PACKED_FORMAT 1620 && format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT 1621 && max_files_per_dir > 0) 1622 { 1623 needs_revprop_shard_cleanup = TRUE; 1624 SVN_ERR(upgrade_pack_revprops(fs, pool)); 1625 } 1626 1627 /* Bump the format file. */ 1628 SVN_ERR(write_format(format_path, SVN_FS_FS__FORMAT_NUMBER, 1629 max_files_per_dir, TRUE, pool)); 1630 1631 /* Now, it is safe to remove the redundant revprop files. */ 1632 if (needs_revprop_shard_cleanup) 1633 SVN_ERR(upgrade_cleanup_pack_revprops(fs, pool)); 1634 1635 /* Done */ 1636 return SVN_NO_ERROR; 1637} 1638 1639 1640svn_error_t * 1641svn_fs_fs__upgrade(svn_fs_t *fs, apr_pool_t *pool) 1642{ 1643 return svn_fs_fs__with_write_lock(fs, upgrade_body, (void *)fs, pool); 1644} 1645 1646 1647/* Functions for dealing with recoverable errors on mutable files 1648 * 1649 * Revprops, current, and txn-current files are mutable; that is, they 1650 * change as part of normal fsfs operation, in constrat to revs files, or 1651 * the format file, which are written once at create (or upgrade) time. 1652 * When more than one host writes to the same repository, we will 1653 * sometimes see these recoverable errors when accesssing these files. 1654 * 1655 * These errors all relate to NFS, and thus we only use this retry code if 1656 * ESTALE is defined. 1657 * 1658 ** ESTALE 1659 * 1660 * In NFS v3 and under, the server doesn't track opened files. If you 1661 * unlink(2) or rename(2) a file held open by another process *on the 1662 * same host*, that host's kernel typically renames the file to 1663 * .nfsXXXX and automatically deletes that when it's no longer open, 1664 * but this behavior is not required. 1665 * 1666 * For obvious reasons, this does not work *across hosts*. No one 1667 * knows about the opened file; not the server, and not the deleting 1668 * client. So the file vanishes, and the reader gets stale NFS file 1669 * handle. 1670 * 1671 ** EIO, ENOENT 1672 * 1673 * Some client implementations (at least the 2.6.18.5 kernel that ships 1674 * with Ubuntu Dapper) sometimes give spurious ENOENT (only on open) or 1675 * even EIO errors when trying to read these files that have been renamed 1676 * over on some other host. 1677 * 1678 ** Solution 1679 * 1680 * Try open and read of such files in try_stringbuf_from_file(). Call 1681 * this function within a loop of RECOVERABLE_RETRY_COUNT iterations 1682 * (though, realistically, the second try will succeed). 1683 */ 1684 1685#define RECOVERABLE_RETRY_COUNT 10 1686 1687/* Read the file at PATH and return its content in *CONTENT. *CONTENT will 1688 * not be modified unless the whole file was read successfully. 1689 * 1690 * ESTALE, EIO and ENOENT will not cause this function to return an error 1691 * unless LAST_ATTEMPT has been set. If MISSING is not NULL, indicate 1692 * missing files (ENOENT) there. 1693 * 1694 * Use POOL for allocations. 1695 */ 1696static svn_error_t * 1697try_stringbuf_from_file(svn_stringbuf_t **content, 1698 svn_boolean_t *missing, 1699 const char *path, 1700 svn_boolean_t last_attempt, 1701 apr_pool_t *pool) 1702{ 1703 svn_error_t *err = svn_stringbuf_from_file2(content, path, pool); 1704 if (missing) 1705 *missing = FALSE; 1706 1707 if (err) 1708 { 1709 *content = NULL; 1710 1711 if (APR_STATUS_IS_ENOENT(err->apr_err)) 1712 { 1713 if (!last_attempt) 1714 { 1715 svn_error_clear(err); 1716 if (missing) 1717 *missing = TRUE; 1718 return SVN_NO_ERROR; 1719 } 1720 } 1721#ifdef ESTALE 1722 else if (APR_TO_OS_ERROR(err->apr_err) == ESTALE 1723 || APR_TO_OS_ERROR(err->apr_err) == EIO) 1724 { 1725 if (!last_attempt) 1726 { 1727 svn_error_clear(err); 1728 return SVN_NO_ERROR; 1729 } 1730 } 1731#endif 1732 } 1733 1734 return svn_error_trace(err); 1735} 1736 1737/* Read the 'current' file FNAME and store the contents in *BUF. 1738 Allocations are performed in POOL. */ 1739static svn_error_t * 1740read_content(svn_stringbuf_t **content, const char *fname, apr_pool_t *pool) 1741{ 1742 int i; 1743 *content = NULL; 1744 1745 for (i = 0; !*content && (i < RECOVERABLE_RETRY_COUNT); ++i) 1746 SVN_ERR(try_stringbuf_from_file(content, NULL, 1747 fname, i + 1 < RECOVERABLE_RETRY_COUNT, 1748 pool)); 1749 1750 if (!*content) 1751 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 1752 _("Can't read '%s'"), 1753 svn_dirent_local_style(fname, pool)); 1754 1755 return SVN_NO_ERROR; 1756} 1757 1758/* Find the youngest revision in a repository at path FS_PATH and 1759 return it in *YOUNGEST_P. Perform temporary allocations in 1760 POOL. */ 1761static svn_error_t * 1762get_youngest(svn_revnum_t *youngest_p, 1763 const char *fs_path, 1764 apr_pool_t *pool) 1765{ 1766 svn_stringbuf_t *buf; 1767 SVN_ERR(read_content(&buf, svn_dirent_join(fs_path, PATH_CURRENT, pool), 1768 pool)); 1769 1770 *youngest_p = SVN_STR_TO_REV(buf->data); 1771 1772 return SVN_NO_ERROR; 1773} 1774 1775 1776svn_error_t * 1777svn_fs_fs__youngest_rev(svn_revnum_t *youngest_p, 1778 svn_fs_t *fs, 1779 apr_pool_t *pool) 1780{ 1781 fs_fs_data_t *ffd = fs->fsap_data; 1782 1783 SVN_ERR(get_youngest(youngest_p, fs->path, pool)); 1784 ffd->youngest_rev_cache = *youngest_p; 1785 1786 return SVN_NO_ERROR; 1787} 1788 1789/* Given a revision file FILE that has been pre-positioned at the 1790 beginning of a Node-Rev header block, read in that header block and 1791 store it in the apr_hash_t HEADERS. All allocations will be from 1792 POOL. */ 1793static svn_error_t * read_header_block(apr_hash_t **headers, 1794 svn_stream_t *stream, 1795 apr_pool_t *pool) 1796{ 1797 *headers = apr_hash_make(pool); 1798 1799 while (1) 1800 { 1801 svn_stringbuf_t *header_str; 1802 const char *name, *value; 1803 apr_size_t i = 0; 1804 svn_boolean_t eof; 1805 1806 SVN_ERR(svn_stream_readline(stream, &header_str, "\n", &eof, pool)); 1807 1808 if (eof || header_str->len == 0) 1809 break; /* end of header block */ 1810 1811 while (header_str->data[i] != ':') 1812 { 1813 if (header_str->data[i] == '\0') 1814 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 1815 _("Found malformed header '%s' in " 1816 "revision file"), 1817 header_str->data); 1818 i++; 1819 } 1820 1821 /* Create a 'name' string and point to it. */ 1822 header_str->data[i] = '\0'; 1823 name = header_str->data; 1824 1825 /* Skip over the NULL byte and the space following it. */ 1826 i += 2; 1827 1828 if (i > header_str->len) 1829 { 1830 /* Restore the original line for the error. */ 1831 i -= 2; 1832 header_str->data[i] = ':'; 1833 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 1834 _("Found malformed header '%s' in " 1835 "revision file"), 1836 header_str->data); 1837 } 1838 1839 value = header_str->data + i; 1840 1841 /* header_str is safely in our pool, so we can use bits of it as 1842 key and value. */ 1843 svn_hash_sets(*headers, name, value); 1844 } 1845 1846 return SVN_NO_ERROR; 1847} 1848 1849/* Return SVN_ERR_FS_NO_SUCH_REVISION if the given revision is newer 1850 than the current youngest revision or is simply not a valid 1851 revision number, else return success. 1852 1853 FSFS is based around the concept that commits only take effect when 1854 the number in "current" is bumped. Thus if there happens to be a rev 1855 or revprops file installed for a revision higher than the one recorded 1856 in "current" (because a commit failed between installing the rev file 1857 and bumping "current", or because an administrator rolled back the 1858 repository by resetting "current" without deleting rev files, etc), it 1859 ought to be completely ignored. This function provides the check 1860 by which callers can make that decision. */ 1861static svn_error_t * 1862ensure_revision_exists(svn_fs_t *fs, 1863 svn_revnum_t rev, 1864 apr_pool_t *pool) 1865{ 1866 fs_fs_data_t *ffd = fs->fsap_data; 1867 1868 if (! SVN_IS_VALID_REVNUM(rev)) 1869 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 1870 _("Invalid revision number '%ld'"), rev); 1871 1872 1873 /* Did the revision exist the last time we checked the current 1874 file? */ 1875 if (rev <= ffd->youngest_rev_cache) 1876 return SVN_NO_ERROR; 1877 1878 SVN_ERR(get_youngest(&(ffd->youngest_rev_cache), fs->path, pool)); 1879 1880 /* Check again. */ 1881 if (rev <= ffd->youngest_rev_cache) 1882 return SVN_NO_ERROR; 1883 1884 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 1885 _("No such revision %ld"), rev); 1886} 1887 1888svn_error_t * 1889svn_fs_fs__revision_exists(svn_revnum_t rev, 1890 svn_fs_t *fs, 1891 apr_pool_t *pool) 1892{ 1893 /* Different order of parameters. */ 1894 SVN_ERR(ensure_revision_exists(fs, rev, pool)); 1895 return SVN_NO_ERROR; 1896} 1897 1898/* Open the correct revision file for REV. If the filesystem FS has 1899 been packed, *FILE will be set to the packed file; otherwise, set *FILE 1900 to the revision file for REV. Return SVN_ERR_FS_NO_SUCH_REVISION if the 1901 file doesn't exist. 1902 1903 TODO: Consider returning an indication of whether this is a packed rev 1904 file, so the caller need not rely on is_packed_rev() which in turn 1905 relies on the cached FFD->min_unpacked_rev value not having changed 1906 since the rev file was opened. 1907 1908 Use POOL for allocations. */ 1909static svn_error_t * 1910open_pack_or_rev_file(apr_file_t **file, 1911 svn_fs_t *fs, 1912 svn_revnum_t rev, 1913 apr_pool_t *pool) 1914{ 1915 fs_fs_data_t *ffd = fs->fsap_data; 1916 svn_error_t *err; 1917 const char *path; 1918 svn_boolean_t retry = FALSE; 1919 1920 do 1921 { 1922 err = svn_fs_fs__path_rev_absolute(&path, fs, rev, pool); 1923 1924 /* open the revision file in buffered r/o mode */ 1925 if (! err) 1926 err = svn_io_file_open(file, path, 1927 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool); 1928 1929 if (err && APR_STATUS_IS_ENOENT(err->apr_err)) 1930 { 1931 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 1932 { 1933 /* Could not open the file. This may happen if the 1934 * file once existed but got packed later. */ 1935 svn_error_clear(err); 1936 1937 /* if that was our 2nd attempt, leave it at that. */ 1938 if (retry) 1939 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 1940 _("No such revision %ld"), rev); 1941 1942 /* We failed for the first time. Refresh cache & retry. */ 1943 SVN_ERR(update_min_unpacked_rev(fs, pool)); 1944 1945 retry = TRUE; 1946 } 1947 else 1948 { 1949 svn_error_clear(err); 1950 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 1951 _("No such revision %ld"), rev); 1952 } 1953 } 1954 else 1955 { 1956 retry = FALSE; 1957 } 1958 } 1959 while (retry); 1960 1961 return svn_error_trace(err); 1962} 1963 1964/* Reads a line from STREAM and converts it to a 64 bit integer to be 1965 * returned in *RESULT. If we encounter eof, set *HIT_EOF and leave 1966 * *RESULT unchanged. If HIT_EOF is NULL, EOF causes an "corrupt FS" 1967 * error return. 1968 * SCRATCH_POOL is used for temporary allocations. 1969 */ 1970static svn_error_t * 1971read_number_from_stream(apr_int64_t *result, 1972 svn_boolean_t *hit_eof, 1973 svn_stream_t *stream, 1974 apr_pool_t *scratch_pool) 1975{ 1976 svn_stringbuf_t *sb; 1977 svn_boolean_t eof; 1978 svn_error_t *err; 1979 1980 SVN_ERR(svn_stream_readline(stream, &sb, "\n", &eof, scratch_pool)); 1981 if (hit_eof) 1982 *hit_eof = eof; 1983 else 1984 if (eof) 1985 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, _("Unexpected EOF")); 1986 1987 if (!eof) 1988 { 1989 err = svn_cstring_atoi64(result, sb->data); 1990 if (err) 1991 return svn_error_createf(SVN_ERR_FS_CORRUPT, err, 1992 _("Number '%s' invalid or too large"), 1993 sb->data); 1994 } 1995 1996 return SVN_NO_ERROR; 1997} 1998 1999/* Given REV in FS, set *REV_OFFSET to REV's offset in the packed file. 2000 Use POOL for temporary allocations. */ 2001static svn_error_t * 2002get_packed_offset(apr_off_t *rev_offset, 2003 svn_fs_t *fs, 2004 svn_revnum_t rev, 2005 apr_pool_t *pool) 2006{ 2007 fs_fs_data_t *ffd = fs->fsap_data; 2008 svn_stream_t *manifest_stream; 2009 svn_boolean_t is_cached; 2010 svn_revnum_t shard; 2011 apr_int64_t shard_pos; 2012 apr_array_header_t *manifest; 2013 apr_pool_t *iterpool; 2014 2015 shard = rev / ffd->max_files_per_dir; 2016 2017 /* position of the shard within the manifest */ 2018 shard_pos = rev % ffd->max_files_per_dir; 2019 2020 /* fetch exactly that element into *rev_offset, if the manifest is found 2021 in the cache */ 2022 SVN_ERR(svn_cache__get_partial((void **) rev_offset, &is_cached, 2023 ffd->packed_offset_cache, &shard, 2024 svn_fs_fs__get_sharded_offset, &shard_pos, 2025 pool)); 2026 2027 if (is_cached) 2028 return SVN_NO_ERROR; 2029 2030 /* Open the manifest file. */ 2031 SVN_ERR(svn_stream_open_readonly(&manifest_stream, 2032 path_rev_packed(fs, rev, PATH_MANIFEST, 2033 pool), 2034 pool, pool)); 2035 2036 /* While we're here, let's just read the entire manifest file into an array, 2037 so we can cache the entire thing. */ 2038 iterpool = svn_pool_create(pool); 2039 manifest = apr_array_make(pool, ffd->max_files_per_dir, sizeof(apr_off_t)); 2040 while (1) 2041 { 2042 svn_boolean_t eof; 2043 apr_int64_t val; 2044 2045 svn_pool_clear(iterpool); 2046 SVN_ERR(read_number_from_stream(&val, &eof, manifest_stream, iterpool)); 2047 if (eof) 2048 break; 2049 2050 APR_ARRAY_PUSH(manifest, apr_off_t) = (apr_off_t)val; 2051 } 2052 svn_pool_destroy(iterpool); 2053 2054 *rev_offset = APR_ARRAY_IDX(manifest, rev % ffd->max_files_per_dir, 2055 apr_off_t); 2056 2057 /* Close up shop and cache the array. */ 2058 SVN_ERR(svn_stream_close(manifest_stream)); 2059 return svn_cache__set(ffd->packed_offset_cache, &shard, manifest, pool); 2060} 2061 2062/* Open the revision file for revision REV in filesystem FS and store 2063 the newly opened file in FILE. Seek to location OFFSET before 2064 returning. Perform temporary allocations in POOL. */ 2065static svn_error_t * 2066open_and_seek_revision(apr_file_t **file, 2067 svn_fs_t *fs, 2068 svn_revnum_t rev, 2069 apr_off_t offset, 2070 apr_pool_t *pool) 2071{ 2072 apr_file_t *rev_file; 2073 2074 SVN_ERR(ensure_revision_exists(fs, rev, pool)); 2075 2076 SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, pool)); 2077 2078 if (is_packed_rev(fs, rev)) 2079 { 2080 apr_off_t rev_offset; 2081 2082 SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool)); 2083 offset += rev_offset; 2084 } 2085 2086 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool)); 2087 2088 *file = rev_file; 2089 2090 return SVN_NO_ERROR; 2091} 2092 2093/* Open the representation for a node-revision in transaction TXN_ID 2094 in filesystem FS and store the newly opened file in FILE. Seek to 2095 location OFFSET before returning. Perform temporary allocations in 2096 POOL. Only appropriate for file contents, nor props or directory 2097 contents. */ 2098static svn_error_t * 2099open_and_seek_transaction(apr_file_t **file, 2100 svn_fs_t *fs, 2101 const char *txn_id, 2102 representation_t *rep, 2103 apr_pool_t *pool) 2104{ 2105 apr_file_t *rev_file; 2106 apr_off_t offset; 2107 2108 SVN_ERR(svn_io_file_open(&rev_file, path_txn_proto_rev(fs, txn_id, pool), 2109 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); 2110 2111 offset = rep->offset; 2112 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool)); 2113 2114 *file = rev_file; 2115 2116 return SVN_NO_ERROR; 2117} 2118 2119/* Given a node-id ID, and a representation REP in filesystem FS, open 2120 the correct file and seek to the correction location. Store this 2121 file in *FILE_P. Perform any allocations in POOL. */ 2122static svn_error_t * 2123open_and_seek_representation(apr_file_t **file_p, 2124 svn_fs_t *fs, 2125 representation_t *rep, 2126 apr_pool_t *pool) 2127{ 2128 if (! rep->txn_id) 2129 return open_and_seek_revision(file_p, fs, rep->revision, rep->offset, 2130 pool); 2131 else 2132 return open_and_seek_transaction(file_p, fs, rep->txn_id, rep, pool); 2133} 2134 2135/* Parse the description of a representation from STRING and store it 2136 into *REP_P. If the representation is mutable (the revision is 2137 given as -1), then use TXN_ID for the representation's txn_id 2138 field. If MUTABLE_REP_TRUNCATED is true, then this representation 2139 is for property or directory contents, and no information will be 2140 expected except the "-1" revision number for a mutable 2141 representation. Allocate *REP_P in POOL. */ 2142static svn_error_t * 2143read_rep_offsets_body(representation_t **rep_p, 2144 char *string, 2145 const char *txn_id, 2146 svn_boolean_t mutable_rep_truncated, 2147 apr_pool_t *pool) 2148{ 2149 representation_t *rep; 2150 char *str; 2151 apr_int64_t val; 2152 2153 rep = apr_pcalloc(pool, sizeof(*rep)); 2154 *rep_p = rep; 2155 2156 str = svn_cstring_tokenize(" ", &string); 2157 if (str == NULL) 2158 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2159 _("Malformed text representation offset line in node-rev")); 2160 2161 2162 rep->revision = SVN_STR_TO_REV(str); 2163 if (rep->revision == SVN_INVALID_REVNUM) 2164 { 2165 rep->txn_id = txn_id; 2166 if (mutable_rep_truncated) 2167 return SVN_NO_ERROR; 2168 } 2169 2170 str = svn_cstring_tokenize(" ", &string); 2171 if (str == NULL) 2172 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2173 _("Malformed text representation offset line in node-rev")); 2174 2175 SVN_ERR(svn_cstring_atoi64(&val, str)); 2176 rep->offset = (apr_off_t)val; 2177 2178 str = svn_cstring_tokenize(" ", &string); 2179 if (str == NULL) 2180 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2181 _("Malformed text representation offset line in node-rev")); 2182 2183 SVN_ERR(svn_cstring_atoi64(&val, str)); 2184 rep->size = (svn_filesize_t)val; 2185 2186 str = svn_cstring_tokenize(" ", &string); 2187 if (str == NULL) 2188 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2189 _("Malformed text representation offset line in node-rev")); 2190 2191 SVN_ERR(svn_cstring_atoi64(&val, str)); 2192 rep->expanded_size = (svn_filesize_t)val; 2193 2194 /* Read in the MD5 hash. */ 2195 str = svn_cstring_tokenize(" ", &string); 2196 if ((str == NULL) || (strlen(str) != (APR_MD5_DIGESTSIZE * 2))) 2197 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2198 _("Malformed text representation offset line in node-rev")); 2199 2200 SVN_ERR(svn_checksum_parse_hex(&rep->md5_checksum, svn_checksum_md5, str, 2201 pool)); 2202 2203 /* The remaining fields are only used for formats >= 4, so check that. */ 2204 str = svn_cstring_tokenize(" ", &string); 2205 if (str == NULL) 2206 return SVN_NO_ERROR; 2207 2208 /* Read the SHA1 hash. */ 2209 if (strlen(str) != (APR_SHA1_DIGESTSIZE * 2)) 2210 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2211 _("Malformed text representation offset line in node-rev")); 2212 2213 SVN_ERR(svn_checksum_parse_hex(&rep->sha1_checksum, svn_checksum_sha1, str, 2214 pool)); 2215 2216 /* Read the uniquifier. */ 2217 str = svn_cstring_tokenize(" ", &string); 2218 if (str == NULL) 2219 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2220 _("Malformed text representation offset line in node-rev")); 2221 2222 rep->uniquifier = apr_pstrdup(pool, str); 2223 2224 return SVN_NO_ERROR; 2225} 2226 2227/* Wrap read_rep_offsets_body(), extracting its TXN_ID from our NODEREV_ID, 2228 and adding an error message. */ 2229static svn_error_t * 2230read_rep_offsets(representation_t **rep_p, 2231 char *string, 2232 const svn_fs_id_t *noderev_id, 2233 svn_boolean_t mutable_rep_truncated, 2234 apr_pool_t *pool) 2235{ 2236 svn_error_t *err; 2237 const char *txn_id; 2238 2239 if (noderev_id) 2240 txn_id = svn_fs_fs__id_txn_id(noderev_id); 2241 else 2242 txn_id = NULL; 2243 2244 err = read_rep_offsets_body(rep_p, string, txn_id, mutable_rep_truncated, 2245 pool); 2246 if (err) 2247 { 2248 const svn_string_t *id_unparsed = svn_fs_fs__id_unparse(noderev_id, pool); 2249 const char *where; 2250 where = apr_psprintf(pool, 2251 _("While reading representation offsets " 2252 "for node-revision '%s':"), 2253 noderev_id ? id_unparsed->data : "(null)"); 2254 2255 return svn_error_quick_wrap(err, where); 2256 } 2257 else 2258 return SVN_NO_ERROR; 2259} 2260 2261static svn_error_t * 2262err_dangling_id(svn_fs_t *fs, const svn_fs_id_t *id) 2263{ 2264 svn_string_t *id_str = svn_fs_fs__id_unparse(id, fs->pool); 2265 return svn_error_createf 2266 (SVN_ERR_FS_ID_NOT_FOUND, 0, 2267 _("Reference to non-existent node '%s' in filesystem '%s'"), 2268 id_str->data, fs->path); 2269} 2270 2271/* Look up the NODEREV_P for ID in FS' node revsion cache. If noderev 2272 * caching has been enabled and the data can be found, IS_CACHED will 2273 * be set to TRUE. The noderev will be allocated from POOL. 2274 * 2275 * Non-permanent ids (e.g. ids within a TXN) will not be cached. 2276 */ 2277static svn_error_t * 2278get_cached_node_revision_body(node_revision_t **noderev_p, 2279 svn_fs_t *fs, 2280 const svn_fs_id_t *id, 2281 svn_boolean_t *is_cached, 2282 apr_pool_t *pool) 2283{ 2284 fs_fs_data_t *ffd = fs->fsap_data; 2285 if (! ffd->node_revision_cache || svn_fs_fs__id_txn_id(id)) 2286 { 2287 *is_cached = FALSE; 2288 } 2289 else 2290 { 2291 pair_cache_key_t key = { 0 }; 2292 2293 key.revision = svn_fs_fs__id_rev(id); 2294 key.second = svn_fs_fs__id_offset(id); 2295 SVN_ERR(svn_cache__get((void **) noderev_p, 2296 is_cached, 2297 ffd->node_revision_cache, 2298 &key, 2299 pool)); 2300 } 2301 2302 return SVN_NO_ERROR; 2303} 2304 2305/* If noderev caching has been enabled, store the NODEREV_P for the given ID 2306 * in FS' node revsion cache. SCRATCH_POOL is used for temporary allcations. 2307 * 2308 * Non-permanent ids (e.g. ids within a TXN) will not be cached. 2309 */ 2310static svn_error_t * 2311set_cached_node_revision_body(node_revision_t *noderev_p, 2312 svn_fs_t *fs, 2313 const svn_fs_id_t *id, 2314 apr_pool_t *scratch_pool) 2315{ 2316 fs_fs_data_t *ffd = fs->fsap_data; 2317 2318 if (ffd->node_revision_cache && !svn_fs_fs__id_txn_id(id)) 2319 { 2320 pair_cache_key_t key = { 0 }; 2321 2322 key.revision = svn_fs_fs__id_rev(id); 2323 key.second = svn_fs_fs__id_offset(id); 2324 return svn_cache__set(ffd->node_revision_cache, 2325 &key, 2326 noderev_p, 2327 scratch_pool); 2328 } 2329 2330 return SVN_NO_ERROR; 2331} 2332 2333/* Get the node-revision for the node ID in FS. 2334 Set *NODEREV_P to the new node-revision structure, allocated in POOL. 2335 See svn_fs_fs__get_node_revision, which wraps this and adds another 2336 error. */ 2337static svn_error_t * 2338get_node_revision_body(node_revision_t **noderev_p, 2339 svn_fs_t *fs, 2340 const svn_fs_id_t *id, 2341 apr_pool_t *pool) 2342{ 2343 apr_file_t *revision_file; 2344 svn_error_t *err; 2345 svn_boolean_t is_cached = FALSE; 2346 2347 /* First, try a cache lookup. If that succeeds, we are done here. */ 2348 SVN_ERR(get_cached_node_revision_body(noderev_p, fs, id, &is_cached, pool)); 2349 if (is_cached) 2350 return SVN_NO_ERROR; 2351 2352 if (svn_fs_fs__id_txn_id(id)) 2353 { 2354 /* This is a transaction node-rev. */ 2355 err = svn_io_file_open(&revision_file, path_txn_node_rev(fs, id, pool), 2356 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool); 2357 } 2358 else 2359 { 2360 /* This is a revision node-rev. */ 2361 err = open_and_seek_revision(&revision_file, fs, 2362 svn_fs_fs__id_rev(id), 2363 svn_fs_fs__id_offset(id), 2364 pool); 2365 } 2366 2367 if (err) 2368 { 2369 if (APR_STATUS_IS_ENOENT(err->apr_err)) 2370 { 2371 svn_error_clear(err); 2372 return svn_error_trace(err_dangling_id(fs, id)); 2373 } 2374 2375 return svn_error_trace(err); 2376 } 2377 2378 SVN_ERR(svn_fs_fs__read_noderev(noderev_p, 2379 svn_stream_from_aprfile2(revision_file, FALSE, 2380 pool), 2381 pool)); 2382 2383 /* The noderev is not in cache, yet. Add it, if caching has been enabled. */ 2384 return set_cached_node_revision_body(*noderev_p, fs, id, pool); 2385} 2386 2387svn_error_t * 2388svn_fs_fs__read_noderev(node_revision_t **noderev_p, 2389 svn_stream_t *stream, 2390 apr_pool_t *pool) 2391{ 2392 apr_hash_t *headers; 2393 node_revision_t *noderev; 2394 char *value; 2395 const char *noderev_id; 2396 2397 SVN_ERR(read_header_block(&headers, stream, pool)); 2398 2399 noderev = apr_pcalloc(pool, sizeof(*noderev)); 2400 2401 /* Read the node-rev id. */ 2402 value = svn_hash_gets(headers, HEADER_ID); 2403 if (value == NULL) 2404 /* ### More information: filename/offset coordinates */ 2405 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 2406 _("Missing id field in node-rev")); 2407 2408 SVN_ERR(svn_stream_close(stream)); 2409 2410 noderev->id = svn_fs_fs__id_parse(value, strlen(value), pool); 2411 noderev_id = value; /* for error messages later */ 2412 2413 /* Read the type. */ 2414 value = svn_hash_gets(headers, HEADER_TYPE); 2415 2416 if ((value == NULL) || 2417 (strcmp(value, KIND_FILE) != 0 && strcmp(value, KIND_DIR))) 2418 /* ### s/kind/type/ */ 2419 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2420 _("Missing kind field in node-rev '%s'"), 2421 noderev_id); 2422 2423 noderev->kind = (strcmp(value, KIND_FILE) == 0) ? svn_node_file 2424 : svn_node_dir; 2425 2426 /* Read the 'count' field. */ 2427 value = svn_hash_gets(headers, HEADER_COUNT); 2428 if (value) 2429 SVN_ERR(svn_cstring_atoi(&noderev->predecessor_count, value)); 2430 else 2431 noderev->predecessor_count = 0; 2432 2433 /* Get the properties location. */ 2434 value = svn_hash_gets(headers, HEADER_PROPS); 2435 if (value) 2436 { 2437 SVN_ERR(read_rep_offsets(&noderev->prop_rep, value, 2438 noderev->id, TRUE, pool)); 2439 } 2440 2441 /* Get the data location. */ 2442 value = svn_hash_gets(headers, HEADER_TEXT); 2443 if (value) 2444 { 2445 SVN_ERR(read_rep_offsets(&noderev->data_rep, value, 2446 noderev->id, 2447 (noderev->kind == svn_node_dir), pool)); 2448 } 2449 2450 /* Get the created path. */ 2451 value = svn_hash_gets(headers, HEADER_CPATH); 2452 if (value == NULL) 2453 { 2454 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2455 _("Missing cpath field in node-rev '%s'"), 2456 noderev_id); 2457 } 2458 else 2459 { 2460 noderev->created_path = apr_pstrdup(pool, value); 2461 } 2462 2463 /* Get the predecessor ID. */ 2464 value = svn_hash_gets(headers, HEADER_PRED); 2465 if (value) 2466 noderev->predecessor_id = svn_fs_fs__id_parse(value, strlen(value), 2467 pool); 2468 2469 /* Get the copyroot. */ 2470 value = svn_hash_gets(headers, HEADER_COPYROOT); 2471 if (value == NULL) 2472 { 2473 noderev->copyroot_path = apr_pstrdup(pool, noderev->created_path); 2474 noderev->copyroot_rev = svn_fs_fs__id_rev(noderev->id); 2475 } 2476 else 2477 { 2478 char *str; 2479 2480 str = svn_cstring_tokenize(" ", &value); 2481 if (str == NULL) 2482 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2483 _("Malformed copyroot line in node-rev '%s'"), 2484 noderev_id); 2485 2486 noderev->copyroot_rev = SVN_STR_TO_REV(str); 2487 2488 if (*value == '\0') 2489 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2490 _("Malformed copyroot line in node-rev '%s'"), 2491 noderev_id); 2492 noderev->copyroot_path = apr_pstrdup(pool, value); 2493 } 2494 2495 /* Get the copyfrom. */ 2496 value = svn_hash_gets(headers, HEADER_COPYFROM); 2497 if (value == NULL) 2498 { 2499 noderev->copyfrom_path = NULL; 2500 noderev->copyfrom_rev = SVN_INVALID_REVNUM; 2501 } 2502 else 2503 { 2504 char *str = svn_cstring_tokenize(" ", &value); 2505 if (str == NULL) 2506 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2507 _("Malformed copyfrom line in node-rev '%s'"), 2508 noderev_id); 2509 2510 noderev->copyfrom_rev = SVN_STR_TO_REV(str); 2511 2512 if (*value == 0) 2513 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2514 _("Malformed copyfrom line in node-rev '%s'"), 2515 noderev_id); 2516 noderev->copyfrom_path = apr_pstrdup(pool, value); 2517 } 2518 2519 /* Get whether this is a fresh txn root. */ 2520 value = svn_hash_gets(headers, HEADER_FRESHTXNRT); 2521 noderev->is_fresh_txn_root = (value != NULL); 2522 2523 /* Get the mergeinfo count. */ 2524 value = svn_hash_gets(headers, HEADER_MINFO_CNT); 2525 if (value) 2526 SVN_ERR(svn_cstring_atoi64(&noderev->mergeinfo_count, value)); 2527 else 2528 noderev->mergeinfo_count = 0; 2529 2530 /* Get whether *this* node has mergeinfo. */ 2531 value = svn_hash_gets(headers, HEADER_MINFO_HERE); 2532 noderev->has_mergeinfo = (value != NULL); 2533 2534 *noderev_p = noderev; 2535 2536 return SVN_NO_ERROR; 2537} 2538 2539svn_error_t * 2540svn_fs_fs__get_node_revision(node_revision_t **noderev_p, 2541 svn_fs_t *fs, 2542 const svn_fs_id_t *id, 2543 apr_pool_t *pool) 2544{ 2545 svn_error_t *err = get_node_revision_body(noderev_p, fs, id, pool); 2546 if (err && err->apr_err == SVN_ERR_FS_CORRUPT) 2547 { 2548 svn_string_t *id_string = svn_fs_fs__id_unparse(id, pool); 2549 return svn_error_createf(SVN_ERR_FS_CORRUPT, err, 2550 "Corrupt node-revision '%s'", 2551 id_string->data); 2552 } 2553 return svn_error_trace(err); 2554} 2555 2556 2557/* Return a formatted string, compatible with filesystem format FORMAT, 2558 that represents the location of representation REP. If 2559 MUTABLE_REP_TRUNCATED is given, the rep is for props or dir contents, 2560 and only a "-1" revision number will be given for a mutable rep. 2561 If MAY_BE_CORRUPT is true, guard for NULL when constructing the string. 2562 Perform the allocation from POOL. */ 2563static const char * 2564representation_string(representation_t *rep, 2565 int format, 2566 svn_boolean_t mutable_rep_truncated, 2567 svn_boolean_t may_be_corrupt, 2568 apr_pool_t *pool) 2569{ 2570 if (rep->txn_id && mutable_rep_truncated) 2571 return "-1"; 2572 2573#define DISPLAY_MAYBE_NULL_CHECKSUM(checksum) \ 2574 ((!may_be_corrupt || (checksum) != NULL) \ 2575 ? svn_checksum_to_cstring_display((checksum), pool) \ 2576 : "(null)") 2577 2578 if (format < SVN_FS_FS__MIN_REP_SHARING_FORMAT || rep->sha1_checksum == NULL) 2579 return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT 2580 " %" SVN_FILESIZE_T_FMT " %s", 2581 rep->revision, rep->offset, rep->size, 2582 rep->expanded_size, 2583 DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum)); 2584 2585 return apr_psprintf(pool, "%ld %" APR_OFF_T_FMT " %" SVN_FILESIZE_T_FMT 2586 " %" SVN_FILESIZE_T_FMT " %s %s %s", 2587 rep->revision, rep->offset, rep->size, 2588 rep->expanded_size, 2589 DISPLAY_MAYBE_NULL_CHECKSUM(rep->md5_checksum), 2590 DISPLAY_MAYBE_NULL_CHECKSUM(rep->sha1_checksum), 2591 rep->uniquifier); 2592 2593#undef DISPLAY_MAYBE_NULL_CHECKSUM 2594 2595} 2596 2597 2598svn_error_t * 2599svn_fs_fs__write_noderev(svn_stream_t *outfile, 2600 node_revision_t *noderev, 2601 int format, 2602 svn_boolean_t include_mergeinfo, 2603 apr_pool_t *pool) 2604{ 2605 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_ID ": %s\n", 2606 svn_fs_fs__id_unparse(noderev->id, 2607 pool)->data)); 2608 2609 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_TYPE ": %s\n", 2610 (noderev->kind == svn_node_file) ? 2611 KIND_FILE : KIND_DIR)); 2612 2613 if (noderev->predecessor_id) 2614 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PRED ": %s\n", 2615 svn_fs_fs__id_unparse(noderev->predecessor_id, 2616 pool)->data)); 2617 2618 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COUNT ": %d\n", 2619 noderev->predecessor_count)); 2620 2621 if (noderev->data_rep) 2622 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_TEXT ": %s\n", 2623 representation_string(noderev->data_rep, 2624 format, 2625 (noderev->kind 2626 == svn_node_dir), 2627 FALSE, 2628 pool))); 2629 2630 if (noderev->prop_rep) 2631 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_PROPS ": %s\n", 2632 representation_string(noderev->prop_rep, format, 2633 TRUE, FALSE, pool))); 2634 2635 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_CPATH ": %s\n", 2636 noderev->created_path)); 2637 2638 if (noderev->copyfrom_path) 2639 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COPYFROM ": %ld" 2640 " %s\n", 2641 noderev->copyfrom_rev, 2642 noderev->copyfrom_path)); 2643 2644 if ((noderev->copyroot_rev != svn_fs_fs__id_rev(noderev->id)) || 2645 (strcmp(noderev->copyroot_path, noderev->created_path) != 0)) 2646 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_COPYROOT ": %ld" 2647 " %s\n", 2648 noderev->copyroot_rev, 2649 noderev->copyroot_path)); 2650 2651 if (noderev->is_fresh_txn_root) 2652 SVN_ERR(svn_stream_puts(outfile, HEADER_FRESHTXNRT ": y\n")); 2653 2654 if (include_mergeinfo) 2655 { 2656 if (noderev->mergeinfo_count > 0) 2657 SVN_ERR(svn_stream_printf(outfile, pool, HEADER_MINFO_CNT ": %" 2658 APR_INT64_T_FMT "\n", 2659 noderev->mergeinfo_count)); 2660 2661 if (noderev->has_mergeinfo) 2662 SVN_ERR(svn_stream_puts(outfile, HEADER_MINFO_HERE ": y\n")); 2663 } 2664 2665 return svn_stream_puts(outfile, "\n"); 2666} 2667 2668svn_error_t * 2669svn_fs_fs__put_node_revision(svn_fs_t *fs, 2670 const svn_fs_id_t *id, 2671 node_revision_t *noderev, 2672 svn_boolean_t fresh_txn_root, 2673 apr_pool_t *pool) 2674{ 2675 fs_fs_data_t *ffd = fs->fsap_data; 2676 apr_file_t *noderev_file; 2677 const char *txn_id = svn_fs_fs__id_txn_id(id); 2678 2679 noderev->is_fresh_txn_root = fresh_txn_root; 2680 2681 if (! txn_id) 2682 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2683 _("Attempted to write to non-transaction '%s'"), 2684 svn_fs_fs__id_unparse(id, pool)->data); 2685 2686 SVN_ERR(svn_io_file_open(&noderev_file, path_txn_node_rev(fs, id, pool), 2687 APR_WRITE | APR_CREATE | APR_TRUNCATE 2688 | APR_BUFFERED, APR_OS_DEFAULT, pool)); 2689 2690 SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(noderev_file, TRUE, 2691 pool), 2692 noderev, ffd->format, 2693 svn_fs_fs__fs_supports_mergeinfo(fs), 2694 pool)); 2695 2696 SVN_ERR(svn_io_file_close(noderev_file, pool)); 2697 2698 return SVN_NO_ERROR; 2699} 2700 2701/* For the in-transaction NODEREV within FS, write the sha1->rep mapping 2702 * file in the respective transaction, if rep sharing has been enabled etc. 2703 * Use POOL for temporary allocations. 2704 */ 2705static svn_error_t * 2706store_sha1_rep_mapping(svn_fs_t *fs, 2707 node_revision_t *noderev, 2708 apr_pool_t *pool) 2709{ 2710 fs_fs_data_t *ffd = fs->fsap_data; 2711 2712 /* if rep sharing has been enabled and the noderev has a data rep and 2713 * its SHA-1 is known, store the rep struct under its SHA1. */ 2714 if ( ffd->rep_sharing_allowed 2715 && noderev->data_rep 2716 && noderev->data_rep->sha1_checksum) 2717 { 2718 apr_file_t *rep_file; 2719 const char *file_name = path_txn_sha1(fs, 2720 svn_fs_fs__id_txn_id(noderev->id), 2721 noderev->data_rep->sha1_checksum, 2722 pool); 2723 const char *rep_string = representation_string(noderev->data_rep, 2724 ffd->format, 2725 (noderev->kind 2726 == svn_node_dir), 2727 FALSE, 2728 pool); 2729 SVN_ERR(svn_io_file_open(&rep_file, file_name, 2730 APR_WRITE | APR_CREATE | APR_TRUNCATE 2731 | APR_BUFFERED, APR_OS_DEFAULT, pool)); 2732 2733 SVN_ERR(svn_io_file_write_full(rep_file, rep_string, 2734 strlen(rep_string), NULL, pool)); 2735 2736 SVN_ERR(svn_io_file_close(rep_file, pool)); 2737 } 2738 2739 return SVN_NO_ERROR; 2740} 2741 2742 2743/* This structure is used to hold the information associated with a 2744 REP line. */ 2745struct rep_args 2746{ 2747 svn_boolean_t is_delta; 2748 svn_boolean_t is_delta_vs_empty; 2749 2750 svn_revnum_t base_revision; 2751 apr_off_t base_offset; 2752 svn_filesize_t base_length; 2753}; 2754 2755/* Read the next line from file FILE and parse it as a text 2756 representation entry. Return the parsed entry in *REP_ARGS_P. 2757 Perform all allocations in POOL. */ 2758static svn_error_t * 2759read_rep_line(struct rep_args **rep_args_p, 2760 apr_file_t *file, 2761 apr_pool_t *pool) 2762{ 2763 char buffer[160]; 2764 apr_size_t limit; 2765 struct rep_args *rep_args; 2766 char *str, *last_str = buffer; 2767 apr_int64_t val; 2768 2769 limit = sizeof(buffer); 2770 SVN_ERR(svn_io_read_length_line(file, buffer, &limit, pool)); 2771 2772 rep_args = apr_pcalloc(pool, sizeof(*rep_args)); 2773 rep_args->is_delta = FALSE; 2774 2775 if (strcmp(buffer, REP_PLAIN) == 0) 2776 { 2777 *rep_args_p = rep_args; 2778 return SVN_NO_ERROR; 2779 } 2780 2781 if (strcmp(buffer, REP_DELTA) == 0) 2782 { 2783 /* This is a delta against the empty stream. */ 2784 rep_args->is_delta = TRUE; 2785 rep_args->is_delta_vs_empty = TRUE; 2786 *rep_args_p = rep_args; 2787 return SVN_NO_ERROR; 2788 } 2789 2790 rep_args->is_delta = TRUE; 2791 rep_args->is_delta_vs_empty = FALSE; 2792 2793 /* We have hopefully a DELTA vs. a non-empty base revision. */ 2794 str = svn_cstring_tokenize(" ", &last_str); 2795 if (! str || (strcmp(str, REP_DELTA) != 0)) 2796 goto error; 2797 2798 str = svn_cstring_tokenize(" ", &last_str); 2799 if (! str) 2800 goto error; 2801 rep_args->base_revision = SVN_STR_TO_REV(str); 2802 2803 str = svn_cstring_tokenize(" ", &last_str); 2804 if (! str) 2805 goto error; 2806 SVN_ERR(svn_cstring_atoi64(&val, str)); 2807 rep_args->base_offset = (apr_off_t)val; 2808 2809 str = svn_cstring_tokenize(" ", &last_str); 2810 if (! str) 2811 goto error; 2812 SVN_ERR(svn_cstring_atoi64(&val, str)); 2813 rep_args->base_length = (svn_filesize_t)val; 2814 2815 *rep_args_p = rep_args; 2816 return SVN_NO_ERROR; 2817 2818 error: 2819 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2820 _("Malformed representation header at %s"), 2821 path_and_offset_of(file, pool)); 2822} 2823 2824/* Given a revision file REV_FILE, opened to REV in FS, find the Node-ID 2825 of the header located at OFFSET and store it in *ID_P. Allocate 2826 temporary variables from POOL. */ 2827static svn_error_t * 2828get_fs_id_at_offset(svn_fs_id_t **id_p, 2829 apr_file_t *rev_file, 2830 svn_fs_t *fs, 2831 svn_revnum_t rev, 2832 apr_off_t offset, 2833 apr_pool_t *pool) 2834{ 2835 svn_fs_id_t *id; 2836 apr_hash_t *headers; 2837 const char *node_id_str; 2838 2839 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool)); 2840 2841 SVN_ERR(read_header_block(&headers, 2842 svn_stream_from_aprfile2(rev_file, TRUE, pool), 2843 pool)); 2844 2845 /* In error messages, the offset is relative to the pack file, 2846 not to the rev file. */ 2847 2848 node_id_str = svn_hash_gets(headers, HEADER_ID); 2849 2850 if (node_id_str == NULL) 2851 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2852 _("Missing node-id in node-rev at r%ld " 2853 "(offset %s)"), 2854 rev, 2855 apr_psprintf(pool, "%" APR_OFF_T_FMT, offset)); 2856 2857 id = svn_fs_fs__id_parse(node_id_str, strlen(node_id_str), pool); 2858 2859 if (id == NULL) 2860 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2861 _("Corrupt node-id '%s' in node-rev at r%ld " 2862 "(offset %s)"), 2863 node_id_str, rev, 2864 apr_psprintf(pool, "%" APR_OFF_T_FMT, offset)); 2865 2866 *id_p = id; 2867 2868 /* ### assert that the txn_id is REV/OFFSET ? */ 2869 2870 return SVN_NO_ERROR; 2871} 2872 2873 2874/* Given an open revision file REV_FILE in FS for REV, locate the trailer that 2875 specifies the offset to the root node-id and to the changed path 2876 information. Store the root node offset in *ROOT_OFFSET and the 2877 changed path offset in *CHANGES_OFFSET. If either of these 2878 pointers is NULL, do nothing with it. 2879 2880 If PACKED is true, REV_FILE should be a packed shard file. 2881 ### There is currently no such parameter. This function assumes that 2882 is_packed_rev(FS, REV) will indicate whether REV_FILE is a packed 2883 file. Therefore FS->fsap_data->min_unpacked_rev must not have been 2884 refreshed since REV_FILE was opened if there is a possibility that 2885 revision REV may have become packed since then. 2886 TODO: Take an IS_PACKED parameter instead, in order to remove this 2887 requirement. 2888 2889 Allocate temporary variables from POOL. */ 2890static svn_error_t * 2891get_root_changes_offset(apr_off_t *root_offset, 2892 apr_off_t *changes_offset, 2893 apr_file_t *rev_file, 2894 svn_fs_t *fs, 2895 svn_revnum_t rev, 2896 apr_pool_t *pool) 2897{ 2898 fs_fs_data_t *ffd = fs->fsap_data; 2899 apr_off_t offset; 2900 apr_off_t rev_offset; 2901 char buf[64]; 2902 int i, num_bytes; 2903 const char *str; 2904 apr_size_t len; 2905 apr_seek_where_t seek_relative; 2906 2907 /* Determine where to seek to in the file. 2908 2909 If we've got a pack file, we want to seek to the end of the desired 2910 revision. But we don't track that, so we seek to the beginning of the 2911 next revision. 2912 2913 Unless the next revision is in a different file, in which case, we can 2914 just seek to the end of the pack file -- just like we do in the 2915 non-packed case. */ 2916 if (is_packed_rev(fs, rev) && ((rev + 1) % ffd->max_files_per_dir != 0)) 2917 { 2918 SVN_ERR(get_packed_offset(&offset, fs, rev + 1, pool)); 2919 seek_relative = APR_SET; 2920 } 2921 else 2922 { 2923 seek_relative = APR_END; 2924 offset = 0; 2925 } 2926 2927 /* Offset of the revision from the start of the pack file, if applicable. */ 2928 if (is_packed_rev(fs, rev)) 2929 SVN_ERR(get_packed_offset(&rev_offset, fs, rev, pool)); 2930 else 2931 rev_offset = 0; 2932 2933 /* We will assume that the last line containing the two offsets 2934 will never be longer than 64 characters. */ 2935 SVN_ERR(svn_io_file_seek(rev_file, seek_relative, &offset, pool)); 2936 2937 offset -= sizeof(buf); 2938 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool)); 2939 2940 /* Read in this last block, from which we will identify the last line. */ 2941 len = sizeof(buf); 2942 SVN_ERR(svn_io_file_read(rev_file, buf, &len, pool)); 2943 2944 /* This cast should be safe since the maximum amount read, 64, will 2945 never be bigger than the size of an int. */ 2946 num_bytes = (int) len; 2947 2948 /* The last byte should be a newline. */ 2949 if (buf[num_bytes - 1] != '\n') 2950 { 2951 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2952 _("Revision file (r%ld) lacks trailing newline"), 2953 rev); 2954 } 2955 2956 /* Look for the next previous newline. */ 2957 for (i = num_bytes - 2; i >= 0; i--) 2958 { 2959 if (buf[i] == '\n') 2960 break; 2961 } 2962 2963 if (i < 0) 2964 { 2965 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2966 _("Final line in revision file (r%ld) longer " 2967 "than 64 characters"), 2968 rev); 2969 } 2970 2971 i++; 2972 str = &buf[i]; 2973 2974 /* find the next space */ 2975 for ( ; i < (num_bytes - 2) ; i++) 2976 if (buf[i] == ' ') 2977 break; 2978 2979 if (i == (num_bytes - 2)) 2980 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 2981 _("Final line in revision file r%ld missing space"), 2982 rev); 2983 2984 if (root_offset) 2985 { 2986 apr_int64_t val; 2987 2988 buf[i] = '\0'; 2989 SVN_ERR(svn_cstring_atoi64(&val, str)); 2990 *root_offset = rev_offset + (apr_off_t)val; 2991 } 2992 2993 i++; 2994 str = &buf[i]; 2995 2996 /* find the next newline */ 2997 for ( ; i < num_bytes; i++) 2998 if (buf[i] == '\n') 2999 break; 3000 3001 if (changes_offset) 3002 { 3003 apr_int64_t val; 3004 3005 buf[i] = '\0'; 3006 SVN_ERR(svn_cstring_atoi64(&val, str)); 3007 *changes_offset = rev_offset + (apr_off_t)val; 3008 } 3009 3010 return SVN_NO_ERROR; 3011} 3012 3013/* Move a file into place from OLD_FILENAME in the transactions 3014 directory to its final location NEW_FILENAME in the repository. On 3015 Unix, match the permissions of the new file to the permissions of 3016 PERMS_REFERENCE. Temporary allocations are from POOL. 3017 3018 This function almost duplicates svn_io_file_move(), but it tries to 3019 guarantee a flush. */ 3020static svn_error_t * 3021move_into_place(const char *old_filename, 3022 const char *new_filename, 3023 const char *perms_reference, 3024 apr_pool_t *pool) 3025{ 3026 svn_error_t *err; 3027 3028 SVN_ERR(svn_io_copy_perms(perms_reference, old_filename, pool)); 3029 3030 /* Move the file into place. */ 3031 err = svn_io_file_rename(old_filename, new_filename, pool); 3032 if (err && APR_STATUS_IS_EXDEV(err->apr_err)) 3033 { 3034 apr_file_t *file; 3035 3036 /* Can't rename across devices; fall back to copying. */ 3037 svn_error_clear(err); 3038 err = SVN_NO_ERROR; 3039 SVN_ERR(svn_io_copy_file(old_filename, new_filename, TRUE, pool)); 3040 3041 /* Flush the target of the copy to disk. */ 3042 SVN_ERR(svn_io_file_open(&file, new_filename, APR_READ, 3043 APR_OS_DEFAULT, pool)); 3044 /* ### BH: Does this really guarantee a flush of the data written 3045 ### via a completely different handle on all operating systems? 3046 ### 3047 ### Maybe we should perform the copy ourselves instead of making 3048 ### apr do that and flush the real handle? */ 3049 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 3050 SVN_ERR(svn_io_file_close(file, pool)); 3051 } 3052 if (err) 3053 return svn_error_trace(err); 3054 3055#ifdef __linux__ 3056 { 3057 /* Linux has the unusual feature that fsync() on a file is not 3058 enough to ensure that a file's directory entries have been 3059 flushed to disk; you have to fsync the directory as well. 3060 On other operating systems, we'd only be asking for trouble 3061 by trying to open and fsync a directory. */ 3062 const char *dirname; 3063 apr_file_t *file; 3064 3065 dirname = svn_dirent_dirname(new_filename, pool); 3066 SVN_ERR(svn_io_file_open(&file, dirname, APR_READ, APR_OS_DEFAULT, 3067 pool)); 3068 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 3069 SVN_ERR(svn_io_file_close(file, pool)); 3070 } 3071#endif 3072 3073 return SVN_NO_ERROR; 3074} 3075 3076svn_error_t * 3077svn_fs_fs__rev_get_root(svn_fs_id_t **root_id_p, 3078 svn_fs_t *fs, 3079 svn_revnum_t rev, 3080 apr_pool_t *pool) 3081{ 3082 fs_fs_data_t *ffd = fs->fsap_data; 3083 apr_file_t *revision_file; 3084 apr_off_t root_offset; 3085 svn_fs_id_t *root_id = NULL; 3086 svn_boolean_t is_cached; 3087 3088 SVN_ERR(ensure_revision_exists(fs, rev, pool)); 3089 3090 SVN_ERR(svn_cache__get((void **) root_id_p, &is_cached, 3091 ffd->rev_root_id_cache, &rev, pool)); 3092 if (is_cached) 3093 return SVN_NO_ERROR; 3094 3095 SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool)); 3096 SVN_ERR(get_root_changes_offset(&root_offset, NULL, revision_file, fs, rev, 3097 pool)); 3098 3099 SVN_ERR(get_fs_id_at_offset(&root_id, revision_file, fs, rev, 3100 root_offset, pool)); 3101 3102 SVN_ERR(svn_io_file_close(revision_file, pool)); 3103 3104 SVN_ERR(svn_cache__set(ffd->rev_root_id_cache, &rev, root_id, pool)); 3105 3106 *root_id_p = root_id; 3107 3108 return SVN_NO_ERROR; 3109} 3110 3111/* Revprop caching management. 3112 * 3113 * Mechanism: 3114 * ---------- 3115 * 3116 * Revprop caching needs to be activated and will be deactivated for the 3117 * respective FS instance if the necessary infrastructure could not be 3118 * initialized. In deactivated mode, there is almost no runtime overhead 3119 * associated with revprop caching. As long as no revprops are being read 3120 * or changed, revprop caching imposes no overhead. 3121 * 3122 * When activated, we cache revprops using (revision, generation) pairs 3123 * as keys with the generation being incremented upon every revprop change. 3124 * Since the cache is process-local, the generation needs to be tracked 3125 * for at least as long as the process lives but may be reset afterwards. 3126 * 3127 * To track the revprop generation, we use two-layer approach. On the lower 3128 * level, we use named atomics to have a system-wide consistent value for 3129 * the current revprop generation. However, those named atomics will only 3130 * remain valid for as long as at least one process / thread in the system 3131 * accesses revprops in the respective repository. The underlying shared 3132 * memory gets cleaned up afterwards. 3133 * 3134 * On the second level, we will use a persistent file to track the latest 3135 * revprop generation. It will be written upon each revprop change but 3136 * only be read if we are the first process to initialize the named atomics 3137 * with that value. 3138 * 3139 * The overhead for the second and following accesses to revprops is 3140 * almost zero on most systems. 3141 * 3142 * 3143 * Tech aspects: 3144 * ------------- 3145 * 3146 * A problem is that we need to provide a globally available file name to 3147 * back the SHM implementation on OSes that need it. We can only assume 3148 * write access to some file within the respective repositories. Because 3149 * a given server process may access thousands of repositories during its 3150 * lifetime, keeping the SHM data alive for all of them is also not an 3151 * option. 3152 * 3153 * So, we store the new revprop generation on disk as part of each 3154 * setrevprop call, i.e. this write will be serialized and the write order 3155 * be guaranteed by the repository write lock. 3156 * 3157 * The only racy situation occurs when the data is being read again by two 3158 * processes concurrently but in that situation, the first process to 3159 * finish that procedure is guaranteed to be the only one that initializes 3160 * the SHM data. Since even writers will first go through that 3161 * initialization phase, they will never operate on stale data. 3162 */ 3163 3164/* Read revprop generation as stored on disk for repository FS. The result 3165 * is returned in *CURRENT. Default to 2 if no such file is available. 3166 */ 3167static svn_error_t * 3168read_revprop_generation_file(apr_int64_t *current, 3169 svn_fs_t *fs, 3170 apr_pool_t *pool) 3171{ 3172 svn_error_t *err; 3173 apr_file_t *file; 3174 char buf[80]; 3175 apr_size_t len; 3176 const char *path = path_revprop_generation(fs, pool); 3177 3178 err = svn_io_file_open(&file, path, 3179 APR_READ | APR_BUFFERED, 3180 APR_OS_DEFAULT, pool); 3181 if (err && APR_STATUS_IS_ENOENT(err->apr_err)) 3182 { 3183 svn_error_clear(err); 3184 *current = 2; 3185 3186 return SVN_NO_ERROR; 3187 } 3188 SVN_ERR(err); 3189 3190 len = sizeof(buf); 3191 SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); 3192 3193 /* Check that the first line contains only digits. */ 3194 SVN_ERR(check_file_buffer_numeric(buf, 0, path, 3195 "Revprop Generation", pool)); 3196 SVN_ERR(svn_cstring_atoi64(current, buf)); 3197 3198 return svn_io_file_close(file, pool); 3199} 3200 3201/* Write the CURRENT revprop generation to disk for repository FS. 3202 */ 3203static svn_error_t * 3204write_revprop_generation_file(svn_fs_t *fs, 3205 apr_int64_t current, 3206 apr_pool_t *pool) 3207{ 3208 apr_file_t *file; 3209 const char *tmp_path; 3210 3211 char buf[SVN_INT64_BUFFER_SIZE]; 3212 apr_size_t len = svn__i64toa(buf, current); 3213 buf[len] = '\n'; 3214 3215 SVN_ERR(svn_io_open_unique_file3(&file, &tmp_path, fs->path, 3216 svn_io_file_del_none, pool, pool)); 3217 SVN_ERR(svn_io_file_write_full(file, buf, len + 1, NULL, pool)); 3218 SVN_ERR(svn_io_file_close(file, pool)); 3219 3220 return move_into_place(tmp_path, path_revprop_generation(fs, pool), 3221 tmp_path, pool); 3222} 3223 3224/* Make sure the revprop_namespace member in FS is set. */ 3225static svn_error_t * 3226ensure_revprop_namespace(svn_fs_t *fs) 3227{ 3228 fs_fs_data_t *ffd = fs->fsap_data; 3229 3230 return ffd->revprop_namespace == NULL 3231 ? svn_atomic_namespace__create(&ffd->revprop_namespace, 3232 svn_dirent_join(fs->path, 3233 ATOMIC_REVPROP_NAMESPACE, 3234 fs->pool), 3235 fs->pool) 3236 : SVN_NO_ERROR; 3237} 3238 3239/* Make sure the revprop_namespace member in FS is set. */ 3240static svn_error_t * 3241cleanup_revprop_namespace(svn_fs_t *fs) 3242{ 3243 const char *name = svn_dirent_join(fs->path, 3244 ATOMIC_REVPROP_NAMESPACE, 3245 fs->pool); 3246 return svn_error_trace(svn_atomic_namespace__cleanup(name, fs->pool)); 3247} 3248 3249/* Make sure the revprop_generation member in FS is set and, if necessary, 3250 * initialized with the latest value stored on disk. 3251 */ 3252static svn_error_t * 3253ensure_revprop_generation(svn_fs_t *fs, apr_pool_t *pool) 3254{ 3255 fs_fs_data_t *ffd = fs->fsap_data; 3256 3257 SVN_ERR(ensure_revprop_namespace(fs)); 3258 if (ffd->revprop_generation == NULL) 3259 { 3260 apr_int64_t current = 0; 3261 3262 SVN_ERR(svn_named_atomic__get(&ffd->revprop_generation, 3263 ffd->revprop_namespace, 3264 ATOMIC_REVPROP_GENERATION, 3265 TRUE)); 3266 3267 /* If the generation is at 0, we just created a new namespace 3268 * (it would be at least 2 otherwise). Read the latest generation 3269 * from disk and if we are the first one to initialize the atomic 3270 * (i.e. is still 0), set it to the value just gotten. 3271 */ 3272 SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); 3273 if (current == 0) 3274 { 3275 SVN_ERR(read_revprop_generation_file(¤t, fs, pool)); 3276 SVN_ERR(svn_named_atomic__cmpxchg(NULL, current, 0, 3277 ffd->revprop_generation)); 3278 } 3279 } 3280 3281 return SVN_NO_ERROR; 3282} 3283 3284/* Make sure the revprop_timeout member in FS is set. */ 3285static svn_error_t * 3286ensure_revprop_timeout(svn_fs_t *fs) 3287{ 3288 fs_fs_data_t *ffd = fs->fsap_data; 3289 3290 SVN_ERR(ensure_revprop_namespace(fs)); 3291 return ffd->revprop_timeout == NULL 3292 ? svn_named_atomic__get(&ffd->revprop_timeout, 3293 ffd->revprop_namespace, 3294 ATOMIC_REVPROP_TIMEOUT, 3295 TRUE) 3296 : SVN_NO_ERROR; 3297} 3298 3299/* Create an error object with the given MESSAGE and pass it to the 3300 WARNING member of FS. */ 3301static void 3302log_revprop_cache_init_warning(svn_fs_t *fs, 3303 svn_error_t *underlying_err, 3304 const char *message) 3305{ 3306 svn_error_t *err = svn_error_createf(SVN_ERR_FS_REVPROP_CACHE_INIT_FAILURE, 3307 underlying_err, 3308 message, fs->path); 3309 3310 if (fs->warning) 3311 (fs->warning)(fs->warning_baton, err); 3312 3313 svn_error_clear(err); 3314} 3315 3316/* Test whether revprop cache and necessary infrastructure are 3317 available in FS. */ 3318static svn_boolean_t 3319has_revprop_cache(svn_fs_t *fs, apr_pool_t *pool) 3320{ 3321 fs_fs_data_t *ffd = fs->fsap_data; 3322 svn_error_t *error; 3323 3324 /* is the cache (still) enabled? */ 3325 if (ffd->revprop_cache == NULL) 3326 return FALSE; 3327 3328 /* is it efficient? */ 3329 if (!svn_named_atomic__is_efficient()) 3330 { 3331 /* access to it would be quite slow 3332 * -> disable the revprop cache for good 3333 */ 3334 ffd->revprop_cache = NULL; 3335 log_revprop_cache_init_warning(fs, NULL, 3336 "Revprop caching for '%s' disabled" 3337 " because it would be inefficient."); 3338 3339 return FALSE; 3340 } 3341 3342 /* try to access our SHM-backed infrastructure */ 3343 error = ensure_revprop_generation(fs, pool); 3344 if (error) 3345 { 3346 /* failure -> disable revprop cache for good */ 3347 3348 ffd->revprop_cache = NULL; 3349 log_revprop_cache_init_warning(fs, error, 3350 "Revprop caching for '%s' disabled " 3351 "because SHM infrastructure for revprop " 3352 "caching failed to initialize."); 3353 3354 return FALSE; 3355 } 3356 3357 return TRUE; 3358} 3359 3360/* Baton structure for revprop_generation_fixup. */ 3361typedef struct revprop_generation_fixup_t 3362{ 3363 /* revprop generation to read */ 3364 apr_int64_t *generation; 3365 3366 /* containing the revprop_generation member to query */ 3367 fs_fs_data_t *ffd; 3368} revprop_generation_upgrade_t; 3369 3370/* If the revprop generation has an odd value, it means the original writer 3371 of the revprop got killed. We don't know whether that process as able 3372 to change the revprop data but we assume that it was. Therefore, we 3373 increase the generation in that case to basically invalidate everyones 3374 cache content. 3375 Execute this onlx while holding the write lock to the repo in baton->FFD. 3376 */ 3377static svn_error_t * 3378revprop_generation_fixup(void *void_baton, 3379 apr_pool_t *pool) 3380{ 3381 revprop_generation_upgrade_t *baton = void_baton; 3382 assert(baton->ffd->has_write_lock); 3383 3384 /* Maybe, either the original revprop writer or some other reader has 3385 already corrected / bumped the revprop generation. Thus, we need 3386 to read it again. */ 3387 SVN_ERR(svn_named_atomic__read(baton->generation, 3388 baton->ffd->revprop_generation)); 3389 3390 /* Cause everyone to re-read revprops upon their next access, if the 3391 last revprop write did not complete properly. */ 3392 while (*baton->generation % 2) 3393 SVN_ERR(svn_named_atomic__add(baton->generation, 3394 1, 3395 baton->ffd->revprop_generation)); 3396 3397 return SVN_NO_ERROR; 3398} 3399 3400/* Read the current revprop generation and return it in *GENERATION. 3401 Also, detect aborted / crashed writers and recover from that. 3402 Use the access object in FS to set the shared mem values. */ 3403static svn_error_t * 3404read_revprop_generation(apr_int64_t *generation, 3405 svn_fs_t *fs, 3406 apr_pool_t *pool) 3407{ 3408 apr_int64_t current = 0; 3409 fs_fs_data_t *ffd = fs->fsap_data; 3410 3411 /* read the current revprop generation number */ 3412 SVN_ERR(ensure_revprop_generation(fs, pool)); 3413 SVN_ERR(svn_named_atomic__read(¤t, ffd->revprop_generation)); 3414 3415 /* is an unfinished revprop write under the way? */ 3416 if (current % 2) 3417 { 3418 apr_int64_t timeout = 0; 3419 3420 /* read timeout for the write operation */ 3421 SVN_ERR(ensure_revprop_timeout(fs)); 3422 SVN_ERR(svn_named_atomic__read(&timeout, ffd->revprop_timeout)); 3423 3424 /* has the writer process been aborted, 3425 * i.e. has the timeout been reached? 3426 */ 3427 if (apr_time_now() > timeout) 3428 { 3429 revprop_generation_upgrade_t baton; 3430 baton.generation = ¤t; 3431 baton.ffd = ffd; 3432 3433 /* Ensure that the original writer process no longer exists by 3434 * acquiring the write lock to this repository. Then, fix up 3435 * the revprop generation. 3436 */ 3437 if (ffd->has_write_lock) 3438 SVN_ERR(revprop_generation_fixup(&baton, pool)); 3439 else 3440 SVN_ERR(svn_fs_fs__with_write_lock(fs, revprop_generation_fixup, 3441 &baton, pool)); 3442 } 3443 } 3444 3445 /* return the value we just got */ 3446 *generation = current; 3447 return SVN_NO_ERROR; 3448} 3449 3450/* Set the revprop generation to the next odd number to indicate that 3451 there is a revprop write process under way. If that times out, 3452 readers shall recover from that state & re-read revprops. 3453 Use the access object in FS to set the shared mem value. */ 3454static svn_error_t * 3455begin_revprop_change(svn_fs_t *fs, apr_pool_t *pool) 3456{ 3457 apr_int64_t current; 3458 fs_fs_data_t *ffd = fs->fsap_data; 3459 3460 /* set the timeout for the write operation */ 3461 SVN_ERR(ensure_revprop_timeout(fs)); 3462 SVN_ERR(svn_named_atomic__write(NULL, 3463 apr_time_now() + REVPROP_CHANGE_TIMEOUT, 3464 ffd->revprop_timeout)); 3465 3466 /* set the revprop generation to an odd value to indicate 3467 * that a write is in progress 3468 */ 3469 SVN_ERR(ensure_revprop_generation(fs, pool)); 3470 do 3471 { 3472 SVN_ERR(svn_named_atomic__add(¤t, 3473 1, 3474 ffd->revprop_generation)); 3475 } 3476 while (current % 2 == 0); 3477 3478 return SVN_NO_ERROR; 3479} 3480 3481/* Set the revprop generation to the next even number to indicate that 3482 a) readers shall re-read revprops, and 3483 b) the write process has been completed (no recovery required) 3484 Use the access object in FS to set the shared mem value. */ 3485static svn_error_t * 3486end_revprop_change(svn_fs_t *fs, apr_pool_t *pool) 3487{ 3488 apr_int64_t current = 1; 3489 fs_fs_data_t *ffd = fs->fsap_data; 3490 3491 /* set the revprop generation to an even value to indicate 3492 * that a write has been completed 3493 */ 3494 SVN_ERR(ensure_revprop_generation(fs, pool)); 3495 do 3496 { 3497 SVN_ERR(svn_named_atomic__add(¤t, 3498 1, 3499 ffd->revprop_generation)); 3500 } 3501 while (current % 2); 3502 3503 /* Save the latest generation to disk. FS is currently in a "locked" 3504 * state such that we can be sure the be the only ones to write that 3505 * file. 3506 */ 3507 return write_revprop_generation_file(fs, current, pool); 3508} 3509 3510/* Container for all data required to access the packed revprop file 3511 * for a given REVISION. This structure will be filled incrementally 3512 * by read_pack_revprops() its sub-routines. 3513 */ 3514typedef struct packed_revprops_t 3515{ 3516 /* revision number to read (not necessarily the first in the pack) */ 3517 svn_revnum_t revision; 3518 3519 /* current revprop generation. Used when populating the revprop cache */ 3520 apr_int64_t generation; 3521 3522 /* the actual revision properties */ 3523 apr_hash_t *properties; 3524 3525 /* their size when serialized to a single string 3526 * (as found in PACKED_REVPROPS) */ 3527 apr_size_t serialized_size; 3528 3529 3530 /* name of the pack file (without folder path) */ 3531 const char *filename; 3532 3533 /* packed shard folder path */ 3534 const char *folder; 3535 3536 /* sum of values in SIZES */ 3537 apr_size_t total_size; 3538 3539 /* first revision in the pack */ 3540 svn_revnum_t start_revision; 3541 3542 /* size of the revprops in PACKED_REVPROPS */ 3543 apr_array_header_t *sizes; 3544 3545 /* offset of the revprops in PACKED_REVPROPS */ 3546 apr_array_header_t *offsets; 3547 3548 3549 /* concatenation of the serialized representation of all revprops 3550 * in the pack, i.e. the pack content without header and compression */ 3551 svn_stringbuf_t *packed_revprops; 3552 3553 /* content of the manifest. 3554 * Maps long(rev - START_REVISION) to const char* pack file name */ 3555 apr_array_header_t *manifest; 3556} packed_revprops_t; 3557 3558/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. 3559 * Also, put them into the revprop cache, if activated, for future use. 3560 * Three more parameters are being used to update the revprop cache: FS is 3561 * our file system, the revprops belong to REVISION and the global revprop 3562 * GENERATION is used as well. 3563 * 3564 * The returned hash will be allocated in POOL, SCRATCH_POOL is being used 3565 * for temporary allocations. 3566 */ 3567static svn_error_t * 3568parse_revprop(apr_hash_t **properties, 3569 svn_fs_t *fs, 3570 svn_revnum_t revision, 3571 apr_int64_t generation, 3572 svn_string_t *content, 3573 apr_pool_t *pool, 3574 apr_pool_t *scratch_pool) 3575{ 3576 svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); 3577 *properties = apr_hash_make(pool); 3578 3579 SVN_ERR(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool)); 3580 if (has_revprop_cache(fs, pool)) 3581 { 3582 fs_fs_data_t *ffd = fs->fsap_data; 3583 pair_cache_key_t key = { 0 }; 3584 3585 key.revision = revision; 3586 key.second = generation; 3587 SVN_ERR(svn_cache__set(ffd->revprop_cache, &key, *properties, 3588 scratch_pool)); 3589 } 3590 3591 return SVN_NO_ERROR; 3592} 3593 3594/* Read the non-packed revprops for revision REV in FS, put them into the 3595 * revprop cache if activated and return them in *PROPERTIES. GENERATION 3596 * is the current revprop generation. 3597 * 3598 * If the data could not be read due to an otherwise recoverable error, 3599 * leave *PROPERTIES unchanged. No error will be returned in that case. 3600 * 3601 * Allocations will be done in POOL. 3602 */ 3603static svn_error_t * 3604read_non_packed_revprop(apr_hash_t **properties, 3605 svn_fs_t *fs, 3606 svn_revnum_t rev, 3607 apr_int64_t generation, 3608 apr_pool_t *pool) 3609{ 3610 svn_stringbuf_t *content = NULL; 3611 apr_pool_t *iterpool = svn_pool_create(pool); 3612 svn_boolean_t missing = FALSE; 3613 int i; 3614 3615 for (i = 0; i < RECOVERABLE_RETRY_COUNT && !missing && !content; ++i) 3616 { 3617 svn_pool_clear(iterpool); 3618 SVN_ERR(try_stringbuf_from_file(&content, 3619 &missing, 3620 path_revprops(fs, rev, iterpool), 3621 i + 1 < RECOVERABLE_RETRY_COUNT, 3622 iterpool)); 3623 } 3624 3625 if (content) 3626 SVN_ERR(parse_revprop(properties, fs, rev, generation, 3627 svn_stringbuf__morph_into_string(content), 3628 pool, iterpool)); 3629 3630 svn_pool_clear(iterpool); 3631 3632 return SVN_NO_ERROR; 3633} 3634 3635/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST 3636 * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. 3637 */ 3638static svn_error_t * 3639get_revprop_packname(svn_fs_t *fs, 3640 packed_revprops_t *revprops, 3641 apr_pool_t *pool, 3642 apr_pool_t *scratch_pool) 3643{ 3644 fs_fs_data_t *ffd = fs->fsap_data; 3645 svn_stringbuf_t *content = NULL; 3646 const char *manifest_file_path; 3647 int idx; 3648 3649 /* read content of the manifest file */ 3650 revprops->folder = path_revprops_pack_shard(fs, revprops->revision, pool); 3651 manifest_file_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); 3652 3653 SVN_ERR(read_content(&content, manifest_file_path, pool)); 3654 3655 /* parse the manifest. Every line is a file name */ 3656 revprops->manifest = apr_array_make(pool, ffd->max_files_per_dir, 3657 sizeof(const char*)); 3658 while (content->data) 3659 { 3660 APR_ARRAY_PUSH(revprops->manifest, const char*) = content->data; 3661 content->data = strchr(content->data, '\n'); 3662 if (content->data) 3663 { 3664 *content->data = 0; 3665 content->data++; 3666 } 3667 } 3668 3669 /* Index for our revision. Rev 0 is excluded from the first shard. */ 3670 idx = (int)(revprops->revision % ffd->max_files_per_dir); 3671 if (revprops->revision < ffd->max_files_per_dir) 3672 --idx; 3673 3674 if (revprops->manifest->nelts <= idx) 3675 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 3676 _("Packed revprop manifest for rev %ld too " 3677 "small"), revprops->revision); 3678 3679 /* Now get the file name */ 3680 revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); 3681 3682 return SVN_NO_ERROR; 3683} 3684 3685/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, 3686 * fill the START_REVISION, SIZES, OFFSETS members. Also, make 3687 * PACKED_REVPROPS point to the first serialized revprop. 3688 * 3689 * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as 3690 * well as the SERIALIZED_SIZE member. If revprop caching has been 3691 * enabled, parse all revprops in the pack and cache them. 3692 */ 3693static svn_error_t * 3694parse_packed_revprops(svn_fs_t *fs, 3695 packed_revprops_t *revprops, 3696 apr_pool_t *pool, 3697 apr_pool_t *scratch_pool) 3698{ 3699 svn_stream_t *stream; 3700 apr_int64_t first_rev, count, i; 3701 apr_off_t offset; 3702 const char *header_end; 3703 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 3704 3705 /* decompress (even if the data is only "stored", there is still a 3706 * length header to remove) */ 3707 svn_string_t *compressed 3708 = svn_stringbuf__morph_into_string(revprops->packed_revprops); 3709 svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool); 3710 SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); 3711 3712 /* read first revision number and number of revisions in the pack */ 3713 stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 3714 SVN_ERR(read_number_from_stream(&first_rev, NULL, stream, iterpool)); 3715 SVN_ERR(read_number_from_stream(&count, NULL, stream, iterpool)); 3716 3717 /* make PACKED_REVPROPS point to the first char after the header. 3718 * This is where the serialized revprops are. */ 3719 header_end = strstr(uncompressed->data, "\n\n"); 3720 if (header_end == NULL) 3721 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 3722 _("Header end not found")); 3723 3724 offset = header_end - uncompressed->data + 2; 3725 3726 revprops->packed_revprops = svn_stringbuf_create_empty(pool); 3727 revprops->packed_revprops->data = uncompressed->data + offset; 3728 revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); 3729 revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); 3730 3731 /* STREAM still points to the first entry in the sizes list. 3732 * Init / construct REVPROPS members. */ 3733 revprops->start_revision = (svn_revnum_t)first_rev; 3734 revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); 3735 revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); 3736 3737 /* Now parse, revision by revision, the size and content of each 3738 * revisions' revprops. */ 3739 for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) 3740 { 3741 apr_int64_t size; 3742 svn_string_t serialized; 3743 apr_hash_t *properties; 3744 svn_revnum_t revision = (svn_revnum_t)(first_rev + i); 3745 3746 /* read & check the serialized size */ 3747 SVN_ERR(read_number_from_stream(&size, NULL, stream, iterpool)); 3748 if (size + offset > (apr_int64_t)revprops->packed_revprops->len) 3749 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 3750 _("Packed revprop size exceeds pack file size")); 3751 3752 /* Parse this revprops list, if necessary */ 3753 serialized.data = revprops->packed_revprops->data + offset; 3754 serialized.len = (apr_size_t)size; 3755 3756 if (revision == revprops->revision) 3757 { 3758 SVN_ERR(parse_revprop(&revprops->properties, fs, revision, 3759 revprops->generation, &serialized, 3760 pool, iterpool)); 3761 revprops->serialized_size = serialized.len; 3762 } 3763 else 3764 { 3765 /* If revprop caching is enabled, parse any revprops. 3766 * They will get cached as a side-effect of this. */ 3767 if (has_revprop_cache(fs, pool)) 3768 SVN_ERR(parse_revprop(&properties, fs, revision, 3769 revprops->generation, &serialized, 3770 iterpool, iterpool)); 3771 } 3772 3773 /* fill REVPROPS data structures */ 3774 APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; 3775 APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; 3776 revprops->total_size += serialized.len; 3777 3778 offset += serialized.len; 3779 3780 svn_pool_clear(iterpool); 3781 } 3782 3783 return SVN_NO_ERROR; 3784} 3785 3786/* In filesystem FS, read the packed revprops for revision REV into 3787 * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. 3788 * Allocate data in POOL. 3789 */ 3790static svn_error_t * 3791read_pack_revprop(packed_revprops_t **revprops, 3792 svn_fs_t *fs, 3793 svn_revnum_t rev, 3794 apr_int64_t generation, 3795 apr_pool_t *pool) 3796{ 3797 apr_pool_t *iterpool = svn_pool_create(pool); 3798 svn_boolean_t missing = FALSE; 3799 svn_error_t *err; 3800 packed_revprops_t *result; 3801 int i; 3802 3803 /* someone insisted that REV is packed. Double-check if necessary */ 3804 if (!is_packed_revprop(fs, rev)) 3805 SVN_ERR(update_min_unpacked_rev(fs, iterpool)); 3806 3807 if (!is_packed_revprop(fs, rev)) 3808 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 3809 _("No such packed revision %ld"), rev); 3810 3811 /* initialize the result data structure */ 3812 result = apr_pcalloc(pool, sizeof(*result)); 3813 result->revision = rev; 3814 result->generation = generation; 3815 3816 /* try to read the packed revprops. This may require retries if we have 3817 * concurrent writers. */ 3818 for (i = 0; i < RECOVERABLE_RETRY_COUNT && !result->packed_revprops; ++i) 3819 { 3820 const char *file_path; 3821 3822 /* there might have been concurrent writes. 3823 * Re-read the manifest and the pack file. 3824 */ 3825 SVN_ERR(get_revprop_packname(fs, result, pool, iterpool)); 3826 file_path = svn_dirent_join(result->folder, 3827 result->filename, 3828 iterpool); 3829 SVN_ERR(try_stringbuf_from_file(&result->packed_revprops, 3830 &missing, 3831 file_path, 3832 i + 1 < RECOVERABLE_RETRY_COUNT, 3833 pool)); 3834 3835 /* If we could not find the file, there was a write. 3836 * So, we should refresh our revprop generation info as well such 3837 * that others may find data we will put into the cache. They would 3838 * consider it outdated, otherwise. 3839 */ 3840 if (missing && has_revprop_cache(fs, pool)) 3841 SVN_ERR(read_revprop_generation(&result->generation, fs, pool)); 3842 3843 svn_pool_clear(iterpool); 3844 } 3845 3846 /* the file content should be available now */ 3847 if (!result->packed_revprops) 3848 return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, 3849 _("Failed to read revprop pack file for rev %ld"), rev); 3850 3851 /* parse it. RESULT will be complete afterwards. */ 3852 err = parse_packed_revprops(fs, result, pool, iterpool); 3853 svn_pool_destroy(iterpool); 3854 if (err) 3855 return svn_error_createf(SVN_ERR_FS_CORRUPT, err, 3856 _("Revprop pack file for rev %ld is corrupt"), rev); 3857 3858 *revprops = result; 3859 3860 return SVN_NO_ERROR; 3861} 3862 3863/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. 3864 * 3865 * Allocations will be done in POOL. 3866 */ 3867static svn_error_t * 3868get_revision_proplist(apr_hash_t **proplist_p, 3869 svn_fs_t *fs, 3870 svn_revnum_t rev, 3871 apr_pool_t *pool) 3872{ 3873 fs_fs_data_t *ffd = fs->fsap_data; 3874 apr_int64_t generation = 0; 3875 3876 /* not found, yet */ 3877 *proplist_p = NULL; 3878 3879 /* should they be available at all? */ 3880 SVN_ERR(ensure_revision_exists(fs, rev, pool)); 3881 3882 /* Try cache lookup first. */ 3883 if (has_revprop_cache(fs, pool)) 3884 { 3885 svn_boolean_t is_cached; 3886 pair_cache_key_t key = { 0 }; 3887 3888 SVN_ERR(read_revprop_generation(&generation, fs, pool)); 3889 3890 key.revision = rev; 3891 key.second = generation; 3892 SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, 3893 ffd->revprop_cache, &key, pool)); 3894 if (is_cached) 3895 return SVN_NO_ERROR; 3896 } 3897 3898 /* if REV had not been packed when we began, try reading it from the 3899 * non-packed shard. If that fails, we will fall through to packed 3900 * shard reads. */ 3901 if (!is_packed_revprop(fs, rev)) 3902 { 3903 svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, 3904 generation, pool); 3905 if (err) 3906 { 3907 if (!APR_STATUS_IS_ENOENT(err->apr_err) 3908 || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 3909 return svn_error_trace(err); 3910 3911 svn_error_clear(err); 3912 *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ 3913 } 3914 } 3915 3916 /* if revprop packing is available and we have not read the revprops, yet, 3917 * try reading them from a packed shard. If that fails, REV is most 3918 * likely invalid (or its revprops highly contested). */ 3919 if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) 3920 { 3921 packed_revprops_t *packed_revprops; 3922 SVN_ERR(read_pack_revprop(&packed_revprops, fs, rev, generation, pool)); 3923 *proplist_p = packed_revprops->properties; 3924 } 3925 3926 /* The revprops should have been there. Did we get them? */ 3927 if (!*proplist_p) 3928 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 3929 _("Could not read revprops for revision %ld"), 3930 rev); 3931 3932 return SVN_NO_ERROR; 3933} 3934 3935/* Serialize the revision property list PROPLIST of revision REV in 3936 * filesystem FS to a non-packed file. Return the name of that temporary 3937 * file in *TMP_PATH and the file path that it must be moved to in 3938 * *FINAL_PATH. 3939 * 3940 * Use POOL for allocations. 3941 */ 3942static svn_error_t * 3943write_non_packed_revprop(const char **final_path, 3944 const char **tmp_path, 3945 svn_fs_t *fs, 3946 svn_revnum_t rev, 3947 apr_hash_t *proplist, 3948 apr_pool_t *pool) 3949{ 3950 svn_stream_t *stream; 3951 *final_path = path_revprops(fs, rev, pool); 3952 3953 /* ### do we have a directory sitting around already? we really shouldn't 3954 ### have to get the dirname here. */ 3955 SVN_ERR(svn_stream_open_unique(&stream, tmp_path, 3956 svn_dirent_dirname(*final_path, pool), 3957 svn_io_file_del_none, pool, pool)); 3958 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 3959 SVN_ERR(svn_stream_close(stream)); 3960 3961 return SVN_NO_ERROR; 3962} 3963 3964/* After writing the new revprop file(s), call this function to move the 3965 * file at TMP_PATH to FINAL_PATH and give it the permissions from 3966 * PERMS_REFERENCE. 3967 * 3968 * If indicated in BUMP_GENERATION, increase FS' revprop generation. 3969 * Finally, delete all the temporary files given in FILES_TO_DELETE. 3970 * The latter may be NULL. 3971 * 3972 * Use POOL for temporary allocations. 3973 */ 3974static svn_error_t * 3975switch_to_new_revprop(svn_fs_t *fs, 3976 const char *final_path, 3977 const char *tmp_path, 3978 const char *perms_reference, 3979 apr_array_header_t *files_to_delete, 3980 svn_boolean_t bump_generation, 3981 apr_pool_t *pool) 3982{ 3983 /* Now, we may actually be replacing revprops. Make sure that all other 3984 threads and processes will know about this. */ 3985 if (bump_generation) 3986 SVN_ERR(begin_revprop_change(fs, pool)); 3987 3988 SVN_ERR(move_into_place(tmp_path, final_path, perms_reference, pool)); 3989 3990 /* Indicate that the update (if relevant) has been completed. */ 3991 if (bump_generation) 3992 SVN_ERR(end_revprop_change(fs, pool)); 3993 3994 /* Clean up temporary files, if necessary. */ 3995 if (files_to_delete) 3996 { 3997 apr_pool_t *iterpool = svn_pool_create(pool); 3998 int i; 3999 4000 for (i = 0; i < files_to_delete->nelts; ++i) 4001 { 4002 const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); 4003 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 4004 svn_pool_clear(iterpool); 4005 } 4006 4007 svn_pool_destroy(iterpool); 4008 } 4009 return SVN_NO_ERROR; 4010} 4011 4012/* Write a pack file header to STREAM that starts at revision START_REVISION 4013 * and contains the indexes [START,END) of SIZES. 4014 */ 4015static svn_error_t * 4016serialize_revprops_header(svn_stream_t *stream, 4017 svn_revnum_t start_revision, 4018 apr_array_header_t *sizes, 4019 int start, 4020 int end, 4021 apr_pool_t *pool) 4022{ 4023 apr_pool_t *iterpool = svn_pool_create(pool); 4024 int i; 4025 4026 SVN_ERR_ASSERT(start < end); 4027 4028 /* start revision and entry count */ 4029 SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision)); 4030 SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start)); 4031 4032 /* the sizes array */ 4033 for (i = start; i < end; ++i) 4034 { 4035 apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); 4036 SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", 4037 size)); 4038 } 4039 4040 /* the double newline char indicates the end of the header */ 4041 SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); 4042 4043 svn_pool_clear(iterpool); 4044 return SVN_NO_ERROR; 4045} 4046 4047/* Writes the a pack file to FILE_STREAM. It copies the serialized data 4048 * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. 4049 * 4050 * The data for the latter is taken from NEW_SERIALIZED. Note, that 4051 * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is 4052 * taken in that case but only a subset of the old data will be copied. 4053 * 4054 * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. 4055 * POOL is used for temporary allocations. 4056 */ 4057static svn_error_t * 4058repack_revprops(svn_fs_t *fs, 4059 packed_revprops_t *revprops, 4060 int start, 4061 int end, 4062 int changed_index, 4063 svn_stringbuf_t *new_serialized, 4064 apr_off_t new_total_size, 4065 svn_stream_t *file_stream, 4066 apr_pool_t *pool) 4067{ 4068 fs_fs_data_t *ffd = fs->fsap_data; 4069 svn_stream_t *stream; 4070 int i; 4071 4072 /* create data empty buffers and the stream object */ 4073 svn_stringbuf_t *uncompressed 4074 = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool); 4075 svn_stringbuf_t *compressed 4076 = svn_stringbuf_create_empty(pool); 4077 stream = svn_stream_from_stringbuf(uncompressed, pool); 4078 4079 /* write the header*/ 4080 SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, 4081 revprops->sizes, start, end, pool)); 4082 4083 /* append the serialized revprops */ 4084 for (i = start; i < end; ++i) 4085 if (i == changed_index) 4086 { 4087 SVN_ERR(svn_stream_write(stream, 4088 new_serialized->data, 4089 &new_serialized->len)); 4090 } 4091 else 4092 { 4093 apr_size_t size 4094 = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); 4095 apr_size_t offset 4096 = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); 4097 4098 SVN_ERR(svn_stream_write(stream, 4099 revprops->packed_revprops->data + offset, 4100 &size)); 4101 } 4102 4103 /* flush the stream buffer (if any) to our underlying data buffer */ 4104 SVN_ERR(svn_stream_close(stream)); 4105 4106 /* compress / store the data */ 4107 SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed), 4108 compressed, 4109 ffd->compress_packed_revprops 4110 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 4111 : SVN_DELTA_COMPRESSION_LEVEL_NONE)); 4112 4113 /* finally, write the content to the target stream and close it */ 4114 SVN_ERR(svn_stream_write(file_stream, compressed->data, &compressed->len)); 4115 SVN_ERR(svn_stream_close(file_stream)); 4116 4117 return SVN_NO_ERROR; 4118} 4119 4120/* Allocate a new pack file name for the revisions at index [START,END) 4121 * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, 4122 * auto-create that array if necessary. Return an open file stream to 4123 * the new file in *STREAM allocated in POOL. 4124 */ 4125static svn_error_t * 4126repack_stream_open(svn_stream_t **stream, 4127 svn_fs_t *fs, 4128 packed_revprops_t *revprops, 4129 int start, 4130 int end, 4131 apr_array_header_t **files_to_delete, 4132 apr_pool_t *pool) 4133{ 4134 apr_int64_t tag; 4135 const char *tag_string; 4136 svn_string_t *new_filename; 4137 int i; 4138 apr_file_t *file; 4139 4140 /* get the old (= current) file name and enlist it for later deletion */ 4141 const char *old_filename 4142 = APR_ARRAY_IDX(revprops->manifest, start, const char*); 4143 4144 if (*files_to_delete == NULL) 4145 *files_to_delete = apr_array_make(pool, 3, sizeof(const char*)); 4146 4147 APR_ARRAY_PUSH(*files_to_delete, const char*) 4148 = svn_dirent_join(revprops->folder, old_filename, pool); 4149 4150 /* increase the tag part, i.e. the counter after the dot */ 4151 tag_string = strchr(old_filename, '.'); 4152 if (tag_string == NULL) 4153 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 4154 _("Packed file '%s' misses a tag"), 4155 old_filename); 4156 4157 SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); 4158 new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT, 4159 revprops->start_revision + start, 4160 ++tag); 4161 4162 /* update the manifest to point to the new file */ 4163 for (i = start; i < end; ++i) 4164 APR_ARRAY_IDX(revprops->manifest, i, const char*) = new_filename->data; 4165 4166 /* create a file stream for the new file */ 4167 SVN_ERR(svn_io_file_open(&file, svn_dirent_join(revprops->folder, 4168 new_filename->data, 4169 pool), 4170 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); 4171 *stream = svn_stream_from_aprfile2(file, FALSE, pool); 4172 4173 return SVN_NO_ERROR; 4174} 4175 4176/* For revision REV in filesystem FS, set the revision properties to 4177 * PROPLIST. Return a new file in *TMP_PATH that the caller shall move 4178 * to *FINAL_PATH to make the change visible. Files to be deleted will 4179 * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. 4180 * Use POOL for allocations. 4181 */ 4182static svn_error_t * 4183write_packed_revprop(const char **final_path, 4184 const char **tmp_path, 4185 apr_array_header_t **files_to_delete, 4186 svn_fs_t *fs, 4187 svn_revnum_t rev, 4188 apr_hash_t *proplist, 4189 apr_pool_t *pool) 4190{ 4191 fs_fs_data_t *ffd = fs->fsap_data; 4192 packed_revprops_t *revprops; 4193 apr_int64_t generation = 0; 4194 svn_stream_t *stream; 4195 svn_stringbuf_t *serialized; 4196 apr_off_t new_total_size; 4197 int changed_index; 4198 4199 /* read the current revprop generation. This value will not change 4200 * while we hold the global write lock to this FS. */ 4201 if (has_revprop_cache(fs, pool)) 4202 SVN_ERR(read_revprop_generation(&generation, fs, pool)); 4203 4204 /* read contents of the current pack file */ 4205 SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, pool)); 4206 4207 /* serialize the new revprops */ 4208 serialized = svn_stringbuf_create_empty(pool); 4209 stream = svn_stream_from_stringbuf(serialized, pool); 4210 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 4211 SVN_ERR(svn_stream_close(stream)); 4212 4213 /* calculate the size of the new data */ 4214 changed_index = (int)(rev - revprops->start_revision); 4215 new_total_size = revprops->total_size - revprops->serialized_size 4216 + serialized->len 4217 + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; 4218 4219 APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; 4220 4221 /* can we put the new data into the same pack as the before? */ 4222 if ( new_total_size < ffd->revprop_pack_size 4223 || revprops->sizes->nelts == 1) 4224 { 4225 /* simply replace the old pack file with new content as we do it 4226 * in the non-packed case */ 4227 4228 *final_path = svn_dirent_join(revprops->folder, revprops->filename, 4229 pool); 4230 SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, 4231 svn_io_file_del_none, pool, pool)); 4232 SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, 4233 changed_index, serialized, new_total_size, 4234 stream, pool)); 4235 } 4236 else 4237 { 4238 /* split the pack file into two of roughly equal size */ 4239 int right_count, left_count, i; 4240 4241 int left = 0; 4242 int right = revprops->sizes->nelts - 1; 4243 apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; 4244 apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; 4245 4246 /* let left and right side grow such that their size difference 4247 * is minimal after each step. */ 4248 while (left <= right) 4249 if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) 4250 < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) 4251 { 4252 left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) 4253 + SVN_INT64_BUFFER_SIZE; 4254 ++left; 4255 } 4256 else 4257 { 4258 right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) 4259 + SVN_INT64_BUFFER_SIZE; 4260 --right; 4261 } 4262 4263 /* since the items need much less than SVN_INT64_BUFFER_SIZE 4264 * bytes to represent their length, the split may not be optimal */ 4265 left_count = left; 4266 right_count = revprops->sizes->nelts - left; 4267 4268 /* if new_size is large, one side may exceed the pack size limit. 4269 * In that case, split before and after the modified revprop.*/ 4270 if ( left_size > ffd->revprop_pack_size 4271 || right_size > ffd->revprop_pack_size) 4272 { 4273 left_count = changed_index; 4274 right_count = revprops->sizes->nelts - left_count - 1; 4275 } 4276 4277 /* write the new, split files */ 4278 if (left_count) 4279 { 4280 SVN_ERR(repack_stream_open(&stream, fs, revprops, 0, 4281 left_count, files_to_delete, pool)); 4282 SVN_ERR(repack_revprops(fs, revprops, 0, left_count, 4283 changed_index, serialized, new_total_size, 4284 stream, pool)); 4285 } 4286 4287 if (left_count + right_count < revprops->sizes->nelts) 4288 { 4289 SVN_ERR(repack_stream_open(&stream, fs, revprops, changed_index, 4290 changed_index + 1, files_to_delete, 4291 pool)); 4292 SVN_ERR(repack_revprops(fs, revprops, changed_index, 4293 changed_index + 1, 4294 changed_index, serialized, new_total_size, 4295 stream, pool)); 4296 } 4297 4298 if (right_count) 4299 { 4300 SVN_ERR(repack_stream_open(&stream, fs, revprops, 4301 revprops->sizes->nelts - right_count, 4302 revprops->sizes->nelts, 4303 files_to_delete, pool)); 4304 SVN_ERR(repack_revprops(fs, revprops, 4305 revprops->sizes->nelts - right_count, 4306 revprops->sizes->nelts, changed_index, 4307 serialized, new_total_size, stream, 4308 pool)); 4309 } 4310 4311 /* write the new manifest */ 4312 *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); 4313 SVN_ERR(svn_stream_open_unique(&stream, tmp_path, revprops->folder, 4314 svn_io_file_del_none, pool, pool)); 4315 4316 for (i = 0; i < revprops->manifest->nelts; ++i) 4317 { 4318 const char *filename = APR_ARRAY_IDX(revprops->manifest, i, 4319 const char*); 4320 SVN_ERR(svn_stream_printf(stream, pool, "%s\n", filename)); 4321 } 4322 4323 SVN_ERR(svn_stream_close(stream)); 4324 } 4325 4326 return SVN_NO_ERROR; 4327} 4328 4329/* Set the revision property list of revision REV in filesystem FS to 4330 PROPLIST. Use POOL for temporary allocations. */ 4331static svn_error_t * 4332set_revision_proplist(svn_fs_t *fs, 4333 svn_revnum_t rev, 4334 apr_hash_t *proplist, 4335 apr_pool_t *pool) 4336{ 4337 svn_boolean_t is_packed; 4338 svn_boolean_t bump_generation = FALSE; 4339 const char *final_path; 4340 const char *tmp_path; 4341 const char *perms_reference; 4342 apr_array_header_t *files_to_delete = NULL; 4343 4344 SVN_ERR(ensure_revision_exists(fs, rev, pool)); 4345 4346 /* this info will not change while we hold the global FS write lock */ 4347 is_packed = is_packed_revprop(fs, rev); 4348 4349 /* Test whether revprops already exist for this revision. 4350 * Only then will we need to bump the revprop generation. */ 4351 if (has_revprop_cache(fs, pool)) 4352 { 4353 if (is_packed) 4354 { 4355 bump_generation = TRUE; 4356 } 4357 else 4358 { 4359 svn_node_kind_t kind; 4360 SVN_ERR(svn_io_check_path(path_revprops(fs, rev, pool), &kind, 4361 pool)); 4362 bump_generation = kind != svn_node_none; 4363 } 4364 } 4365 4366 /* Serialize the new revprop data */ 4367 if (is_packed) 4368 SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, 4369 fs, rev, proplist, pool)); 4370 else 4371 SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, 4372 fs, rev, proplist, pool)); 4373 4374 /* We use the rev file of this revision as the perms reference, 4375 * because when setting revprops for the first time, the revprop 4376 * file won't exist and therefore can't serve as its own reference. 4377 * (Whereas the rev file should already exist at this point.) 4378 */ 4379 SVN_ERR(svn_fs_fs__path_rev_absolute(&perms_reference, fs, rev, pool)); 4380 4381 /* Now, switch to the new revprop data. */ 4382 SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, 4383 files_to_delete, bump_generation, pool)); 4384 4385 return SVN_NO_ERROR; 4386} 4387 4388svn_error_t * 4389svn_fs_fs__revision_proplist(apr_hash_t **proplist_p, 4390 svn_fs_t *fs, 4391 svn_revnum_t rev, 4392 apr_pool_t *pool) 4393{ 4394 SVN_ERR(get_revision_proplist(proplist_p, fs, rev, pool)); 4395 4396 return SVN_NO_ERROR; 4397} 4398 4399/* Represents where in the current svndiff data block each 4400 representation is. */ 4401struct rep_state 4402{ 4403 apr_file_t *file; 4404 /* The txdelta window cache to use or NULL. */ 4405 svn_cache__t *window_cache; 4406 /* Caches un-deltified windows. May be NULL. */ 4407 svn_cache__t *combined_cache; 4408 apr_off_t start; /* The starting offset for the raw 4409 svndiff/plaintext data minus header. */ 4410 apr_off_t off; /* The current offset into the file. */ 4411 apr_off_t end; /* The end offset of the raw data. */ 4412 int ver; /* If a delta, what svndiff version? */ 4413 int chunk_index; 4414}; 4415 4416/* See create_rep_state, which wraps this and adds another error. */ 4417static svn_error_t * 4418create_rep_state_body(struct rep_state **rep_state, 4419 struct rep_args **rep_args, 4420 apr_file_t **file_hint, 4421 svn_revnum_t *rev_hint, 4422 representation_t *rep, 4423 svn_fs_t *fs, 4424 apr_pool_t *pool) 4425{ 4426 fs_fs_data_t *ffd = fs->fsap_data; 4427 struct rep_state *rs = apr_pcalloc(pool, sizeof(*rs)); 4428 struct rep_args *ra; 4429 unsigned char buf[4]; 4430 4431 /* If the hint is 4432 * - given, 4433 * - refers to a valid revision, 4434 * - refers to a packed revision, 4435 * - as does the rep we want to read, and 4436 * - refers to the same pack file as the rep 4437 * ... 4438 */ 4439 if ( file_hint && rev_hint && *file_hint 4440 && SVN_IS_VALID_REVNUM(*rev_hint) 4441 && *rev_hint < ffd->min_unpacked_rev 4442 && rep->revision < ffd->min_unpacked_rev 4443 && ( (*rev_hint / ffd->max_files_per_dir) 4444 == (rep->revision / ffd->max_files_per_dir))) 4445 { 4446 /* ... we can re-use the same, already open file object 4447 */ 4448 apr_off_t offset; 4449 SVN_ERR(get_packed_offset(&offset, fs, rep->revision, pool)); 4450 4451 offset += rep->offset; 4452 SVN_ERR(svn_io_file_seek(*file_hint, APR_SET, &offset, pool)); 4453 4454 rs->file = *file_hint; 4455 } 4456 else 4457 { 4458 /* otherwise, create a new file object 4459 */ 4460 SVN_ERR(open_and_seek_representation(&rs->file, fs, rep, pool)); 4461 } 4462 4463 /* remember the current file, if suggested by the caller */ 4464 if (file_hint) 4465 *file_hint = rs->file; 4466 if (rev_hint) 4467 *rev_hint = rep->revision; 4468 4469 /* continue constructing RS and RA */ 4470 rs->window_cache = ffd->txdelta_window_cache; 4471 rs->combined_cache = ffd->combined_window_cache; 4472 4473 SVN_ERR(read_rep_line(&ra, rs->file, pool)); 4474 SVN_ERR(get_file_offset(&rs->start, rs->file, pool)); 4475 rs->off = rs->start; 4476 rs->end = rs->start + rep->size; 4477 *rep_state = rs; 4478 *rep_args = ra; 4479 4480 if (!ra->is_delta) 4481 /* This is a plaintext, so just return the current rep_state. */ 4482 return SVN_NO_ERROR; 4483 4484 /* We are dealing with a delta, find out what version. */ 4485 SVN_ERR(svn_io_file_read_full2(rs->file, buf, sizeof(buf), 4486 NULL, NULL, pool)); 4487 /* ### Layering violation */ 4488 if (! ((buf[0] == 'S') && (buf[1] == 'V') && (buf[2] == 'N'))) 4489 return svn_error_create 4490 (SVN_ERR_FS_CORRUPT, NULL, 4491 _("Malformed svndiff data in representation")); 4492 rs->ver = buf[3]; 4493 rs->chunk_index = 0; 4494 rs->off += 4; 4495 4496 return SVN_NO_ERROR; 4497} 4498 4499/* Read the rep args for REP in filesystem FS and create a rep_state 4500 for reading the representation. Return the rep_state in *REP_STATE 4501 and the rep args in *REP_ARGS, both allocated in POOL. 4502 4503 When reading multiple reps, i.e. a skip delta chain, you may provide 4504 non-NULL FILE_HINT and REV_HINT. (If FILE_HINT is not NULL, in the first 4505 call it should be a pointer to NULL.) The function will use these variables 4506 to store the previous call results and tries to re-use them. This may 4507 result in significant savings in I/O for packed files. 4508 */ 4509static svn_error_t * 4510create_rep_state(struct rep_state **rep_state, 4511 struct rep_args **rep_args, 4512 apr_file_t **file_hint, 4513 svn_revnum_t *rev_hint, 4514 representation_t *rep, 4515 svn_fs_t *fs, 4516 apr_pool_t *pool) 4517{ 4518 svn_error_t *err = create_rep_state_body(rep_state, rep_args, 4519 file_hint, rev_hint, 4520 rep, fs, pool); 4521 if (err && err->apr_err == SVN_ERR_FS_CORRUPT) 4522 { 4523 fs_fs_data_t *ffd = fs->fsap_data; 4524 4525 /* ### This always returns "-1" for transaction reps, because 4526 ### this particular bit of code doesn't know if the rep is 4527 ### stored in the protorev or in the mutable area (for props 4528 ### or dir contents). It is pretty rare for FSFS to *read* 4529 ### from the protorev file, though, so this is probably OK. 4530 ### And anyone going to debug corruption errors is probably 4531 ### going to jump straight to this comment anyway! */ 4532 return svn_error_createf(SVN_ERR_FS_CORRUPT, err, 4533 "Corrupt representation '%s'", 4534 rep 4535 ? representation_string(rep, ffd->format, TRUE, 4536 TRUE, pool) 4537 : "(null)"); 4538 } 4539 /* ### Call representation_string() ? */ 4540 return svn_error_trace(err); 4541} 4542 4543struct rep_read_baton 4544{ 4545 /* The FS from which we're reading. */ 4546 svn_fs_t *fs; 4547 4548 /* If not NULL, this is the base for the first delta window in rs_list */ 4549 svn_stringbuf_t *base_window; 4550 4551 /* The state of all prior delta representations. */ 4552 apr_array_header_t *rs_list; 4553 4554 /* The plaintext state, if there is a plaintext. */ 4555 struct rep_state *src_state; 4556 4557 /* The index of the current delta chunk, if we are reading a delta. */ 4558 int chunk_index; 4559 4560 /* The buffer where we store undeltified data. */ 4561 char *buf; 4562 apr_size_t buf_pos; 4563 apr_size_t buf_len; 4564 4565 /* A checksum context for summing the data read in order to verify it. 4566 Note: we don't need to use the sha1 checksum because we're only doing 4567 data verification, for which md5 is perfectly safe. */ 4568 svn_checksum_ctx_t *md5_checksum_ctx; 4569 4570 svn_boolean_t checksum_finalized; 4571 4572 /* The stored checksum of the representation we are reading, its 4573 length, and the amount we've read so far. Some of this 4574 information is redundant with rs_list and src_state, but it's 4575 convenient for the checksumming code to have it here. */ 4576 svn_checksum_t *md5_checksum; 4577 4578 svn_filesize_t len; 4579 svn_filesize_t off; 4580 4581 /* The key for the fulltext cache for this rep, if there is a 4582 fulltext cache. */ 4583 pair_cache_key_t fulltext_cache_key; 4584 /* The text we've been reading, if we're going to cache it. */ 4585 svn_stringbuf_t *current_fulltext; 4586 4587 /* Used for temporary allocations during the read. */ 4588 apr_pool_t *pool; 4589 4590 /* Pool used to store file handles and other data that is persistant 4591 for the entire stream read. */ 4592 apr_pool_t *filehandle_pool; 4593}; 4594 4595/* Combine the name of the rev file in RS with the given OFFSET to form 4596 * a cache lookup key. Allocations will be made from POOL. May return 4597 * NULL if the key cannot be constructed. */ 4598static const char* 4599get_window_key(struct rep_state *rs, apr_off_t offset, apr_pool_t *pool) 4600{ 4601 const char *name; 4602 const char *last_part; 4603 const char *name_last; 4604 4605 /* the rev file name containing the txdelta window. 4606 * If this fails we are in serious trouble anyways. 4607 * And if nobody else detects the problems, the file content checksum 4608 * comparison _will_ find them. 4609 */ 4610 if (apr_file_name_get(&name, rs->file)) 4611 return NULL; 4612 4613 /* Handle packed files as well by scanning backwards until we find the 4614 * revision or pack number. */ 4615 name_last = name + strlen(name) - 1; 4616 while (! svn_ctype_isdigit(*name_last)) 4617 --name_last; 4618 4619 last_part = name_last; 4620 while (svn_ctype_isdigit(*last_part)) 4621 --last_part; 4622 4623 /* We must differentiate between packed files (as of today, the number 4624 * is being followed by a dot) and non-packed files (followed by \0). 4625 * Otherwise, there might be overlaps in the numbering range if the 4626 * repo gets packed after caching the txdeltas of non-packed revs. 4627 * => add the first non-digit char to the packed number. */ 4628 if (name_last[1] != '\0') 4629 ++name_last; 4630 4631 /* copy one char MORE than the actual number to mark packed files, 4632 * i.e. packed revision file content uses different key space then 4633 * non-packed ones: keys for packed rev file content ends with a dot 4634 * for non-packed rev files they end with a digit. */ 4635 name = apr_pstrndup(pool, last_part + 1, name_last - last_part); 4636 return svn_fs_fs__combine_number_and_string(offset, name, pool); 4637} 4638 4639/* Read the WINDOW_P for the rep state RS from the current FSFS session's 4640 * cache. This will be a no-op and IS_CACHED will be set to FALSE if no 4641 * cache has been given. If a cache is available IS_CACHED will inform 4642 * the caller about the success of the lookup. Allocations (of the window 4643 * in particualar) will be made from POOL. 4644 * 4645 * If the information could be found, put RS and the position within the 4646 * rev file into the same state as if the data had just been read from it. 4647 */ 4648static svn_error_t * 4649get_cached_window(svn_txdelta_window_t **window_p, 4650 struct rep_state *rs, 4651 svn_boolean_t *is_cached, 4652 apr_pool_t *pool) 4653{ 4654 if (! rs->window_cache) 4655 { 4656 /* txdelta window has not been enabled */ 4657 *is_cached = FALSE; 4658 } 4659 else 4660 { 4661 /* ask the cache for the desired txdelta window */ 4662 svn_fs_fs__txdelta_cached_window_t *cached_window; 4663 SVN_ERR(svn_cache__get((void **) &cached_window, 4664 is_cached, 4665 rs->window_cache, 4666 get_window_key(rs, rs->off, pool), 4667 pool)); 4668 4669 if (*is_cached) 4670 { 4671 /* found it. Pass it back to the caller. */ 4672 *window_p = cached_window->window; 4673 4674 /* manipulate the RS as if we just read the data */ 4675 rs->chunk_index++; 4676 rs->off = cached_window->end_offset; 4677 4678 /* manipulate the rev file as if we just read from it */ 4679 SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool)); 4680 } 4681 } 4682 4683 return SVN_NO_ERROR; 4684} 4685 4686/* Store the WINDOW read at OFFSET for the rep state RS in the current 4687 * FSFS session's cache. This will be a no-op if no cache has been given. 4688 * Temporary allocations will be made from SCRATCH_POOL. */ 4689static svn_error_t * 4690set_cached_window(svn_txdelta_window_t *window, 4691 struct rep_state *rs, 4692 apr_off_t offset, 4693 apr_pool_t *scratch_pool) 4694{ 4695 if (rs->window_cache) 4696 { 4697 /* store the window and the first offset _past_ it */ 4698 svn_fs_fs__txdelta_cached_window_t cached_window; 4699 4700 cached_window.window = window; 4701 cached_window.end_offset = rs->off; 4702 4703 /* but key it with the start offset because that is the known state 4704 * when we will look it up */ 4705 return svn_cache__set(rs->window_cache, 4706 get_window_key(rs, offset, scratch_pool), 4707 &cached_window, 4708 scratch_pool); 4709 } 4710 4711 return SVN_NO_ERROR; 4712} 4713 4714/* Read the WINDOW_P for the rep state RS from the current FSFS session's 4715 * cache. This will be a no-op and IS_CACHED will be set to FALSE if no 4716 * cache has been given. If a cache is available IS_CACHED will inform 4717 * the caller about the success of the lookup. Allocations (of the window 4718 * in particualar) will be made from POOL. 4719 */ 4720static svn_error_t * 4721get_cached_combined_window(svn_stringbuf_t **window_p, 4722 struct rep_state *rs, 4723 svn_boolean_t *is_cached, 4724 apr_pool_t *pool) 4725{ 4726 if (! rs->combined_cache) 4727 { 4728 /* txdelta window has not been enabled */ 4729 *is_cached = FALSE; 4730 } 4731 else 4732 { 4733 /* ask the cache for the desired txdelta window */ 4734 return svn_cache__get((void **)window_p, 4735 is_cached, 4736 rs->combined_cache, 4737 get_window_key(rs, rs->start, pool), 4738 pool); 4739 } 4740 4741 return SVN_NO_ERROR; 4742} 4743 4744/* Store the WINDOW read at OFFSET for the rep state RS in the current 4745 * FSFS session's cache. This will be a no-op if no cache has been given. 4746 * Temporary allocations will be made from SCRATCH_POOL. */ 4747static svn_error_t * 4748set_cached_combined_window(svn_stringbuf_t *window, 4749 struct rep_state *rs, 4750 apr_off_t offset, 4751 apr_pool_t *scratch_pool) 4752{ 4753 if (rs->combined_cache) 4754 { 4755 /* but key it with the start offset because that is the known state 4756 * when we will look it up */ 4757 return svn_cache__set(rs->combined_cache, 4758 get_window_key(rs, offset, scratch_pool), 4759 window, 4760 scratch_pool); 4761 } 4762 4763 return SVN_NO_ERROR; 4764} 4765 4766/* Build an array of rep_state structures in *LIST giving the delta 4767 reps from first_rep to a plain-text or self-compressed rep. Set 4768 *SRC_STATE to the plain-text rep we find at the end of the chain, 4769 or to NULL if the final delta representation is self-compressed. 4770 The representation to start from is designated by filesystem FS, id 4771 ID, and representation REP. 4772 Also, set *WINDOW_P to the base window content for *LIST, if it 4773 could be found in cache. Otherwise, *LIST will contain the base 4774 representation for the whole delta chain. 4775 Finally, return the expanded size of the representation in 4776 *EXPANDED_SIZE. It will take care of cases where only the on-disk 4777 size is known. */ 4778static svn_error_t * 4779build_rep_list(apr_array_header_t **list, 4780 svn_stringbuf_t **window_p, 4781 struct rep_state **src_state, 4782 svn_filesize_t *expanded_size, 4783 svn_fs_t *fs, 4784 representation_t *first_rep, 4785 apr_pool_t *pool) 4786{ 4787 representation_t rep; 4788 struct rep_state *rs = NULL; 4789 struct rep_args *rep_args; 4790 svn_boolean_t is_cached = FALSE; 4791 apr_file_t *last_file = NULL; 4792 svn_revnum_t last_revision; 4793 4794 *list = apr_array_make(pool, 1, sizeof(struct rep_state *)); 4795 rep = *first_rep; 4796 4797 /* The value as stored in the data struct. 4798 0 is either for unknown length or actually zero length. */ 4799 *expanded_size = first_rep->expanded_size; 4800 4801 /* for the top-level rep, we need the rep_args */ 4802 SVN_ERR(create_rep_state(&rs, &rep_args, &last_file, 4803 &last_revision, &rep, fs, pool)); 4804 4805 /* Unknown size or empty representation? 4806 That implies the this being the first iteration. 4807 Usually size equals on-disk size, except for empty, 4808 compressed representations (delta, size = 4). 4809 Please note that for all non-empty deltas have 4810 a 4-byte header _plus_ some data. */ 4811 if (*expanded_size == 0) 4812 if (! rep_args->is_delta || first_rep->size != 4) 4813 *expanded_size = first_rep->size; 4814 4815 while (1) 4816 { 4817 /* fetch state, if that has not been done already */ 4818 if (!rs) 4819 SVN_ERR(create_rep_state(&rs, &rep_args, &last_file, 4820 &last_revision, &rep, fs, pool)); 4821 4822 SVN_ERR(get_cached_combined_window(window_p, rs, &is_cached, pool)); 4823 if (is_cached) 4824 { 4825 /* We already have a reconstructed window in our cache. 4826 Write a pseudo rep_state with the full length. */ 4827 rs->off = rs->start; 4828 rs->end = rs->start + (*window_p)->len; 4829 *src_state = rs; 4830 return SVN_NO_ERROR; 4831 } 4832 4833 if (!rep_args->is_delta) 4834 { 4835 /* This is a plaintext, so just return the current rep_state. */ 4836 *src_state = rs; 4837 return SVN_NO_ERROR; 4838 } 4839 4840 /* Push this rep onto the list. If it's self-compressed, we're done. */ 4841 APR_ARRAY_PUSH(*list, struct rep_state *) = rs; 4842 if (rep_args->is_delta_vs_empty) 4843 { 4844 *src_state = NULL; 4845 return SVN_NO_ERROR; 4846 } 4847 4848 rep.revision = rep_args->base_revision; 4849 rep.offset = rep_args->base_offset; 4850 rep.size = rep_args->base_length; 4851 rep.txn_id = NULL; 4852 4853 rs = NULL; 4854 } 4855} 4856 4857 4858/* Create a rep_read_baton structure for node revision NODEREV in 4859 filesystem FS and store it in *RB_P. If FULLTEXT_CACHE_KEY is not 4860 NULL, it is the rep's key in the fulltext cache, and a stringbuf 4861 must be allocated to store the text. Perform all allocations in 4862 POOL. If rep is mutable, it must be for file contents. */ 4863static svn_error_t * 4864rep_read_get_baton(struct rep_read_baton **rb_p, 4865 svn_fs_t *fs, 4866 representation_t *rep, 4867 pair_cache_key_t fulltext_cache_key, 4868 apr_pool_t *pool) 4869{ 4870 struct rep_read_baton *b; 4871 4872 b = apr_pcalloc(pool, sizeof(*b)); 4873 b->fs = fs; 4874 b->base_window = NULL; 4875 b->chunk_index = 0; 4876 b->buf = NULL; 4877 b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); 4878 b->checksum_finalized = FALSE; 4879 b->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool); 4880 b->len = rep->expanded_size; 4881 b->off = 0; 4882 b->fulltext_cache_key = fulltext_cache_key; 4883 b->pool = svn_pool_create(pool); 4884 b->filehandle_pool = svn_pool_create(pool); 4885 4886 SVN_ERR(build_rep_list(&b->rs_list, &b->base_window, 4887 &b->src_state, &b->len, fs, rep, 4888 b->filehandle_pool)); 4889 4890 if (SVN_IS_VALID_REVNUM(fulltext_cache_key.revision)) 4891 b->current_fulltext = svn_stringbuf_create_ensure 4892 ((apr_size_t)b->len, 4893 b->filehandle_pool); 4894 else 4895 b->current_fulltext = NULL; 4896 4897 /* Save our output baton. */ 4898 *rb_p = b; 4899 4900 return SVN_NO_ERROR; 4901} 4902 4903/* Skip forwards to THIS_CHUNK in REP_STATE and then read the next delta 4904 window into *NWIN. */ 4905static svn_error_t * 4906read_delta_window(svn_txdelta_window_t **nwin, int this_chunk, 4907 struct rep_state *rs, apr_pool_t *pool) 4908{ 4909 svn_stream_t *stream; 4910 svn_boolean_t is_cached; 4911 apr_off_t old_offset; 4912 4913 SVN_ERR_ASSERT(rs->chunk_index <= this_chunk); 4914 4915 /* RS->FILE may be shared between RS instances -> make sure we point 4916 * to the right data. */ 4917 SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool)); 4918 4919 /* Skip windows to reach the current chunk if we aren't there yet. */ 4920 while (rs->chunk_index < this_chunk) 4921 { 4922 SVN_ERR(svn_txdelta_skip_svndiff_window(rs->file, rs->ver, pool)); 4923 rs->chunk_index++; 4924 SVN_ERR(get_file_offset(&rs->off, rs->file, pool)); 4925 if (rs->off >= rs->end) 4926 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 4927 _("Reading one svndiff window read " 4928 "beyond the end of the " 4929 "representation")); 4930 } 4931 4932 /* Read the next window. But first, try to find it in the cache. */ 4933 SVN_ERR(get_cached_window(nwin, rs, &is_cached, pool)); 4934 if (is_cached) 4935 return SVN_NO_ERROR; 4936 4937 /* Actually read the next window. */ 4938 old_offset = rs->off; 4939 stream = svn_stream_from_aprfile2(rs->file, TRUE, pool); 4940 SVN_ERR(svn_txdelta_read_svndiff_window(nwin, stream, rs->ver, pool)); 4941 rs->chunk_index++; 4942 SVN_ERR(get_file_offset(&rs->off, rs->file, pool)); 4943 4944 if (rs->off > rs->end) 4945 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 4946 _("Reading one svndiff window read beyond " 4947 "the end of the representation")); 4948 4949 /* the window has not been cached before, thus cache it now 4950 * (if caching is used for them at all) */ 4951 return set_cached_window(*nwin, rs, old_offset, pool); 4952} 4953 4954/* Read SIZE bytes from the representation RS and return it in *NWIN. */ 4955static svn_error_t * 4956read_plain_window(svn_stringbuf_t **nwin, struct rep_state *rs, 4957 apr_size_t size, apr_pool_t *pool) 4958{ 4959 /* RS->FILE may be shared between RS instances -> make sure we point 4960 * to the right data. */ 4961 SVN_ERR(svn_io_file_seek(rs->file, APR_SET, &rs->off, pool)); 4962 4963 /* Read the plain data. */ 4964 *nwin = svn_stringbuf_create_ensure(size, pool); 4965 SVN_ERR(svn_io_file_read_full2(rs->file, (*nwin)->data, size, NULL, NULL, 4966 pool)); 4967 (*nwin)->data[size] = 0; 4968 4969 /* Update RS. */ 4970 rs->off += (apr_off_t)size; 4971 4972 return SVN_NO_ERROR; 4973} 4974 4975/* Get the undeltified window that is a result of combining all deltas 4976 from the current desired representation identified in *RB with its 4977 base representation. Store the window in *RESULT. */ 4978static svn_error_t * 4979get_combined_window(svn_stringbuf_t **result, 4980 struct rep_read_baton *rb) 4981{ 4982 apr_pool_t *pool, *new_pool, *window_pool; 4983 int i; 4984 svn_txdelta_window_t *window; 4985 apr_array_header_t *windows; 4986 svn_stringbuf_t *source, *buf = rb->base_window; 4987 struct rep_state *rs; 4988 4989 /* Read all windows that we need to combine. This is fine because 4990 the size of each window is relatively small (100kB) and skip- 4991 delta limits the number of deltas in a chain to well under 100. 4992 Stop early if one of them does not depend on its predecessors. */ 4993 window_pool = svn_pool_create(rb->pool); 4994 windows = apr_array_make(window_pool, 0, sizeof(svn_txdelta_window_t *)); 4995 for (i = 0; i < rb->rs_list->nelts; ++i) 4996 { 4997 rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *); 4998 SVN_ERR(read_delta_window(&window, rb->chunk_index, rs, window_pool)); 4999 5000 APR_ARRAY_PUSH(windows, svn_txdelta_window_t *) = window; 5001 if (window->src_ops == 0) 5002 { 5003 ++i; 5004 break; 5005 } 5006 } 5007 5008 /* Combine in the windows from the other delta reps. */ 5009 pool = svn_pool_create(rb->pool); 5010 for (--i; i >= 0; --i) 5011 { 5012 5013 rs = APR_ARRAY_IDX(rb->rs_list, i, struct rep_state *); 5014 window = APR_ARRAY_IDX(windows, i, svn_txdelta_window_t *); 5015 5016 /* Maybe, we've got a PLAIN start representation. If we do, read 5017 as much data from it as the needed for the txdelta window's source 5018 view. 5019 Note that BUF / SOURCE may only be NULL in the first iteration. */ 5020 source = buf; 5021 if (source == NULL && rb->src_state != NULL) 5022 SVN_ERR(read_plain_window(&source, rb->src_state, window->sview_len, 5023 pool)); 5024 5025 /* Combine this window with the current one. */ 5026 new_pool = svn_pool_create(rb->pool); 5027 buf = svn_stringbuf_create_ensure(window->tview_len, new_pool); 5028 buf->len = window->tview_len; 5029 5030 svn_txdelta_apply_instructions(window, source ? source->data : NULL, 5031 buf->data, &buf->len); 5032 if (buf->len != window->tview_len) 5033 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 5034 _("svndiff window length is " 5035 "corrupt")); 5036 5037 /* Cache windows only if the whole rep content could be read as a 5038 single chunk. Only then will no other chunk need a deeper RS 5039 list than the cached chunk. */ 5040 if ((rb->chunk_index == 0) && (rs->off == rs->end)) 5041 SVN_ERR(set_cached_combined_window(buf, rs, rs->start, new_pool)); 5042 5043 /* Cycle pools so that we only need to hold three windows at a time. */ 5044 svn_pool_destroy(pool); 5045 pool = new_pool; 5046 } 5047 5048 svn_pool_destroy(window_pool); 5049 5050 *result = buf; 5051 return SVN_NO_ERROR; 5052} 5053 5054/* Returns whether or not the expanded fulltext of the file is cachable 5055 * based on its size SIZE. The decision depends on the cache used by RB. 5056 */ 5057static svn_boolean_t 5058fulltext_size_is_cachable(fs_fs_data_t *ffd, svn_filesize_t size) 5059{ 5060 return (size < APR_SIZE_MAX) 5061 && svn_cache__is_cachable(ffd->fulltext_cache, (apr_size_t)size); 5062} 5063 5064/* Close method used on streams returned by read_representation(). 5065 */ 5066static svn_error_t * 5067rep_read_contents_close(void *baton) 5068{ 5069 struct rep_read_baton *rb = baton; 5070 5071 svn_pool_destroy(rb->pool); 5072 svn_pool_destroy(rb->filehandle_pool); 5073 5074 return SVN_NO_ERROR; 5075} 5076 5077/* Return the next *LEN bytes of the rep and store them in *BUF. */ 5078static svn_error_t * 5079get_contents(struct rep_read_baton *rb, 5080 char *buf, 5081 apr_size_t *len) 5082{ 5083 apr_size_t copy_len, remaining = *len; 5084 char *cur = buf; 5085 struct rep_state *rs; 5086 5087 /* Special case for when there are no delta reps, only a plain 5088 text. */ 5089 if (rb->rs_list->nelts == 0) 5090 { 5091 copy_len = remaining; 5092 rs = rb->src_state; 5093 5094 if (rb->base_window != NULL) 5095 { 5096 /* We got the desired rep directly from the cache. 5097 This is where we need the pseudo rep_state created 5098 by build_rep_list(). */ 5099 apr_size_t offset = (apr_size_t)(rs->off - rs->start); 5100 if (copy_len + offset > rb->base_window->len) 5101 copy_len = offset < rb->base_window->len 5102 ? rb->base_window->len - offset 5103 : 0ul; 5104 5105 memcpy (cur, rb->base_window->data + offset, copy_len); 5106 } 5107 else 5108 { 5109 if (((apr_off_t) copy_len) > rs->end - rs->off) 5110 copy_len = (apr_size_t) (rs->end - rs->off); 5111 SVN_ERR(svn_io_file_read_full2(rs->file, cur, copy_len, NULL, 5112 NULL, rb->pool)); 5113 } 5114 5115 rs->off += copy_len; 5116 *len = copy_len; 5117 return SVN_NO_ERROR; 5118 } 5119 5120 while (remaining > 0) 5121 { 5122 /* If we have buffered data from a previous chunk, use that. */ 5123 if (rb->buf) 5124 { 5125 /* Determine how much to copy from the buffer. */ 5126 copy_len = rb->buf_len - rb->buf_pos; 5127 if (copy_len > remaining) 5128 copy_len = remaining; 5129 5130 /* Actually copy the data. */ 5131 memcpy(cur, rb->buf + rb->buf_pos, copy_len); 5132 rb->buf_pos += copy_len; 5133 cur += copy_len; 5134 remaining -= copy_len; 5135 5136 /* If the buffer is all used up, clear it and empty the 5137 local pool. */ 5138 if (rb->buf_pos == rb->buf_len) 5139 { 5140 svn_pool_clear(rb->pool); 5141 rb->buf = NULL; 5142 } 5143 } 5144 else 5145 { 5146 svn_stringbuf_t *sbuf = NULL; 5147 5148 rs = APR_ARRAY_IDX(rb->rs_list, 0, struct rep_state *); 5149 if (rs->off == rs->end) 5150 break; 5151 5152 /* Get more buffered data by evaluating a chunk. */ 5153 SVN_ERR(get_combined_window(&sbuf, rb)); 5154 5155 rb->chunk_index++; 5156 rb->buf_len = sbuf->len; 5157 rb->buf = sbuf->data; 5158 rb->buf_pos = 0; 5159 } 5160 } 5161 5162 *len = cur - buf; 5163 5164 return SVN_NO_ERROR; 5165} 5166 5167/* BATON is of type `rep_read_baton'; read the next *LEN bytes of the 5168 representation and store them in *BUF. Sum as we read and verify 5169 the MD5 sum at the end. */ 5170static svn_error_t * 5171rep_read_contents(void *baton, 5172 char *buf, 5173 apr_size_t *len) 5174{ 5175 struct rep_read_baton *rb = baton; 5176 5177 /* Get the next block of data. */ 5178 SVN_ERR(get_contents(rb, buf, len)); 5179 5180 if (rb->current_fulltext) 5181 svn_stringbuf_appendbytes(rb->current_fulltext, buf, *len); 5182 5183 /* Perform checksumming. We want to check the checksum as soon as 5184 the last byte of data is read, in case the caller never performs 5185 a short read, but we don't want to finalize the MD5 context 5186 twice. */ 5187 if (!rb->checksum_finalized) 5188 { 5189 SVN_ERR(svn_checksum_update(rb->md5_checksum_ctx, buf, *len)); 5190 rb->off += *len; 5191 if (rb->off == rb->len) 5192 { 5193 svn_checksum_t *md5_checksum; 5194 5195 rb->checksum_finalized = TRUE; 5196 SVN_ERR(svn_checksum_final(&md5_checksum, rb->md5_checksum_ctx, 5197 rb->pool)); 5198 if (!svn_checksum_match(md5_checksum, rb->md5_checksum)) 5199 return svn_error_create(SVN_ERR_FS_CORRUPT, 5200 svn_checksum_mismatch_err(rb->md5_checksum, md5_checksum, 5201 rb->pool, 5202 _("Checksum mismatch while reading representation")), 5203 NULL); 5204 } 5205 } 5206 5207 if (rb->off == rb->len && rb->current_fulltext) 5208 { 5209 fs_fs_data_t *ffd = rb->fs->fsap_data; 5210 SVN_ERR(svn_cache__set(ffd->fulltext_cache, &rb->fulltext_cache_key, 5211 rb->current_fulltext, rb->pool)); 5212 rb->current_fulltext = NULL; 5213 } 5214 5215 return SVN_NO_ERROR; 5216} 5217 5218 5219/* Return a stream in *CONTENTS_P that will read the contents of a 5220 representation stored at the location given by REP. Appropriate 5221 for any kind of immutable representation, but only for file 5222 contents (not props or directory contents) in mutable 5223 representations. 5224 5225 If REP is NULL, the representation is assumed to be empty, and the 5226 empty stream is returned. 5227*/ 5228static svn_error_t * 5229read_representation(svn_stream_t **contents_p, 5230 svn_fs_t *fs, 5231 representation_t *rep, 5232 apr_pool_t *pool) 5233{ 5234 if (! rep) 5235 { 5236 *contents_p = svn_stream_empty(pool); 5237 } 5238 else 5239 { 5240 fs_fs_data_t *ffd = fs->fsap_data; 5241 pair_cache_key_t fulltext_cache_key = { 0 }; 5242 svn_filesize_t len = rep->expanded_size ? rep->expanded_size : rep->size; 5243 struct rep_read_baton *rb; 5244 5245 fulltext_cache_key.revision = rep->revision; 5246 fulltext_cache_key.second = rep->offset; 5247 if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) 5248 && fulltext_size_is_cachable(ffd, len)) 5249 { 5250 svn_stringbuf_t *fulltext; 5251 svn_boolean_t is_cached; 5252 SVN_ERR(svn_cache__get((void **) &fulltext, &is_cached, 5253 ffd->fulltext_cache, &fulltext_cache_key, 5254 pool)); 5255 if (is_cached) 5256 { 5257 *contents_p = svn_stream_from_stringbuf(fulltext, pool); 5258 return SVN_NO_ERROR; 5259 } 5260 } 5261 else 5262 fulltext_cache_key.revision = SVN_INVALID_REVNUM; 5263 5264 SVN_ERR(rep_read_get_baton(&rb, fs, rep, fulltext_cache_key, pool)); 5265 5266 *contents_p = svn_stream_create(rb, pool); 5267 svn_stream_set_read(*contents_p, rep_read_contents); 5268 svn_stream_set_close(*contents_p, rep_read_contents_close); 5269 } 5270 5271 return SVN_NO_ERROR; 5272} 5273 5274svn_error_t * 5275svn_fs_fs__get_contents(svn_stream_t **contents_p, 5276 svn_fs_t *fs, 5277 node_revision_t *noderev, 5278 apr_pool_t *pool) 5279{ 5280 return read_representation(contents_p, fs, noderev->data_rep, pool); 5281} 5282 5283/* Baton used when reading delta windows. */ 5284struct delta_read_baton 5285{ 5286 struct rep_state *rs; 5287 svn_checksum_t *checksum; 5288}; 5289 5290/* This implements the svn_txdelta_next_window_fn_t interface. */ 5291static svn_error_t * 5292delta_read_next_window(svn_txdelta_window_t **window, void *baton, 5293 apr_pool_t *pool) 5294{ 5295 struct delta_read_baton *drb = baton; 5296 5297 if (drb->rs->off == drb->rs->end) 5298 { 5299 *window = NULL; 5300 return SVN_NO_ERROR; 5301 } 5302 5303 return read_delta_window(window, drb->rs->chunk_index, drb->rs, pool); 5304} 5305 5306/* This implements the svn_txdelta_md5_digest_fn_t interface. */ 5307static const unsigned char * 5308delta_read_md5_digest(void *baton) 5309{ 5310 struct delta_read_baton *drb = baton; 5311 5312 if (drb->checksum->kind == svn_checksum_md5) 5313 return drb->checksum->digest; 5314 else 5315 return NULL; 5316} 5317 5318svn_error_t * 5319svn_fs_fs__get_file_delta_stream(svn_txdelta_stream_t **stream_p, 5320 svn_fs_t *fs, 5321 node_revision_t *source, 5322 node_revision_t *target, 5323 apr_pool_t *pool) 5324{ 5325 svn_stream_t *source_stream, *target_stream; 5326 5327 /* Try a shortcut: if the target is stored as a delta against the source, 5328 then just use that delta. */ 5329 if (source && source->data_rep && target->data_rep) 5330 { 5331 struct rep_state *rep_state; 5332 struct rep_args *rep_args; 5333 5334 /* Read target's base rep if any. */ 5335 SVN_ERR(create_rep_state(&rep_state, &rep_args, NULL, NULL, 5336 target->data_rep, fs, pool)); 5337 /* If that matches source, then use this delta as is. */ 5338 if (rep_args->is_delta 5339 && (rep_args->is_delta_vs_empty 5340 || (rep_args->base_revision == source->data_rep->revision 5341 && rep_args->base_offset == source->data_rep->offset))) 5342 { 5343 /* Create the delta read baton. */ 5344 struct delta_read_baton *drb = apr_pcalloc(pool, sizeof(*drb)); 5345 drb->rs = rep_state; 5346 drb->checksum = svn_checksum_dup(target->data_rep->md5_checksum, 5347 pool); 5348 *stream_p = svn_txdelta_stream_create(drb, delta_read_next_window, 5349 delta_read_md5_digest, pool); 5350 return SVN_NO_ERROR; 5351 } 5352 else 5353 SVN_ERR(svn_io_file_close(rep_state->file, pool)); 5354 } 5355 5356 /* Read both fulltexts and construct a delta. */ 5357 if (source) 5358 SVN_ERR(read_representation(&source_stream, fs, source->data_rep, pool)); 5359 else 5360 source_stream = svn_stream_empty(pool); 5361 SVN_ERR(read_representation(&target_stream, fs, target->data_rep, pool)); 5362 5363 /* Because source and target stream will already verify their content, 5364 * there is no need to do this once more. In particular if the stream 5365 * content is being fetched from cache. */ 5366 svn_txdelta2(stream_p, source_stream, target_stream, FALSE, pool); 5367 5368 return SVN_NO_ERROR; 5369} 5370 5371/* Baton for cache_access_wrapper. Wraps the original parameters of 5372 * svn_fs_fs__try_process_file_content(). 5373 */ 5374typedef struct cache_access_wrapper_baton_t 5375{ 5376 svn_fs_process_contents_func_t func; 5377 void* baton; 5378} cache_access_wrapper_baton_t; 5379 5380/* Wrapper to translate between svn_fs_process_contents_func_t and 5381 * svn_cache__partial_getter_func_t. 5382 */ 5383static svn_error_t * 5384cache_access_wrapper(void **out, 5385 const void *data, 5386 apr_size_t data_len, 5387 void *baton, 5388 apr_pool_t *pool) 5389{ 5390 cache_access_wrapper_baton_t *wrapper_baton = baton; 5391 5392 SVN_ERR(wrapper_baton->func((const unsigned char *)data, 5393 data_len - 1, /* cache adds terminating 0 */ 5394 wrapper_baton->baton, 5395 pool)); 5396 5397 /* non-NULL value to signal the calling cache that all went well */ 5398 *out = baton; 5399 5400 return SVN_NO_ERROR; 5401} 5402 5403svn_error_t * 5404svn_fs_fs__try_process_file_contents(svn_boolean_t *success, 5405 svn_fs_t *fs, 5406 node_revision_t *noderev, 5407 svn_fs_process_contents_func_t processor, 5408 void* baton, 5409 apr_pool_t *pool) 5410{ 5411 representation_t *rep = noderev->data_rep; 5412 if (rep) 5413 { 5414 fs_fs_data_t *ffd = fs->fsap_data; 5415 pair_cache_key_t fulltext_cache_key = { 0 }; 5416 5417 fulltext_cache_key.revision = rep->revision; 5418 fulltext_cache_key.second = rep->offset; 5419 if (ffd->fulltext_cache && SVN_IS_VALID_REVNUM(rep->revision) 5420 && fulltext_size_is_cachable(ffd, rep->expanded_size)) 5421 { 5422 cache_access_wrapper_baton_t wrapper_baton; 5423 void *dummy = NULL; 5424 5425 wrapper_baton.func = processor; 5426 wrapper_baton.baton = baton; 5427 return svn_cache__get_partial(&dummy, success, 5428 ffd->fulltext_cache, 5429 &fulltext_cache_key, 5430 cache_access_wrapper, 5431 &wrapper_baton, 5432 pool); 5433 } 5434 } 5435 5436 *success = FALSE; 5437 return SVN_NO_ERROR; 5438} 5439 5440/* Fetch the contents of a directory into ENTRIES. Values are stored 5441 as filename to string mappings; further conversion is necessary to 5442 convert them into svn_fs_dirent_t values. */ 5443static svn_error_t * 5444get_dir_contents(apr_hash_t *entries, 5445 svn_fs_t *fs, 5446 node_revision_t *noderev, 5447 apr_pool_t *pool) 5448{ 5449 svn_stream_t *contents; 5450 5451 if (noderev->data_rep && noderev->data_rep->txn_id) 5452 { 5453 const char *filename = path_txn_node_children(fs, noderev->id, pool); 5454 5455 /* The representation is mutable. Read the old directory 5456 contents from the mutable children file, followed by the 5457 changes we've made in this transaction. */ 5458 SVN_ERR(svn_stream_open_readonly(&contents, filename, pool, pool)); 5459 SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); 5460 SVN_ERR(svn_hash_read_incremental(entries, contents, NULL, pool)); 5461 SVN_ERR(svn_stream_close(contents)); 5462 } 5463 else if (noderev->data_rep) 5464 { 5465 /* use a temporary pool for temp objects. 5466 * Also undeltify content before parsing it. Otherwise, we could only 5467 * parse it byte-by-byte. 5468 */ 5469 apr_pool_t *text_pool = svn_pool_create(pool); 5470 apr_size_t len = noderev->data_rep->expanded_size 5471 ? (apr_size_t)noderev->data_rep->expanded_size 5472 : (apr_size_t)noderev->data_rep->size; 5473 svn_stringbuf_t *text = svn_stringbuf_create_ensure(len, text_pool); 5474 text->len = len; 5475 5476 /* The representation is immutable. Read it normally. */ 5477 SVN_ERR(read_representation(&contents, fs, noderev->data_rep, text_pool)); 5478 SVN_ERR(svn_stream_read(contents, text->data, &text->len)); 5479 SVN_ERR(svn_stream_close(contents)); 5480 5481 /* de-serialize hash */ 5482 contents = svn_stream_from_stringbuf(text, text_pool); 5483 SVN_ERR(svn_hash_read2(entries, contents, SVN_HASH_TERMINATOR, pool)); 5484 5485 svn_pool_destroy(text_pool); 5486 } 5487 5488 return SVN_NO_ERROR; 5489} 5490 5491 5492static const char * 5493unparse_dir_entry(svn_node_kind_t kind, const svn_fs_id_t *id, 5494 apr_pool_t *pool) 5495{ 5496 return apr_psprintf(pool, "%s %s", 5497 (kind == svn_node_file) ? KIND_FILE : KIND_DIR, 5498 svn_fs_fs__id_unparse(id, pool)->data); 5499} 5500 5501/* Given a hash ENTRIES of dirent structions, return a hash in 5502 *STR_ENTRIES_P, that has svn_string_t as the values in the format 5503 specified by the fs_fs directory contents file. Perform 5504 allocations in POOL. */ 5505static svn_error_t * 5506unparse_dir_entries(apr_hash_t **str_entries_p, 5507 apr_hash_t *entries, 5508 apr_pool_t *pool) 5509{ 5510 apr_hash_index_t *hi; 5511 5512 /* For now, we use a our own hash function to ensure that we get a 5513 * (largely) stable order when serializing the data. It also gives 5514 * us some performance improvement. 5515 * 5516 * ### TODO ### 5517 * Use some sorted or other fixed order data container. 5518 */ 5519 *str_entries_p = svn_hash__make(pool); 5520 5521 for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi)) 5522 { 5523 const void *key; 5524 apr_ssize_t klen; 5525 svn_fs_dirent_t *dirent = svn__apr_hash_index_val(hi); 5526 const char *new_val; 5527 5528 apr_hash_this(hi, &key, &klen, NULL); 5529 new_val = unparse_dir_entry(dirent->kind, dirent->id, pool); 5530 apr_hash_set(*str_entries_p, key, klen, 5531 svn_string_create(new_val, pool)); 5532 } 5533 5534 return SVN_NO_ERROR; 5535} 5536 5537 5538/* Given a hash STR_ENTRIES with values as svn_string_t as specified 5539 in an FSFS directory contents listing, return a hash of dirents in 5540 *ENTRIES_P. Perform allocations in POOL. */ 5541static svn_error_t * 5542parse_dir_entries(apr_hash_t **entries_p, 5543 apr_hash_t *str_entries, 5544 const char *unparsed_id, 5545 apr_pool_t *pool) 5546{ 5547 apr_hash_index_t *hi; 5548 5549 *entries_p = apr_hash_make(pool); 5550 5551 /* Translate the string dir entries into real entries. */ 5552 for (hi = apr_hash_first(pool, str_entries); hi; hi = apr_hash_next(hi)) 5553 { 5554 const char *name = svn__apr_hash_index_key(hi); 5555 svn_string_t *str_val = svn__apr_hash_index_val(hi); 5556 char *str, *last_str; 5557 svn_fs_dirent_t *dirent = apr_pcalloc(pool, sizeof(*dirent)); 5558 5559 last_str = apr_pstrdup(pool, str_val->data); 5560 dirent->name = apr_pstrdup(pool, name); 5561 5562 str = svn_cstring_tokenize(" ", &last_str); 5563 if (str == NULL) 5564 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 5565 _("Directory entry corrupt in '%s'"), 5566 unparsed_id); 5567 5568 if (strcmp(str, KIND_FILE) == 0) 5569 { 5570 dirent->kind = svn_node_file; 5571 } 5572 else if (strcmp(str, KIND_DIR) == 0) 5573 { 5574 dirent->kind = svn_node_dir; 5575 } 5576 else 5577 { 5578 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 5579 _("Directory entry corrupt in '%s'"), 5580 unparsed_id); 5581 } 5582 5583 str = svn_cstring_tokenize(" ", &last_str); 5584 if (str == NULL) 5585 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 5586 _("Directory entry corrupt in '%s'"), 5587 unparsed_id); 5588 5589 dirent->id = svn_fs_fs__id_parse(str, strlen(str), pool); 5590 5591 svn_hash_sets(*entries_p, dirent->name, dirent); 5592 } 5593 5594 return SVN_NO_ERROR; 5595} 5596 5597/* Return the cache object in FS responsible to storing the directory 5598 * the NODEREV. If none exists, return NULL. */ 5599static svn_cache__t * 5600locate_dir_cache(svn_fs_t *fs, 5601 node_revision_t *noderev) 5602{ 5603 fs_fs_data_t *ffd = fs->fsap_data; 5604 return svn_fs_fs__id_txn_id(noderev->id) 5605 ? ffd->txn_dir_cache 5606 : ffd->dir_cache; 5607} 5608 5609svn_error_t * 5610svn_fs_fs__rep_contents_dir(apr_hash_t **entries_p, 5611 svn_fs_t *fs, 5612 node_revision_t *noderev, 5613 apr_pool_t *pool) 5614{ 5615 const char *unparsed_id = NULL; 5616 apr_hash_t *unparsed_entries, *parsed_entries; 5617 5618 /* find the cache we may use */ 5619 svn_cache__t *cache = locate_dir_cache(fs, noderev); 5620 if (cache) 5621 { 5622 svn_boolean_t found; 5623 5624 unparsed_id = svn_fs_fs__id_unparse(noderev->id, pool)->data; 5625 SVN_ERR(svn_cache__get((void **) entries_p, &found, cache, 5626 unparsed_id, pool)); 5627 if (found) 5628 return SVN_NO_ERROR; 5629 } 5630 5631 /* Read in the directory hash. */ 5632 unparsed_entries = apr_hash_make(pool); 5633 SVN_ERR(get_dir_contents(unparsed_entries, fs, noderev, pool)); 5634 SVN_ERR(parse_dir_entries(&parsed_entries, unparsed_entries, 5635 unparsed_id, pool)); 5636 5637 /* Update the cache, if we are to use one. */ 5638 if (cache) 5639 SVN_ERR(svn_cache__set(cache, unparsed_id, parsed_entries, pool)); 5640 5641 *entries_p = parsed_entries; 5642 return SVN_NO_ERROR; 5643} 5644 5645svn_error_t * 5646svn_fs_fs__rep_contents_dir_entry(svn_fs_dirent_t **dirent, 5647 svn_fs_t *fs, 5648 node_revision_t *noderev, 5649 const char *name, 5650 apr_pool_t *result_pool, 5651 apr_pool_t *scratch_pool) 5652{ 5653 svn_boolean_t found = FALSE; 5654 5655 /* find the cache we may use */ 5656 svn_cache__t *cache = locate_dir_cache(fs, noderev); 5657 if (cache) 5658 { 5659 const char *unparsed_id = 5660 svn_fs_fs__id_unparse(noderev->id, scratch_pool)->data; 5661 5662 /* Cache lookup. */ 5663 SVN_ERR(svn_cache__get_partial((void **)dirent, 5664 &found, 5665 cache, 5666 unparsed_id, 5667 svn_fs_fs__extract_dir_entry, 5668 (void*)name, 5669 result_pool)); 5670 } 5671 5672 /* fetch data from disk if we did not find it in the cache */ 5673 if (! found) 5674 { 5675 apr_hash_t *entries; 5676 svn_fs_dirent_t *entry; 5677 svn_fs_dirent_t *entry_copy = NULL; 5678 5679 /* read the dir from the file system. It will probably be put it 5680 into the cache for faster lookup in future calls. */ 5681 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, 5682 scratch_pool)); 5683 5684 /* find desired entry and return a copy in POOL, if found */ 5685 entry = svn_hash_gets(entries, name); 5686 if (entry != NULL) 5687 { 5688 entry_copy = apr_palloc(result_pool, sizeof(*entry_copy)); 5689 entry_copy->name = apr_pstrdup(result_pool, entry->name); 5690 entry_copy->id = svn_fs_fs__id_copy(entry->id, result_pool); 5691 entry_copy->kind = entry->kind; 5692 } 5693 5694 *dirent = entry_copy; 5695 } 5696 5697 return SVN_NO_ERROR; 5698} 5699 5700svn_error_t * 5701svn_fs_fs__get_proplist(apr_hash_t **proplist_p, 5702 svn_fs_t *fs, 5703 node_revision_t *noderev, 5704 apr_pool_t *pool) 5705{ 5706 apr_hash_t *proplist; 5707 svn_stream_t *stream; 5708 5709 if (noderev->prop_rep && noderev->prop_rep->txn_id) 5710 { 5711 const char *filename = path_txn_node_props(fs, noderev->id, pool); 5712 proplist = apr_hash_make(pool); 5713 5714 SVN_ERR(svn_stream_open_readonly(&stream, filename, pool, pool)); 5715 SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 5716 SVN_ERR(svn_stream_close(stream)); 5717 } 5718 else if (noderev->prop_rep) 5719 { 5720 fs_fs_data_t *ffd = fs->fsap_data; 5721 representation_t *rep = noderev->prop_rep; 5722 pair_cache_key_t key = { 0 }; 5723 5724 key.revision = rep->revision; 5725 key.second = rep->offset; 5726 if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) 5727 { 5728 svn_boolean_t is_cached; 5729 SVN_ERR(svn_cache__get((void **) proplist_p, &is_cached, 5730 ffd->properties_cache, &key, pool)); 5731 if (is_cached) 5732 return SVN_NO_ERROR; 5733 } 5734 5735 proplist = apr_hash_make(pool); 5736 SVN_ERR(read_representation(&stream, fs, noderev->prop_rep, pool)); 5737 SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 5738 SVN_ERR(svn_stream_close(stream)); 5739 5740 if (ffd->properties_cache && SVN_IS_VALID_REVNUM(rep->revision)) 5741 SVN_ERR(svn_cache__set(ffd->properties_cache, &key, proplist, pool)); 5742 } 5743 else 5744 { 5745 /* return an empty prop list if the node doesn't have any props */ 5746 proplist = apr_hash_make(pool); 5747 } 5748 5749 *proplist_p = proplist; 5750 5751 return SVN_NO_ERROR; 5752} 5753 5754svn_error_t * 5755svn_fs_fs__file_length(svn_filesize_t *length, 5756 node_revision_t *noderev, 5757 apr_pool_t *pool) 5758{ 5759 if (noderev->data_rep) 5760 *length = noderev->data_rep->expanded_size; 5761 else 5762 *length = 0; 5763 5764 return SVN_NO_ERROR; 5765} 5766 5767svn_boolean_t 5768svn_fs_fs__noderev_same_rep_key(representation_t *a, 5769 representation_t *b) 5770{ 5771 if (a == b) 5772 return TRUE; 5773 5774 if (a == NULL || b == NULL) 5775 return FALSE; 5776 5777 if (a->offset != b->offset) 5778 return FALSE; 5779 5780 if (a->revision != b->revision) 5781 return FALSE; 5782 5783 if (a->uniquifier == b->uniquifier) 5784 return TRUE; 5785 5786 if (a->uniquifier == NULL || b->uniquifier == NULL) 5787 return FALSE; 5788 5789 return strcmp(a->uniquifier, b->uniquifier) == 0; 5790} 5791 5792svn_error_t * 5793svn_fs_fs__file_checksum(svn_checksum_t **checksum, 5794 node_revision_t *noderev, 5795 svn_checksum_kind_t kind, 5796 apr_pool_t *pool) 5797{ 5798 if (noderev->data_rep) 5799 { 5800 switch(kind) 5801 { 5802 case svn_checksum_md5: 5803 *checksum = svn_checksum_dup(noderev->data_rep->md5_checksum, 5804 pool); 5805 break; 5806 case svn_checksum_sha1: 5807 *checksum = svn_checksum_dup(noderev->data_rep->sha1_checksum, 5808 pool); 5809 break; 5810 default: 5811 *checksum = NULL; 5812 } 5813 } 5814 else 5815 *checksum = NULL; 5816 5817 return SVN_NO_ERROR; 5818} 5819 5820representation_t * 5821svn_fs_fs__rep_copy(representation_t *rep, 5822 apr_pool_t *pool) 5823{ 5824 representation_t *rep_new; 5825 5826 if (rep == NULL) 5827 return NULL; 5828 5829 rep_new = apr_pcalloc(pool, sizeof(*rep_new)); 5830 5831 memcpy(rep_new, rep, sizeof(*rep_new)); 5832 rep_new->md5_checksum = svn_checksum_dup(rep->md5_checksum, pool); 5833 rep_new->sha1_checksum = svn_checksum_dup(rep->sha1_checksum, pool); 5834 rep_new->uniquifier = apr_pstrdup(pool, rep->uniquifier); 5835 5836 return rep_new; 5837} 5838 5839/* Merge the internal-use-only CHANGE into a hash of public-FS 5840 svn_fs_path_change2_t CHANGES, collapsing multiple changes into a 5841 single summarical (is that real word?) change per path. Also keep 5842 the COPYFROM_CACHE up to date with new adds and replaces. */ 5843static svn_error_t * 5844fold_change(apr_hash_t *changes, 5845 const change_t *change, 5846 apr_hash_t *copyfrom_cache) 5847{ 5848 apr_pool_t *pool = apr_hash_pool_get(changes); 5849 svn_fs_path_change2_t *old_change, *new_change; 5850 const char *path; 5851 apr_size_t path_len = strlen(change->path); 5852 5853 if ((old_change = apr_hash_get(changes, change->path, path_len))) 5854 { 5855 /* This path already exists in the hash, so we have to merge 5856 this change into the already existing one. */ 5857 5858 /* Sanity check: only allow NULL node revision ID in the 5859 `reset' case. */ 5860 if ((! change->noderev_id) && (change->kind != svn_fs_path_change_reset)) 5861 return svn_error_create 5862 (SVN_ERR_FS_CORRUPT, NULL, 5863 _("Missing required node revision ID")); 5864 5865 /* Sanity check: we should be talking about the same node 5866 revision ID as our last change except where the last change 5867 was a deletion. */ 5868 if (change->noderev_id 5869 && (! svn_fs_fs__id_eq(old_change->node_rev_id, change->noderev_id)) 5870 && (old_change->change_kind != svn_fs_path_change_delete)) 5871 return svn_error_create 5872 (SVN_ERR_FS_CORRUPT, NULL, 5873 _("Invalid change ordering: new node revision ID " 5874 "without delete")); 5875 5876 /* Sanity check: an add, replacement, or reset must be the first 5877 thing to follow a deletion. */ 5878 if ((old_change->change_kind == svn_fs_path_change_delete) 5879 && (! ((change->kind == svn_fs_path_change_replace) 5880 || (change->kind == svn_fs_path_change_reset) 5881 || (change->kind == svn_fs_path_change_add)))) 5882 return svn_error_create 5883 (SVN_ERR_FS_CORRUPT, NULL, 5884 _("Invalid change ordering: non-add change on deleted path")); 5885 5886 /* Sanity check: an add can't follow anything except 5887 a delete or reset. */ 5888 if ((change->kind == svn_fs_path_change_add) 5889 && (old_change->change_kind != svn_fs_path_change_delete) 5890 && (old_change->change_kind != svn_fs_path_change_reset)) 5891 return svn_error_create 5892 (SVN_ERR_FS_CORRUPT, NULL, 5893 _("Invalid change ordering: add change on preexisting path")); 5894 5895 /* Now, merge that change in. */ 5896 switch (change->kind) 5897 { 5898 case svn_fs_path_change_reset: 5899 /* A reset here will simply remove the path change from the 5900 hash. */ 5901 old_change = NULL; 5902 break; 5903 5904 case svn_fs_path_change_delete: 5905 if (old_change->change_kind == svn_fs_path_change_add) 5906 { 5907 /* If the path was introduced in this transaction via an 5908 add, and we are deleting it, just remove the path 5909 altogether. */ 5910 old_change = NULL; 5911 } 5912 else 5913 { 5914 /* A deletion overrules all previous changes. */ 5915 old_change->change_kind = svn_fs_path_change_delete; 5916 old_change->text_mod = change->text_mod; 5917 old_change->prop_mod = change->prop_mod; 5918 old_change->copyfrom_rev = SVN_INVALID_REVNUM; 5919 old_change->copyfrom_path = NULL; 5920 } 5921 break; 5922 5923 case svn_fs_path_change_add: 5924 case svn_fs_path_change_replace: 5925 /* An add at this point must be following a previous delete, 5926 so treat it just like a replace. */ 5927 old_change->change_kind = svn_fs_path_change_replace; 5928 old_change->node_rev_id = svn_fs_fs__id_copy(change->noderev_id, 5929 pool); 5930 old_change->text_mod = change->text_mod; 5931 old_change->prop_mod = change->prop_mod; 5932 if (change->copyfrom_rev == SVN_INVALID_REVNUM) 5933 { 5934 old_change->copyfrom_rev = SVN_INVALID_REVNUM; 5935 old_change->copyfrom_path = NULL; 5936 } 5937 else 5938 { 5939 old_change->copyfrom_rev = change->copyfrom_rev; 5940 old_change->copyfrom_path = apr_pstrdup(pool, 5941 change->copyfrom_path); 5942 } 5943 break; 5944 5945 case svn_fs_path_change_modify: 5946 default: 5947 if (change->text_mod) 5948 old_change->text_mod = TRUE; 5949 if (change->prop_mod) 5950 old_change->prop_mod = TRUE; 5951 break; 5952 } 5953 5954 /* Point our new_change to our (possibly modified) old_change. */ 5955 new_change = old_change; 5956 } 5957 else 5958 { 5959 /* This change is new to the hash, so make a new public change 5960 structure from the internal one (in the hash's pool), and dup 5961 the path into the hash's pool, too. */ 5962 new_change = apr_pcalloc(pool, sizeof(*new_change)); 5963 new_change->node_rev_id = svn_fs_fs__id_copy(change->noderev_id, pool); 5964 new_change->change_kind = change->kind; 5965 new_change->text_mod = change->text_mod; 5966 new_change->prop_mod = change->prop_mod; 5967 /* In FSFS, copyfrom_known is *always* true, since we've always 5968 * stored copyfroms in changed paths lists. */ 5969 new_change->copyfrom_known = TRUE; 5970 if (change->copyfrom_rev != SVN_INVALID_REVNUM) 5971 { 5972 new_change->copyfrom_rev = change->copyfrom_rev; 5973 new_change->copyfrom_path = apr_pstrdup(pool, change->copyfrom_path); 5974 } 5975 else 5976 { 5977 new_change->copyfrom_rev = SVN_INVALID_REVNUM; 5978 new_change->copyfrom_path = NULL; 5979 } 5980 } 5981 5982 if (new_change) 5983 new_change->node_kind = change->node_kind; 5984 5985 /* Add (or update) this path. 5986 5987 Note: this key might already be present, and it would be nice to 5988 re-use its value, but there is no way to fetch it. The API makes no 5989 guarantees that this (new) key will not be retained. Thus, we (again) 5990 copy the key into the target pool to ensure a proper lifetime. */ 5991 path = apr_pstrmemdup(pool, change->path, path_len); 5992 apr_hash_set(changes, path, path_len, new_change); 5993 5994 /* Update the copyfrom cache, if any. */ 5995 if (copyfrom_cache) 5996 { 5997 apr_pool_t *copyfrom_pool = apr_hash_pool_get(copyfrom_cache); 5998 const char *copyfrom_string = NULL, *copyfrom_key = path; 5999 if (new_change) 6000 { 6001 if (SVN_IS_VALID_REVNUM(new_change->copyfrom_rev)) 6002 copyfrom_string = apr_psprintf(copyfrom_pool, "%ld %s", 6003 new_change->copyfrom_rev, 6004 new_change->copyfrom_path); 6005 else 6006 copyfrom_string = ""; 6007 } 6008 /* We need to allocate a copy of the key in the copyfrom_pool if 6009 * we're not doing a deletion and if it isn't already there. */ 6010 if ( copyfrom_string 6011 && ( ! apr_hash_count(copyfrom_cache) 6012 || ! apr_hash_get(copyfrom_cache, copyfrom_key, path_len))) 6013 copyfrom_key = apr_pstrmemdup(copyfrom_pool, copyfrom_key, path_len); 6014 6015 apr_hash_set(copyfrom_cache, copyfrom_key, path_len, 6016 copyfrom_string); 6017 } 6018 6019 return SVN_NO_ERROR; 6020} 6021 6022/* The 256 is an arbitrary size large enough to hold the node id and the 6023 * various flags. */ 6024#define MAX_CHANGE_LINE_LEN FSFS_MAX_PATH_LEN + 256 6025 6026/* Read the next entry in the changes record from file FILE and store 6027 the resulting change in *CHANGE_P. If there is no next record, 6028 store NULL there. Perform all allocations from POOL. */ 6029static svn_error_t * 6030read_change(change_t **change_p, 6031 apr_file_t *file, 6032 apr_pool_t *pool) 6033{ 6034 char buf[MAX_CHANGE_LINE_LEN]; 6035 apr_size_t len = sizeof(buf); 6036 change_t *change; 6037 char *str, *last_str = buf, *kind_str; 6038 svn_error_t *err; 6039 6040 /* Default return value. */ 6041 *change_p = NULL; 6042 6043 err = svn_io_read_length_line(file, buf, &len, pool); 6044 6045 /* Check for a blank line. */ 6046 if (err || (len == 0)) 6047 { 6048 if (err && APR_STATUS_IS_EOF(err->apr_err)) 6049 { 6050 svn_error_clear(err); 6051 return SVN_NO_ERROR; 6052 } 6053 if ((len == 0) && (! err)) 6054 return SVN_NO_ERROR; 6055 return svn_error_trace(err); 6056 } 6057 6058 change = apr_pcalloc(pool, sizeof(*change)); 6059 6060 /* Get the node-id of the change. */ 6061 str = svn_cstring_tokenize(" ", &last_str); 6062 if (str == NULL) 6063 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6064 _("Invalid changes line in rev-file")); 6065 6066 change->noderev_id = svn_fs_fs__id_parse(str, strlen(str), pool); 6067 if (change->noderev_id == NULL) 6068 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6069 _("Invalid changes line in rev-file")); 6070 6071 /* Get the change type. */ 6072 str = svn_cstring_tokenize(" ", &last_str); 6073 if (str == NULL) 6074 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6075 _("Invalid changes line in rev-file")); 6076 6077 /* Don't bother to check the format number before looking for 6078 * node-kinds: just read them if you find them. */ 6079 change->node_kind = svn_node_unknown; 6080 kind_str = strchr(str, '-'); 6081 if (kind_str) 6082 { 6083 /* Cap off the end of "str" (the action). */ 6084 *kind_str = '\0'; 6085 kind_str++; 6086 if (strcmp(kind_str, KIND_FILE) == 0) 6087 change->node_kind = svn_node_file; 6088 else if (strcmp(kind_str, KIND_DIR) == 0) 6089 change->node_kind = svn_node_dir; 6090 else 6091 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6092 _("Invalid changes line in rev-file")); 6093 } 6094 6095 if (strcmp(str, ACTION_MODIFY) == 0) 6096 { 6097 change->kind = svn_fs_path_change_modify; 6098 } 6099 else if (strcmp(str, ACTION_ADD) == 0) 6100 { 6101 change->kind = svn_fs_path_change_add; 6102 } 6103 else if (strcmp(str, ACTION_DELETE) == 0) 6104 { 6105 change->kind = svn_fs_path_change_delete; 6106 } 6107 else if (strcmp(str, ACTION_REPLACE) == 0) 6108 { 6109 change->kind = svn_fs_path_change_replace; 6110 } 6111 else if (strcmp(str, ACTION_RESET) == 0) 6112 { 6113 change->kind = svn_fs_path_change_reset; 6114 } 6115 else 6116 { 6117 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6118 _("Invalid change kind in rev file")); 6119 } 6120 6121 /* Get the text-mod flag. */ 6122 str = svn_cstring_tokenize(" ", &last_str); 6123 if (str == NULL) 6124 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6125 _("Invalid changes line in rev-file")); 6126 6127 if (strcmp(str, FLAG_TRUE) == 0) 6128 { 6129 change->text_mod = TRUE; 6130 } 6131 else if (strcmp(str, FLAG_FALSE) == 0) 6132 { 6133 change->text_mod = FALSE; 6134 } 6135 else 6136 { 6137 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6138 _("Invalid text-mod flag in rev-file")); 6139 } 6140 6141 /* Get the prop-mod flag. */ 6142 str = svn_cstring_tokenize(" ", &last_str); 6143 if (str == NULL) 6144 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6145 _("Invalid changes line in rev-file")); 6146 6147 if (strcmp(str, FLAG_TRUE) == 0) 6148 { 6149 change->prop_mod = TRUE; 6150 } 6151 else if (strcmp(str, FLAG_FALSE) == 0) 6152 { 6153 change->prop_mod = FALSE; 6154 } 6155 else 6156 { 6157 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6158 _("Invalid prop-mod flag in rev-file")); 6159 } 6160 6161 /* Get the changed path. */ 6162 change->path = apr_pstrdup(pool, last_str); 6163 6164 6165 /* Read the next line, the copyfrom line. */ 6166 len = sizeof(buf); 6167 SVN_ERR(svn_io_read_length_line(file, buf, &len, pool)); 6168 6169 if (len == 0) 6170 { 6171 change->copyfrom_rev = SVN_INVALID_REVNUM; 6172 change->copyfrom_path = NULL; 6173 } 6174 else 6175 { 6176 last_str = buf; 6177 str = svn_cstring_tokenize(" ", &last_str); 6178 if (! str) 6179 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6180 _("Invalid changes line in rev-file")); 6181 change->copyfrom_rev = SVN_STR_TO_REV(str); 6182 6183 if (! last_str) 6184 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6185 _("Invalid changes line in rev-file")); 6186 6187 change->copyfrom_path = apr_pstrdup(pool, last_str); 6188 } 6189 6190 *change_p = change; 6191 6192 return SVN_NO_ERROR; 6193} 6194 6195/* Examine all the changed path entries in CHANGES and store them in 6196 *CHANGED_PATHS. Folding is done to remove redundant or unnecessary 6197 *data. Store a hash of paths to copyfrom "REV PATH" strings in 6198 COPYFROM_HASH if it is non-NULL. If PREFOLDED is true, assume that 6199 the changed-path entries have already been folded (by 6200 write_final_changed_path_info) and may be out of order, so we shouldn't 6201 remove children of replaced or deleted directories. Do all 6202 allocations in POOL. */ 6203static svn_error_t * 6204process_changes(apr_hash_t *changed_paths, 6205 apr_hash_t *copyfrom_cache, 6206 apr_array_header_t *changes, 6207 svn_boolean_t prefolded, 6208 apr_pool_t *pool) 6209{ 6210 apr_pool_t *iterpool = svn_pool_create(pool); 6211 int i; 6212 6213 /* Read in the changes one by one, folding them into our local hash 6214 as necessary. */ 6215 6216 for (i = 0; i < changes->nelts; ++i) 6217 { 6218 change_t *change = APR_ARRAY_IDX(changes, i, change_t *); 6219 6220 SVN_ERR(fold_change(changed_paths, change, copyfrom_cache)); 6221 6222 /* Now, if our change was a deletion or replacement, we have to 6223 blow away any changes thus far on paths that are (or, were) 6224 children of this path. 6225 ### i won't bother with another iteration pool here -- at 6226 most we talking about a few extra dups of paths into what 6227 is already a temporary subpool. 6228 */ 6229 6230 if (((change->kind == svn_fs_path_change_delete) 6231 || (change->kind == svn_fs_path_change_replace)) 6232 && ! prefolded) 6233 { 6234 apr_hash_index_t *hi; 6235 6236 /* a potential child path must contain at least 2 more chars 6237 (the path separator plus at least one char for the name). 6238 Also, we should not assume that all paths have been normalized 6239 i.e. some might have trailing path separators. 6240 */ 6241 apr_ssize_t change_path_len = strlen(change->path); 6242 apr_ssize_t min_child_len = change_path_len == 0 6243 ? 1 6244 : change->path[change_path_len-1] == '/' 6245 ? change_path_len + 1 6246 : change_path_len + 2; 6247 6248 /* CAUTION: This is the inner loop of an O(n^2) algorithm. 6249 The number of changes to process may be >> 1000. 6250 Therefore, keep the inner loop as tight as possible. 6251 */ 6252 for (hi = apr_hash_first(iterpool, changed_paths); 6253 hi; 6254 hi = apr_hash_next(hi)) 6255 { 6256 /* KEY is the path. */ 6257 const void *path; 6258 apr_ssize_t klen; 6259 apr_hash_this(hi, &path, &klen, NULL); 6260 6261 /* If we come across a child of our path, remove it. 6262 Call svn_dirent_is_child only if there is a chance that 6263 this is actually a sub-path. 6264 */ 6265 if ( klen >= min_child_len 6266 && svn_dirent_is_child(change->path, path, iterpool)) 6267 apr_hash_set(changed_paths, path, klen, NULL); 6268 } 6269 } 6270 6271 /* Clear the per-iteration subpool. */ 6272 svn_pool_clear(iterpool); 6273 } 6274 6275 /* Destroy the per-iteration subpool. */ 6276 svn_pool_destroy(iterpool); 6277 6278 return SVN_NO_ERROR; 6279} 6280 6281/* Fetch all the changes from FILE and store them in *CHANGES. Do all 6282 allocations in POOL. */ 6283static svn_error_t * 6284read_all_changes(apr_array_header_t **changes, 6285 apr_file_t *file, 6286 apr_pool_t *pool) 6287{ 6288 change_t *change; 6289 6290 /* pre-allocate enough room for most change lists 6291 (will be auto-expanded as necessary) */ 6292 *changes = apr_array_make(pool, 30, sizeof(change_t *)); 6293 6294 SVN_ERR(read_change(&change, file, pool)); 6295 while (change) 6296 { 6297 APR_ARRAY_PUSH(*changes, change_t*) = change; 6298 SVN_ERR(read_change(&change, file, pool)); 6299 } 6300 6301 return SVN_NO_ERROR; 6302} 6303 6304svn_error_t * 6305svn_fs_fs__txn_changes_fetch(apr_hash_t **changed_paths_p, 6306 svn_fs_t *fs, 6307 const char *txn_id, 6308 apr_pool_t *pool) 6309{ 6310 apr_file_t *file; 6311 apr_hash_t *changed_paths = apr_hash_make(pool); 6312 apr_array_header_t *changes; 6313 apr_pool_t *scratch_pool = svn_pool_create(pool); 6314 6315 SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool), 6316 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); 6317 6318 SVN_ERR(read_all_changes(&changes, file, scratch_pool)); 6319 SVN_ERR(process_changes(changed_paths, NULL, changes, FALSE, pool)); 6320 svn_pool_destroy(scratch_pool); 6321 6322 SVN_ERR(svn_io_file_close(file, pool)); 6323 6324 *changed_paths_p = changed_paths; 6325 6326 return SVN_NO_ERROR; 6327} 6328 6329/* Fetch the list of change in revision REV in FS and return it in *CHANGES. 6330 * Allocate the result in POOL. 6331 */ 6332static svn_error_t * 6333get_changes(apr_array_header_t **changes, 6334 svn_fs_t *fs, 6335 svn_revnum_t rev, 6336 apr_pool_t *pool) 6337{ 6338 apr_off_t changes_offset; 6339 apr_file_t *revision_file; 6340 svn_boolean_t found; 6341 fs_fs_data_t *ffd = fs->fsap_data; 6342 6343 /* try cache lookup first */ 6344 6345 if (ffd->changes_cache) 6346 { 6347 SVN_ERR(svn_cache__get((void **) changes, &found, ffd->changes_cache, 6348 &rev, pool)); 6349 if (found) 6350 return SVN_NO_ERROR; 6351 } 6352 6353 /* read changes from revision file */ 6354 6355 SVN_ERR(ensure_revision_exists(fs, rev, pool)); 6356 6357 SVN_ERR(open_pack_or_rev_file(&revision_file, fs, rev, pool)); 6358 6359 SVN_ERR(get_root_changes_offset(NULL, &changes_offset, revision_file, fs, 6360 rev, pool)); 6361 6362 SVN_ERR(svn_io_file_seek(revision_file, APR_SET, &changes_offset, pool)); 6363 SVN_ERR(read_all_changes(changes, revision_file, pool)); 6364 6365 SVN_ERR(svn_io_file_close(revision_file, pool)); 6366 6367 /* cache for future reference */ 6368 6369 if (ffd->changes_cache) 6370 SVN_ERR(svn_cache__set(ffd->changes_cache, &rev, *changes, pool)); 6371 6372 return SVN_NO_ERROR; 6373} 6374 6375 6376svn_error_t * 6377svn_fs_fs__paths_changed(apr_hash_t **changed_paths_p, 6378 svn_fs_t *fs, 6379 svn_revnum_t rev, 6380 apr_hash_t *copyfrom_cache, 6381 apr_pool_t *pool) 6382{ 6383 apr_hash_t *changed_paths; 6384 apr_array_header_t *changes; 6385 apr_pool_t *scratch_pool = svn_pool_create(pool); 6386 6387 SVN_ERR(get_changes(&changes, fs, rev, scratch_pool)); 6388 6389 changed_paths = svn_hash__make(pool); 6390 6391 SVN_ERR(process_changes(changed_paths, copyfrom_cache, changes, 6392 TRUE, pool)); 6393 svn_pool_destroy(scratch_pool); 6394 6395 *changed_paths_p = changed_paths; 6396 6397 return SVN_NO_ERROR; 6398} 6399 6400/* Copy a revision node-rev SRC into the current transaction TXN_ID in 6401 the filesystem FS. This is only used to create the root of a transaction. 6402 Allocations are from POOL. */ 6403static svn_error_t * 6404create_new_txn_noderev_from_rev(svn_fs_t *fs, 6405 const char *txn_id, 6406 svn_fs_id_t *src, 6407 apr_pool_t *pool) 6408{ 6409 node_revision_t *noderev; 6410 const char *node_id, *copy_id; 6411 6412 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, src, pool)); 6413 6414 if (svn_fs_fs__id_txn_id(noderev->id)) 6415 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6416 _("Copying from transactions not allowed")); 6417 6418 noderev->predecessor_id = noderev->id; 6419 noderev->predecessor_count++; 6420 noderev->copyfrom_path = NULL; 6421 noderev->copyfrom_rev = SVN_INVALID_REVNUM; 6422 6423 /* For the transaction root, the copyroot never changes. */ 6424 6425 node_id = svn_fs_fs__id_node_id(noderev->id); 6426 copy_id = svn_fs_fs__id_copy_id(noderev->id); 6427 noderev->id = svn_fs_fs__id_txn_create(node_id, copy_id, txn_id, pool); 6428 6429 return svn_fs_fs__put_node_revision(fs, noderev->id, noderev, TRUE, pool); 6430} 6431 6432/* A structure used by get_and_increment_txn_key_body(). */ 6433struct get_and_increment_txn_key_baton { 6434 svn_fs_t *fs; 6435 char *txn_id; 6436 apr_pool_t *pool; 6437}; 6438 6439/* Callback used in the implementation of create_txn_dir(). This gets 6440 the current base 36 value in PATH_TXN_CURRENT and increments it. 6441 It returns the original value by the baton. */ 6442static svn_error_t * 6443get_and_increment_txn_key_body(void *baton, apr_pool_t *pool) 6444{ 6445 struct get_and_increment_txn_key_baton *cb = baton; 6446 const char *txn_current_filename = path_txn_current(cb->fs, pool); 6447 const char *tmp_filename; 6448 char next_txn_id[MAX_KEY_SIZE+3]; 6449 apr_size_t len; 6450 6451 svn_stringbuf_t *buf; 6452 SVN_ERR(read_content(&buf, txn_current_filename, cb->pool)); 6453 6454 /* remove trailing newlines */ 6455 svn_stringbuf_strip_whitespace(buf); 6456 cb->txn_id = buf->data; 6457 len = buf->len; 6458 6459 /* Increment the key and add a trailing \n to the string so the 6460 txn-current file has a newline in it. */ 6461 svn_fs_fs__next_key(cb->txn_id, &len, next_txn_id); 6462 next_txn_id[len] = '\n'; 6463 ++len; 6464 next_txn_id[len] = '\0'; 6465 6466 SVN_ERR(svn_io_write_unique(&tmp_filename, 6467 svn_dirent_dirname(txn_current_filename, pool), 6468 next_txn_id, len, svn_io_file_del_none, pool)); 6469 SVN_ERR(move_into_place(tmp_filename, txn_current_filename, 6470 txn_current_filename, pool)); 6471 6472 return SVN_NO_ERROR; 6473} 6474 6475/* Create a unique directory for a transaction in FS based on revision 6476 REV. Return the ID for this transaction in *ID_P. Use a sequence 6477 value in the transaction ID to prevent reuse of transaction IDs. */ 6478static svn_error_t * 6479create_txn_dir(const char **id_p, svn_fs_t *fs, svn_revnum_t rev, 6480 apr_pool_t *pool) 6481{ 6482 struct get_and_increment_txn_key_baton cb; 6483 const char *txn_dir; 6484 6485 /* Get the current transaction sequence value, which is a base-36 6486 number, from the txn-current file, and write an 6487 incremented value back out to the file. Place the revision 6488 number the transaction is based off into the transaction id. */ 6489 cb.pool = pool; 6490 cb.fs = fs; 6491 SVN_ERR(with_txn_current_lock(fs, 6492 get_and_increment_txn_key_body, 6493 &cb, 6494 pool)); 6495 *id_p = apr_psprintf(pool, "%ld-%s", rev, cb.txn_id); 6496 6497 txn_dir = svn_dirent_join_many(pool, 6498 fs->path, 6499 PATH_TXNS_DIR, 6500 apr_pstrcat(pool, *id_p, PATH_EXT_TXN, 6501 (char *)NULL), 6502 NULL); 6503 6504 return svn_io_dir_make(txn_dir, APR_OS_DEFAULT, pool); 6505} 6506 6507/* Create a unique directory for a transaction in FS based on revision 6508 REV. Return the ID for this transaction in *ID_P. This 6509 implementation is used in svn 1.4 and earlier repositories and is 6510 kept in 1.5 and greater to support the --pre-1.4-compatible and 6511 --pre-1.5-compatible repository creation options. Reused 6512 transaction IDs are possible with this implementation. */ 6513static svn_error_t * 6514create_txn_dir_pre_1_5(const char **id_p, svn_fs_t *fs, svn_revnum_t rev, 6515 apr_pool_t *pool) 6516{ 6517 unsigned int i; 6518 apr_pool_t *subpool; 6519 const char *unique_path, *prefix; 6520 6521 /* Try to create directories named "<txndir>/<rev>-<uniqueifier>.txn". */ 6522 prefix = svn_dirent_join_many(pool, fs->path, PATH_TXNS_DIR, 6523 apr_psprintf(pool, "%ld", rev), NULL); 6524 6525 subpool = svn_pool_create(pool); 6526 for (i = 1; i <= 99999; i++) 6527 { 6528 svn_error_t *err; 6529 6530 svn_pool_clear(subpool); 6531 unique_path = apr_psprintf(subpool, "%s-%u" PATH_EXT_TXN, prefix, i); 6532 err = svn_io_dir_make(unique_path, APR_OS_DEFAULT, subpool); 6533 if (! err) 6534 { 6535 /* We succeeded. Return the basename minus the ".txn" extension. */ 6536 const char *name = svn_dirent_basename(unique_path, subpool); 6537 *id_p = apr_pstrndup(pool, name, 6538 strlen(name) - strlen(PATH_EXT_TXN)); 6539 svn_pool_destroy(subpool); 6540 return SVN_NO_ERROR; 6541 } 6542 if (! APR_STATUS_IS_EEXIST(err->apr_err)) 6543 return svn_error_trace(err); 6544 svn_error_clear(err); 6545 } 6546 6547 return svn_error_createf(SVN_ERR_IO_UNIQUE_NAMES_EXHAUSTED, 6548 NULL, 6549 _("Unable to create transaction directory " 6550 "in '%s' for revision %ld"), 6551 svn_dirent_local_style(fs->path, pool), 6552 rev); 6553} 6554 6555svn_error_t * 6556svn_fs_fs__create_txn(svn_fs_txn_t **txn_p, 6557 svn_fs_t *fs, 6558 svn_revnum_t rev, 6559 apr_pool_t *pool) 6560{ 6561 fs_fs_data_t *ffd = fs->fsap_data; 6562 svn_fs_txn_t *txn; 6563 svn_fs_id_t *root_id; 6564 6565 txn = apr_pcalloc(pool, sizeof(*txn)); 6566 6567 /* Get the txn_id. */ 6568 if (ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) 6569 SVN_ERR(create_txn_dir(&txn->id, fs, rev, pool)); 6570 else 6571 SVN_ERR(create_txn_dir_pre_1_5(&txn->id, fs, rev, pool)); 6572 6573 txn->fs = fs; 6574 txn->base_rev = rev; 6575 6576 txn->vtable = &txn_vtable; 6577 *txn_p = txn; 6578 6579 /* Create a new root node for this transaction. */ 6580 SVN_ERR(svn_fs_fs__rev_get_root(&root_id, fs, rev, pool)); 6581 SVN_ERR(create_new_txn_noderev_from_rev(fs, txn->id, root_id, pool)); 6582 6583 /* Create an empty rev file. */ 6584 SVN_ERR(svn_io_file_create(path_txn_proto_rev(fs, txn->id, pool), "", 6585 pool)); 6586 6587 /* Create an empty rev-lock file. */ 6588 SVN_ERR(svn_io_file_create(path_txn_proto_rev_lock(fs, txn->id, pool), "", 6589 pool)); 6590 6591 /* Create an empty changes file. */ 6592 SVN_ERR(svn_io_file_create(path_txn_changes(fs, txn->id, pool), "", 6593 pool)); 6594 6595 /* Create the next-ids file. */ 6596 return svn_io_file_create(path_txn_next_ids(fs, txn->id, pool), "0 0\n", 6597 pool); 6598} 6599 6600/* Store the property list for transaction TXN_ID in PROPLIST. 6601 Perform temporary allocations in POOL. */ 6602static svn_error_t * 6603get_txn_proplist(apr_hash_t *proplist, 6604 svn_fs_t *fs, 6605 const char *txn_id, 6606 apr_pool_t *pool) 6607{ 6608 svn_stream_t *stream; 6609 6610 /* Check for issue #3696. (When we find and fix the cause, we can change 6611 * this to an assertion.) */ 6612 if (txn_id == NULL) 6613 return svn_error_create(SVN_ERR_INCORRECT_PARAMS, NULL, 6614 _("Internal error: a null transaction id was " 6615 "passed to get_txn_proplist()")); 6616 6617 /* Open the transaction properties file. */ 6618 SVN_ERR(svn_stream_open_readonly(&stream, path_txn_props(fs, txn_id, pool), 6619 pool, pool)); 6620 6621 /* Read in the property list. */ 6622 SVN_ERR(svn_hash_read2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 6623 6624 return svn_stream_close(stream); 6625} 6626 6627svn_error_t * 6628svn_fs_fs__change_txn_prop(svn_fs_txn_t *txn, 6629 const char *name, 6630 const svn_string_t *value, 6631 apr_pool_t *pool) 6632{ 6633 apr_array_header_t *props = apr_array_make(pool, 1, sizeof(svn_prop_t)); 6634 svn_prop_t prop; 6635 6636 prop.name = name; 6637 prop.value = value; 6638 APR_ARRAY_PUSH(props, svn_prop_t) = prop; 6639 6640 return svn_fs_fs__change_txn_props(txn, props, pool); 6641} 6642 6643svn_error_t * 6644svn_fs_fs__change_txn_props(svn_fs_txn_t *txn, 6645 const apr_array_header_t *props, 6646 apr_pool_t *pool) 6647{ 6648 const char *txn_prop_filename; 6649 svn_stringbuf_t *buf; 6650 svn_stream_t *stream; 6651 apr_hash_t *txn_prop = apr_hash_make(pool); 6652 int i; 6653 svn_error_t *err; 6654 6655 err = get_txn_proplist(txn_prop, txn->fs, txn->id, pool); 6656 /* Here - and here only - we need to deal with the possibility that the 6657 transaction property file doesn't yet exist. The rest of the 6658 implementation assumes that the file exists, but we're called to set the 6659 initial transaction properties as the transaction is being created. */ 6660 if (err && (APR_STATUS_IS_ENOENT(err->apr_err))) 6661 svn_error_clear(err); 6662 else if (err) 6663 return svn_error_trace(err); 6664 6665 for (i = 0; i < props->nelts; i++) 6666 { 6667 svn_prop_t *prop = &APR_ARRAY_IDX(props, i, svn_prop_t); 6668 6669 svn_hash_sets(txn_prop, prop->name, prop->value); 6670 } 6671 6672 /* Create a new version of the file and write out the new props. */ 6673 /* Open the transaction properties file. */ 6674 buf = svn_stringbuf_create_ensure(1024, pool); 6675 stream = svn_stream_from_stringbuf(buf, pool); 6676 SVN_ERR(svn_hash_write2(txn_prop, stream, SVN_HASH_TERMINATOR, pool)); 6677 SVN_ERR(svn_stream_close(stream)); 6678 SVN_ERR(svn_io_write_unique(&txn_prop_filename, 6679 path_txn_dir(txn->fs, txn->id, pool), 6680 buf->data, 6681 buf->len, 6682 svn_io_file_del_none, 6683 pool)); 6684 return svn_io_file_rename(txn_prop_filename, 6685 path_txn_props(txn->fs, txn->id, pool), 6686 pool); 6687} 6688 6689svn_error_t * 6690svn_fs_fs__get_txn(transaction_t **txn_p, 6691 svn_fs_t *fs, 6692 const char *txn_id, 6693 apr_pool_t *pool) 6694{ 6695 transaction_t *txn; 6696 node_revision_t *noderev; 6697 svn_fs_id_t *root_id; 6698 6699 txn = apr_pcalloc(pool, sizeof(*txn)); 6700 txn->proplist = apr_hash_make(pool); 6701 6702 SVN_ERR(get_txn_proplist(txn->proplist, fs, txn_id, pool)); 6703 root_id = svn_fs_fs__id_txn_create("0", "0", txn_id, pool); 6704 6705 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, root_id, pool)); 6706 6707 txn->root_id = svn_fs_fs__id_copy(noderev->id, pool); 6708 txn->base_id = svn_fs_fs__id_copy(noderev->predecessor_id, pool); 6709 txn->copies = NULL; 6710 6711 *txn_p = txn; 6712 6713 return SVN_NO_ERROR; 6714} 6715 6716/* Write out the currently available next node_id NODE_ID and copy_id 6717 COPY_ID for transaction TXN_ID in filesystem FS. The next node-id is 6718 used both for creating new unique nodes for the given transaction, as 6719 well as uniquifying representations. Perform temporary allocations in 6720 POOL. */ 6721static svn_error_t * 6722write_next_ids(svn_fs_t *fs, 6723 const char *txn_id, 6724 const char *node_id, 6725 const char *copy_id, 6726 apr_pool_t *pool) 6727{ 6728 apr_file_t *file; 6729 svn_stream_t *out_stream; 6730 6731 SVN_ERR(svn_io_file_open(&file, path_txn_next_ids(fs, txn_id, pool), 6732 APR_WRITE | APR_TRUNCATE, 6733 APR_OS_DEFAULT, pool)); 6734 6735 out_stream = svn_stream_from_aprfile2(file, TRUE, pool); 6736 6737 SVN_ERR(svn_stream_printf(out_stream, pool, "%s %s\n", node_id, copy_id)); 6738 6739 SVN_ERR(svn_stream_close(out_stream)); 6740 return svn_io_file_close(file, pool); 6741} 6742 6743/* Find out what the next unique node-id and copy-id are for 6744 transaction TXN_ID in filesystem FS. Store the results in *NODE_ID 6745 and *COPY_ID. The next node-id is used both for creating new unique 6746 nodes for the given transaction, as well as uniquifying representations. 6747 Perform all allocations in POOL. */ 6748static svn_error_t * 6749read_next_ids(const char **node_id, 6750 const char **copy_id, 6751 svn_fs_t *fs, 6752 const char *txn_id, 6753 apr_pool_t *pool) 6754{ 6755 apr_file_t *file; 6756 char buf[MAX_KEY_SIZE*2+3]; 6757 apr_size_t limit; 6758 char *str, *last_str = buf; 6759 6760 SVN_ERR(svn_io_file_open(&file, path_txn_next_ids(fs, txn_id, pool), 6761 APR_READ | APR_BUFFERED, APR_OS_DEFAULT, pool)); 6762 6763 limit = sizeof(buf); 6764 SVN_ERR(svn_io_read_length_line(file, buf, &limit, pool)); 6765 6766 SVN_ERR(svn_io_file_close(file, pool)); 6767 6768 /* Parse this into two separate strings. */ 6769 6770 str = svn_cstring_tokenize(" ", &last_str); 6771 if (! str) 6772 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6773 _("next-id file corrupt")); 6774 6775 *node_id = apr_pstrdup(pool, str); 6776 6777 str = svn_cstring_tokenize(" ", &last_str); 6778 if (! str) 6779 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 6780 _("next-id file corrupt")); 6781 6782 *copy_id = apr_pstrdup(pool, str); 6783 6784 return SVN_NO_ERROR; 6785} 6786 6787/* Get a new and unique to this transaction node-id for transaction 6788 TXN_ID in filesystem FS. Store the new node-id in *NODE_ID_P. 6789 Node-ids are guaranteed to be unique to this transction, but may 6790 not necessarily be sequential. Perform all allocations in POOL. */ 6791static svn_error_t * 6792get_new_txn_node_id(const char **node_id_p, 6793 svn_fs_t *fs, 6794 const char *txn_id, 6795 apr_pool_t *pool) 6796{ 6797 const char *cur_node_id, *cur_copy_id; 6798 char *node_id; 6799 apr_size_t len; 6800 6801 /* First read in the current next-ids file. */ 6802 SVN_ERR(read_next_ids(&cur_node_id, &cur_copy_id, fs, txn_id, pool)); 6803 6804 node_id = apr_pcalloc(pool, strlen(cur_node_id) + 2); 6805 6806 len = strlen(cur_node_id); 6807 svn_fs_fs__next_key(cur_node_id, &len, node_id); 6808 6809 SVN_ERR(write_next_ids(fs, txn_id, node_id, cur_copy_id, pool)); 6810 6811 *node_id_p = apr_pstrcat(pool, "_", cur_node_id, (char *)NULL); 6812 6813 return SVN_NO_ERROR; 6814} 6815 6816svn_error_t * 6817svn_fs_fs__create_node(const svn_fs_id_t **id_p, 6818 svn_fs_t *fs, 6819 node_revision_t *noderev, 6820 const char *copy_id, 6821 const char *txn_id, 6822 apr_pool_t *pool) 6823{ 6824 const char *node_id; 6825 const svn_fs_id_t *id; 6826 6827 /* Get a new node-id for this node. */ 6828 SVN_ERR(get_new_txn_node_id(&node_id, fs, txn_id, pool)); 6829 6830 id = svn_fs_fs__id_txn_create(node_id, copy_id, txn_id, pool); 6831 6832 noderev->id = id; 6833 6834 SVN_ERR(svn_fs_fs__put_node_revision(fs, noderev->id, noderev, FALSE, pool)); 6835 6836 *id_p = id; 6837 6838 return SVN_NO_ERROR; 6839} 6840 6841svn_error_t * 6842svn_fs_fs__purge_txn(svn_fs_t *fs, 6843 const char *txn_id, 6844 apr_pool_t *pool) 6845{ 6846 fs_fs_data_t *ffd = fs->fsap_data; 6847 6848 /* Remove the shared transaction object associated with this transaction. */ 6849 SVN_ERR(purge_shared_txn(fs, txn_id, pool)); 6850 /* Remove the directory associated with this transaction. */ 6851 SVN_ERR(svn_io_remove_dir2(path_txn_dir(fs, txn_id, pool), FALSE, 6852 NULL, NULL, pool)); 6853 if (ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) 6854 { 6855 /* Delete protorev and its lock, which aren't in the txn 6856 directory. It's OK if they don't exist (for example, if this 6857 is post-commit and the proto-rev has been moved into 6858 place). */ 6859 SVN_ERR(svn_io_remove_file2(path_txn_proto_rev(fs, txn_id, pool), 6860 TRUE, pool)); 6861 SVN_ERR(svn_io_remove_file2(path_txn_proto_rev_lock(fs, txn_id, pool), 6862 TRUE, pool)); 6863 } 6864 return SVN_NO_ERROR; 6865} 6866 6867 6868svn_error_t * 6869svn_fs_fs__abort_txn(svn_fs_txn_t *txn, 6870 apr_pool_t *pool) 6871{ 6872 SVN_ERR(svn_fs__check_fs(txn->fs, TRUE)); 6873 6874 /* Now, purge the transaction. */ 6875 SVN_ERR_W(svn_fs_fs__purge_txn(txn->fs, txn->id, pool), 6876 apr_psprintf(pool, _("Transaction '%s' cleanup failed"), 6877 txn->id)); 6878 6879 return SVN_NO_ERROR; 6880} 6881 6882 6883svn_error_t * 6884svn_fs_fs__set_entry(svn_fs_t *fs, 6885 const char *txn_id, 6886 node_revision_t *parent_noderev, 6887 const char *name, 6888 const svn_fs_id_t *id, 6889 svn_node_kind_t kind, 6890 apr_pool_t *pool) 6891{ 6892 representation_t *rep = parent_noderev->data_rep; 6893 const char *filename = path_txn_node_children(fs, parent_noderev->id, pool); 6894 apr_file_t *file; 6895 svn_stream_t *out; 6896 fs_fs_data_t *ffd = fs->fsap_data; 6897 apr_pool_t *subpool = svn_pool_create(pool); 6898 6899 if (!rep || !rep->txn_id) 6900 { 6901 const char *unique_suffix; 6902 apr_hash_t *entries; 6903 6904 /* Before we can modify the directory, we need to dump its old 6905 contents into a mutable representation file. */ 6906 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, parent_noderev, 6907 subpool)); 6908 SVN_ERR(unparse_dir_entries(&entries, entries, subpool)); 6909 SVN_ERR(svn_io_file_open(&file, filename, 6910 APR_WRITE | APR_CREATE | APR_BUFFERED, 6911 APR_OS_DEFAULT, pool)); 6912 out = svn_stream_from_aprfile2(file, TRUE, pool); 6913 SVN_ERR(svn_hash_write2(entries, out, SVN_HASH_TERMINATOR, subpool)); 6914 6915 svn_pool_clear(subpool); 6916 6917 /* Mark the node-rev's data rep as mutable. */ 6918 rep = apr_pcalloc(pool, sizeof(*rep)); 6919 rep->revision = SVN_INVALID_REVNUM; 6920 rep->txn_id = txn_id; 6921 SVN_ERR(get_new_txn_node_id(&unique_suffix, fs, txn_id, pool)); 6922 rep->uniquifier = apr_psprintf(pool, "%s/%s", txn_id, unique_suffix); 6923 parent_noderev->data_rep = rep; 6924 SVN_ERR(svn_fs_fs__put_node_revision(fs, parent_noderev->id, 6925 parent_noderev, FALSE, pool)); 6926 } 6927 else 6928 { 6929 /* The directory rep is already mutable, so just open it for append. */ 6930 SVN_ERR(svn_io_file_open(&file, filename, APR_WRITE | APR_APPEND, 6931 APR_OS_DEFAULT, pool)); 6932 out = svn_stream_from_aprfile2(file, TRUE, pool); 6933 } 6934 6935 /* if we have a directory cache for this transaction, update it */ 6936 if (ffd->txn_dir_cache) 6937 { 6938 /* build parameters: (name, new entry) pair */ 6939 const char *key = 6940 svn_fs_fs__id_unparse(parent_noderev->id, subpool)->data; 6941 replace_baton_t baton; 6942 6943 baton.name = name; 6944 baton.new_entry = NULL; 6945 6946 if (id) 6947 { 6948 baton.new_entry = apr_pcalloc(subpool, sizeof(*baton.new_entry)); 6949 baton.new_entry->name = name; 6950 baton.new_entry->kind = kind; 6951 baton.new_entry->id = id; 6952 } 6953 6954 /* actually update the cached directory (if cached) */ 6955 SVN_ERR(svn_cache__set_partial(ffd->txn_dir_cache, key, 6956 svn_fs_fs__replace_dir_entry, &baton, 6957 subpool)); 6958 } 6959 svn_pool_clear(subpool); 6960 6961 /* Append an incremental hash entry for the entry change. */ 6962 if (id) 6963 { 6964 const char *val = unparse_dir_entry(kind, id, subpool); 6965 6966 SVN_ERR(svn_stream_printf(out, subpool, "K %" APR_SIZE_T_FMT "\n%s\n" 6967 "V %" APR_SIZE_T_FMT "\n%s\n", 6968 strlen(name), name, 6969 strlen(val), val)); 6970 } 6971 else 6972 { 6973 SVN_ERR(svn_stream_printf(out, subpool, "D %" APR_SIZE_T_FMT "\n%s\n", 6974 strlen(name), name)); 6975 } 6976 6977 SVN_ERR(svn_io_file_close(file, subpool)); 6978 svn_pool_destroy(subpool); 6979 return SVN_NO_ERROR; 6980} 6981 6982/* Write a single change entry, path PATH, change CHANGE, and copyfrom 6983 string COPYFROM, into the file specified by FILE. Only include the 6984 node kind field if INCLUDE_NODE_KIND is true. All temporary 6985 allocations are in POOL. */ 6986static svn_error_t * 6987write_change_entry(apr_file_t *file, 6988 const char *path, 6989 svn_fs_path_change2_t *change, 6990 svn_boolean_t include_node_kind, 6991 apr_pool_t *pool) 6992{ 6993 const char *idstr, *buf; 6994 const char *change_string = NULL; 6995 const char *kind_string = ""; 6996 6997 switch (change->change_kind) 6998 { 6999 case svn_fs_path_change_modify: 7000 change_string = ACTION_MODIFY; 7001 break; 7002 case svn_fs_path_change_add: 7003 change_string = ACTION_ADD; 7004 break; 7005 case svn_fs_path_change_delete: 7006 change_string = ACTION_DELETE; 7007 break; 7008 case svn_fs_path_change_replace: 7009 change_string = ACTION_REPLACE; 7010 break; 7011 case svn_fs_path_change_reset: 7012 change_string = ACTION_RESET; 7013 break; 7014 default: 7015 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 7016 _("Invalid change type %d"), 7017 change->change_kind); 7018 } 7019 7020 if (change->node_rev_id) 7021 idstr = svn_fs_fs__id_unparse(change->node_rev_id, pool)->data; 7022 else 7023 idstr = ACTION_RESET; 7024 7025 if (include_node_kind) 7026 { 7027 SVN_ERR_ASSERT(change->node_kind == svn_node_dir 7028 || change->node_kind == svn_node_file); 7029 kind_string = apr_psprintf(pool, "-%s", 7030 change->node_kind == svn_node_dir 7031 ? KIND_DIR : KIND_FILE); 7032 } 7033 buf = apr_psprintf(pool, "%s %s%s %s %s %s\n", 7034 idstr, change_string, kind_string, 7035 change->text_mod ? FLAG_TRUE : FLAG_FALSE, 7036 change->prop_mod ? FLAG_TRUE : FLAG_FALSE, 7037 path); 7038 7039 SVN_ERR(svn_io_file_write_full(file, buf, strlen(buf), NULL, pool)); 7040 7041 if (SVN_IS_VALID_REVNUM(change->copyfrom_rev)) 7042 { 7043 buf = apr_psprintf(pool, "%ld %s", change->copyfrom_rev, 7044 change->copyfrom_path); 7045 SVN_ERR(svn_io_file_write_full(file, buf, strlen(buf), NULL, pool)); 7046 } 7047 7048 return svn_io_file_write_full(file, "\n", 1, NULL, pool); 7049} 7050 7051svn_error_t * 7052svn_fs_fs__add_change(svn_fs_t *fs, 7053 const char *txn_id, 7054 const char *path, 7055 const svn_fs_id_t *id, 7056 svn_fs_path_change_kind_t change_kind, 7057 svn_boolean_t text_mod, 7058 svn_boolean_t prop_mod, 7059 svn_node_kind_t node_kind, 7060 svn_revnum_t copyfrom_rev, 7061 const char *copyfrom_path, 7062 apr_pool_t *pool) 7063{ 7064 apr_file_t *file; 7065 svn_fs_path_change2_t *change; 7066 7067 SVN_ERR(svn_io_file_open(&file, path_txn_changes(fs, txn_id, pool), 7068 APR_APPEND | APR_WRITE | APR_CREATE 7069 | APR_BUFFERED, APR_OS_DEFAULT, pool)); 7070 7071 change = svn_fs__path_change_create_internal(id, change_kind, pool); 7072 change->text_mod = text_mod; 7073 change->prop_mod = prop_mod; 7074 change->node_kind = node_kind; 7075 change->copyfrom_rev = copyfrom_rev; 7076 change->copyfrom_path = apr_pstrdup(pool, copyfrom_path); 7077 7078 SVN_ERR(write_change_entry(file, path, change, TRUE, pool)); 7079 7080 return svn_io_file_close(file, pool); 7081} 7082 7083/* This baton is used by the representation writing streams. It keeps 7084 track of the checksum information as well as the total size of the 7085 representation so far. */ 7086struct rep_write_baton 7087{ 7088 /* The FS we are writing to. */ 7089 svn_fs_t *fs; 7090 7091 /* Actual file to which we are writing. */ 7092 svn_stream_t *rep_stream; 7093 7094 /* A stream from the delta combiner. Data written here gets 7095 deltified, then eventually written to rep_stream. */ 7096 svn_stream_t *delta_stream; 7097 7098 /* Where is this representation header stored. */ 7099 apr_off_t rep_offset; 7100 7101 /* Start of the actual data. */ 7102 apr_off_t delta_start; 7103 7104 /* How many bytes have been written to this rep already. */ 7105 svn_filesize_t rep_size; 7106 7107 /* The node revision for which we're writing out info. */ 7108 node_revision_t *noderev; 7109 7110 /* Actual output file. */ 7111 apr_file_t *file; 7112 /* Lock 'cookie' used to unlock the output file once we've finished 7113 writing to it. */ 7114 void *lockcookie; 7115 7116 svn_checksum_ctx_t *md5_checksum_ctx; 7117 svn_checksum_ctx_t *sha1_checksum_ctx; 7118 7119 apr_pool_t *pool; 7120 7121 apr_pool_t *parent_pool; 7122}; 7123 7124/* Handler for the write method of the representation writable stream. 7125 BATON is a rep_write_baton, DATA is the data to write, and *LEN is 7126 the length of this data. */ 7127static svn_error_t * 7128rep_write_contents(void *baton, 7129 const char *data, 7130 apr_size_t *len) 7131{ 7132 struct rep_write_baton *b = baton; 7133 7134 SVN_ERR(svn_checksum_update(b->md5_checksum_ctx, data, *len)); 7135 SVN_ERR(svn_checksum_update(b->sha1_checksum_ctx, data, *len)); 7136 b->rep_size += *len; 7137 7138 /* If we are writing a delta, use that stream. */ 7139 if (b->delta_stream) 7140 return svn_stream_write(b->delta_stream, data, len); 7141 else 7142 return svn_stream_write(b->rep_stream, data, len); 7143} 7144 7145/* Given a node-revision NODEREV in filesystem FS, return the 7146 representation in *REP to use as the base for a text representation 7147 delta if PROPS is FALSE. If PROPS has been set, a suitable props 7148 base representation will be returned. Perform temporary allocations 7149 in *POOL. */ 7150static svn_error_t * 7151choose_delta_base(representation_t **rep, 7152 svn_fs_t *fs, 7153 node_revision_t *noderev, 7154 svn_boolean_t props, 7155 apr_pool_t *pool) 7156{ 7157 int count; 7158 int walk; 7159 node_revision_t *base; 7160 fs_fs_data_t *ffd = fs->fsap_data; 7161 svn_boolean_t maybe_shared_rep = FALSE; 7162 7163 /* If we have no predecessors, then use the empty stream as a 7164 base. */ 7165 if (! noderev->predecessor_count) 7166 { 7167 *rep = NULL; 7168 return SVN_NO_ERROR; 7169 } 7170 7171 /* Flip the rightmost '1' bit of the predecessor count to determine 7172 which file rev (counting from 0) we want to use. (To see why 7173 count & (count - 1) unsets the rightmost set bit, think about how 7174 you decrement a binary number.) */ 7175 count = noderev->predecessor_count; 7176 count = count & (count - 1); 7177 7178 /* We use skip delta for limiting the number of delta operations 7179 along very long node histories. Close to HEAD however, we create 7180 a linear history to minimize delta size. */ 7181 walk = noderev->predecessor_count - count; 7182 if (walk < (int)ffd->max_linear_deltification) 7183 count = noderev->predecessor_count - 1; 7184 7185 /* Finding the delta base over a very long distance can become extremely 7186 expensive for very deep histories, possibly causing client timeouts etc. 7187 OTOH, this is a rare operation and its gains are minimal. Lets simply 7188 start deltification anew close every other 1000 changes or so. */ 7189 if (walk > (int)ffd->max_deltification_walk) 7190 { 7191 *rep = NULL; 7192 return SVN_NO_ERROR; 7193 } 7194 7195 /* Walk back a number of predecessors equal to the difference 7196 between count and the original predecessor count. (For example, 7197 if noderev has ten predecessors and we want the eighth file rev, 7198 walk back two predecessors.) */ 7199 base = noderev; 7200 while ((count++) < noderev->predecessor_count) 7201 { 7202 SVN_ERR(svn_fs_fs__get_node_revision(&base, fs, 7203 base->predecessor_id, pool)); 7204 7205 /* If there is a shared rep along the way, we need to limit the 7206 * length of the deltification chain. 7207 * 7208 * Please note that copied nodes - such as branch directories - will 7209 * look the same (false positive) while reps shared within the same 7210 * revision will not be caught (false negative). 7211 */ 7212 if (props) 7213 { 7214 if ( base->prop_rep 7215 && svn_fs_fs__id_rev(base->id) > base->prop_rep->revision) 7216 maybe_shared_rep = TRUE; 7217 } 7218 else 7219 { 7220 if ( base->data_rep 7221 && svn_fs_fs__id_rev(base->id) > base->data_rep->revision) 7222 maybe_shared_rep = TRUE; 7223 } 7224 } 7225 7226 /* return a suitable base representation */ 7227 *rep = props ? base->prop_rep : base->data_rep; 7228 7229 /* if we encountered a shared rep, it's parent chain may be different 7230 * from the node-rev parent chain. */ 7231 if (*rep && maybe_shared_rep) 7232 { 7233 /* Check whether the length of the deltification chain is acceptable. 7234 * Otherwise, shared reps may form a non-skipping delta chain in 7235 * extreme cases. */ 7236 apr_pool_t *sub_pool = svn_pool_create(pool); 7237 representation_t base_rep = **rep; 7238 7239 /* Some reasonable limit, depending on how acceptable longer linear 7240 * chains are in this repo. Also, allow for some minimal chain. */ 7241 int max_chain_length = 2 * (int)ffd->max_linear_deltification + 2; 7242 7243 /* re-use open files between iterations */ 7244 svn_revnum_t rev_hint = SVN_INVALID_REVNUM; 7245 apr_file_t *file_hint = NULL; 7246 7247 /* follow the delta chain towards the end but for at most 7248 * MAX_CHAIN_LENGTH steps. */ 7249 for (; max_chain_length; --max_chain_length) 7250 { 7251 struct rep_state *rep_state; 7252 struct rep_args *rep_args; 7253 7254 SVN_ERR(create_rep_state_body(&rep_state, 7255 &rep_args, 7256 &file_hint, 7257 &rev_hint, 7258 &base_rep, 7259 fs, 7260 sub_pool)); 7261 if (!rep_args->is_delta || !rep_args->base_revision) 7262 break; 7263 7264 base_rep.revision = rep_args->base_revision; 7265 base_rep.offset = rep_args->base_offset; 7266 base_rep.size = rep_args->base_length; 7267 base_rep.txn_id = NULL; 7268 } 7269 7270 /* start new delta chain if the current one has grown too long */ 7271 if (max_chain_length == 0) 7272 *rep = NULL; 7273 7274 svn_pool_destroy(sub_pool); 7275 } 7276 7277 /* verify that the reps don't form a degenerated '*/ 7278 return SVN_NO_ERROR; 7279} 7280 7281/* Something went wrong and the pool for the rep write is being 7282 cleared before we've finished writing the rep. So we need 7283 to remove the rep from the protorevfile and we need to unlock 7284 the protorevfile. */ 7285static apr_status_t 7286rep_write_cleanup(void *data) 7287{ 7288 struct rep_write_baton *b = data; 7289 const char *txn_id = svn_fs_fs__id_txn_id(b->noderev->id); 7290 svn_error_t *err; 7291 7292 /* Truncate and close the protorevfile. */ 7293 err = svn_io_file_trunc(b->file, b->rep_offset, b->pool); 7294 err = svn_error_compose_create(err, svn_io_file_close(b->file, b->pool)); 7295 7296 /* Remove our lock regardless of any preceeding errors so that the 7297 being_written flag is always removed and stays consistent with the 7298 file lock which will be removed no matter what since the pool is 7299 going away. */ 7300 err = svn_error_compose_create(err, unlock_proto_rev(b->fs, txn_id, 7301 b->lockcookie, b->pool)); 7302 if (err) 7303 { 7304 apr_status_t rc = err->apr_err; 7305 svn_error_clear(err); 7306 return rc; 7307 } 7308 7309 return APR_SUCCESS; 7310} 7311 7312 7313/* Get a rep_write_baton and store it in *WB_P for the representation 7314 indicated by NODEREV in filesystem FS. Perform allocations in 7315 POOL. Only appropriate for file contents, not for props or 7316 directory contents. */ 7317static svn_error_t * 7318rep_write_get_baton(struct rep_write_baton **wb_p, 7319 svn_fs_t *fs, 7320 node_revision_t *noderev, 7321 apr_pool_t *pool) 7322{ 7323 struct rep_write_baton *b; 7324 apr_file_t *file; 7325 representation_t *base_rep; 7326 svn_stream_t *source; 7327 const char *header; 7328 svn_txdelta_window_handler_t wh; 7329 void *whb; 7330 fs_fs_data_t *ffd = fs->fsap_data; 7331 int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0; 7332 7333 b = apr_pcalloc(pool, sizeof(*b)); 7334 7335 b->sha1_checksum_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); 7336 b->md5_checksum_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); 7337 7338 b->fs = fs; 7339 b->parent_pool = pool; 7340 b->pool = svn_pool_create(pool); 7341 b->rep_size = 0; 7342 b->noderev = noderev; 7343 7344 /* Open the prototype rev file and seek to its end. */ 7345 SVN_ERR(get_writable_proto_rev(&file, &b->lockcookie, 7346 fs, svn_fs_fs__id_txn_id(noderev->id), 7347 b->pool)); 7348 7349 b->file = file; 7350 b->rep_stream = svn_stream_from_aprfile2(file, TRUE, b->pool); 7351 7352 SVN_ERR(get_file_offset(&b->rep_offset, file, b->pool)); 7353 7354 /* Get the base for this delta. */ 7355 SVN_ERR(choose_delta_base(&base_rep, fs, noderev, FALSE, b->pool)); 7356 SVN_ERR(read_representation(&source, fs, base_rep, b->pool)); 7357 7358 /* Write out the rep header. */ 7359 if (base_rep) 7360 { 7361 header = apr_psprintf(b->pool, REP_DELTA " %ld %" APR_OFF_T_FMT " %" 7362 SVN_FILESIZE_T_FMT "\n", 7363 base_rep->revision, base_rep->offset, 7364 base_rep->size); 7365 } 7366 else 7367 { 7368 header = REP_DELTA "\n"; 7369 } 7370 SVN_ERR(svn_io_file_write_full(file, header, strlen(header), NULL, 7371 b->pool)); 7372 7373 /* Now determine the offset of the actual svndiff data. */ 7374 SVN_ERR(get_file_offset(&b->delta_start, file, b->pool)); 7375 7376 /* Cleanup in case something goes wrong. */ 7377 apr_pool_cleanup_register(b->pool, b, rep_write_cleanup, 7378 apr_pool_cleanup_null); 7379 7380 /* Prepare to write the svndiff data. */ 7381 svn_txdelta_to_svndiff3(&wh, 7382 &whb, 7383 b->rep_stream, 7384 diff_version, 7385 SVN_DELTA_COMPRESSION_LEVEL_DEFAULT, 7386 pool); 7387 7388 b->delta_stream = svn_txdelta_target_push(wh, whb, source, b->pool); 7389 7390 *wb_p = b; 7391 7392 return SVN_NO_ERROR; 7393} 7394 7395/* For the hash REP->SHA1, try to find an already existing representation 7396 in FS and return it in *OUT_REP. If no such representation exists or 7397 if rep sharing has been disabled for FS, NULL will be returned. Since 7398 there may be new duplicate representations within the same uncommitted 7399 revision, those can be passed in REPS_HASH (maps a sha1 digest onto 7400 representation_t*), otherwise pass in NULL for REPS_HASH. 7401 POOL will be used for allocations. The lifetime of the returned rep is 7402 limited by both, POOL and REP lifetime. 7403 */ 7404static svn_error_t * 7405get_shared_rep(representation_t **old_rep, 7406 svn_fs_t *fs, 7407 representation_t *rep, 7408 apr_hash_t *reps_hash, 7409 apr_pool_t *pool) 7410{ 7411 svn_error_t *err; 7412 fs_fs_data_t *ffd = fs->fsap_data; 7413 7414 /* Return NULL, if rep sharing has been disabled. */ 7415 *old_rep = NULL; 7416 if (!ffd->rep_sharing_allowed) 7417 return SVN_NO_ERROR; 7418 7419 /* Check and see if we already have a representation somewhere that's 7420 identical to the one we just wrote out. Start with the hash lookup 7421 because it is cheepest. */ 7422 if (reps_hash) 7423 *old_rep = apr_hash_get(reps_hash, 7424 rep->sha1_checksum->digest, 7425 APR_SHA1_DIGESTSIZE); 7426 7427 /* If we haven't found anything yet, try harder and consult our DB. */ 7428 if (*old_rep == NULL) 7429 { 7430 err = svn_fs_fs__get_rep_reference(old_rep, fs, rep->sha1_checksum, 7431 pool); 7432 /* ### Other error codes that we shouldn't mask out? */ 7433 if (err == SVN_NO_ERROR) 7434 { 7435 if (*old_rep) 7436 SVN_ERR(verify_walker(*old_rep, NULL, fs, pool)); 7437 } 7438 else if (err->apr_err == SVN_ERR_FS_CORRUPT 7439 || SVN_ERROR_IN_CATEGORY(err->apr_err, 7440 SVN_ERR_MALFUNC_CATEGORY_START)) 7441 { 7442 /* Fatal error; don't mask it. 7443 7444 In particular, this block is triggered when the rep-cache refers 7445 to revisions in the future. We signal that as a corruption situation 7446 since, once those revisions are less than youngest (because of more 7447 commits), the rep-cache would be invalid. 7448 */ 7449 SVN_ERR(err); 7450 } 7451 else 7452 { 7453 /* Something's wrong with the rep-sharing index. We can continue 7454 without rep-sharing, but warn. 7455 */ 7456 (fs->warning)(fs->warning_baton, err); 7457 svn_error_clear(err); 7458 *old_rep = NULL; 7459 } 7460 } 7461 7462 /* look for intra-revision matches (usually data reps but not limited 7463 to them in case props happen to look like some data rep) 7464 */ 7465 if (*old_rep == NULL && rep->txn_id) 7466 { 7467 svn_node_kind_t kind; 7468 const char *file_name 7469 = path_txn_sha1(fs, rep->txn_id, rep->sha1_checksum, pool); 7470 7471 /* in our txn, is there a rep file named with the wanted SHA1? 7472 If so, read it and use that rep. 7473 */ 7474 SVN_ERR(svn_io_check_path(file_name, &kind, pool)); 7475 if (kind == svn_node_file) 7476 { 7477 svn_stringbuf_t *rep_string; 7478 SVN_ERR(svn_stringbuf_from_file2(&rep_string, file_name, pool)); 7479 SVN_ERR(read_rep_offsets_body(old_rep, rep_string->data, 7480 rep->txn_id, FALSE, pool)); 7481 } 7482 } 7483 7484 /* Add information that is missing in the cached data. */ 7485 if (*old_rep) 7486 { 7487 /* Use the old rep for this content. */ 7488 (*old_rep)->md5_checksum = rep->md5_checksum; 7489 (*old_rep)->uniquifier = rep->uniquifier; 7490 } 7491 7492 return SVN_NO_ERROR; 7493} 7494 7495/* Close handler for the representation write stream. BATON is a 7496 rep_write_baton. Writes out a new node-rev that correctly 7497 references the representation we just finished writing. */ 7498static svn_error_t * 7499rep_write_contents_close(void *baton) 7500{ 7501 struct rep_write_baton *b = baton; 7502 const char *unique_suffix; 7503 representation_t *rep; 7504 representation_t *old_rep; 7505 apr_off_t offset; 7506 7507 rep = apr_pcalloc(b->parent_pool, sizeof(*rep)); 7508 rep->offset = b->rep_offset; 7509 7510 /* Close our delta stream so the last bits of svndiff are written 7511 out. */ 7512 if (b->delta_stream) 7513 SVN_ERR(svn_stream_close(b->delta_stream)); 7514 7515 /* Determine the length of the svndiff data. */ 7516 SVN_ERR(get_file_offset(&offset, b->file, b->pool)); 7517 rep->size = offset - b->delta_start; 7518 7519 /* Fill in the rest of the representation field. */ 7520 rep->expanded_size = b->rep_size; 7521 rep->txn_id = svn_fs_fs__id_txn_id(b->noderev->id); 7522 SVN_ERR(get_new_txn_node_id(&unique_suffix, b->fs, rep->txn_id, b->pool)); 7523 rep->uniquifier = apr_psprintf(b->parent_pool, "%s/%s", rep->txn_id, 7524 unique_suffix); 7525 rep->revision = SVN_INVALID_REVNUM; 7526 7527 /* Finalize the checksum. */ 7528 SVN_ERR(svn_checksum_final(&rep->md5_checksum, b->md5_checksum_ctx, 7529 b->parent_pool)); 7530 SVN_ERR(svn_checksum_final(&rep->sha1_checksum, b->sha1_checksum_ctx, 7531 b->parent_pool)); 7532 7533 /* Check and see if we already have a representation somewhere that's 7534 identical to the one we just wrote out. */ 7535 SVN_ERR(get_shared_rep(&old_rep, b->fs, rep, NULL, b->parent_pool)); 7536 7537 if (old_rep) 7538 { 7539 /* We need to erase from the protorev the data we just wrote. */ 7540 SVN_ERR(svn_io_file_trunc(b->file, b->rep_offset, b->pool)); 7541 7542 /* Use the old rep for this content. */ 7543 b->noderev->data_rep = old_rep; 7544 } 7545 else 7546 { 7547 /* Write out our cosmetic end marker. */ 7548 SVN_ERR(svn_stream_puts(b->rep_stream, "ENDREP\n")); 7549 7550 b->noderev->data_rep = rep; 7551 } 7552 7553 /* Remove cleanup callback. */ 7554 apr_pool_cleanup_kill(b->pool, b, rep_write_cleanup); 7555 7556 /* Write out the new node-rev information. */ 7557 SVN_ERR(svn_fs_fs__put_node_revision(b->fs, b->noderev->id, b->noderev, FALSE, 7558 b->pool)); 7559 if (!old_rep) 7560 SVN_ERR(store_sha1_rep_mapping(b->fs, b->noderev, b->pool)); 7561 7562 SVN_ERR(svn_io_file_close(b->file, b->pool)); 7563 SVN_ERR(unlock_proto_rev(b->fs, rep->txn_id, b->lockcookie, b->pool)); 7564 svn_pool_destroy(b->pool); 7565 7566 return SVN_NO_ERROR; 7567} 7568 7569/* Store a writable stream in *CONTENTS_P that will receive all data 7570 written and store it as the file data representation referenced by 7571 NODEREV in filesystem FS. Perform temporary allocations in 7572 POOL. Only appropriate for file data, not props or directory 7573 contents. */ 7574static svn_error_t * 7575set_representation(svn_stream_t **contents_p, 7576 svn_fs_t *fs, 7577 node_revision_t *noderev, 7578 apr_pool_t *pool) 7579{ 7580 struct rep_write_baton *wb; 7581 7582 if (! svn_fs_fs__id_txn_id(noderev->id)) 7583 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 7584 _("Attempted to write to non-transaction '%s'"), 7585 svn_fs_fs__id_unparse(noderev->id, pool)->data); 7586 7587 SVN_ERR(rep_write_get_baton(&wb, fs, noderev, pool)); 7588 7589 *contents_p = svn_stream_create(wb, pool); 7590 svn_stream_set_write(*contents_p, rep_write_contents); 7591 svn_stream_set_close(*contents_p, rep_write_contents_close); 7592 7593 return SVN_NO_ERROR; 7594} 7595 7596svn_error_t * 7597svn_fs_fs__set_contents(svn_stream_t **stream, 7598 svn_fs_t *fs, 7599 node_revision_t *noderev, 7600 apr_pool_t *pool) 7601{ 7602 if (noderev->kind != svn_node_file) 7603 return svn_error_create(SVN_ERR_FS_NOT_FILE, NULL, 7604 _("Can't set text contents of a directory")); 7605 7606 return set_representation(stream, fs, noderev, pool); 7607} 7608 7609svn_error_t * 7610svn_fs_fs__create_successor(const svn_fs_id_t **new_id_p, 7611 svn_fs_t *fs, 7612 const svn_fs_id_t *old_idp, 7613 node_revision_t *new_noderev, 7614 const char *copy_id, 7615 const char *txn_id, 7616 apr_pool_t *pool) 7617{ 7618 const svn_fs_id_t *id; 7619 7620 if (! copy_id) 7621 copy_id = svn_fs_fs__id_copy_id(old_idp); 7622 id = svn_fs_fs__id_txn_create(svn_fs_fs__id_node_id(old_idp), copy_id, 7623 txn_id, pool); 7624 7625 new_noderev->id = id; 7626 7627 if (! new_noderev->copyroot_path) 7628 { 7629 new_noderev->copyroot_path = apr_pstrdup(pool, 7630 new_noderev->created_path); 7631 new_noderev->copyroot_rev = svn_fs_fs__id_rev(new_noderev->id); 7632 } 7633 7634 SVN_ERR(svn_fs_fs__put_node_revision(fs, new_noderev->id, new_noderev, FALSE, 7635 pool)); 7636 7637 *new_id_p = id; 7638 7639 return SVN_NO_ERROR; 7640} 7641 7642svn_error_t * 7643svn_fs_fs__set_proplist(svn_fs_t *fs, 7644 node_revision_t *noderev, 7645 apr_hash_t *proplist, 7646 apr_pool_t *pool) 7647{ 7648 const char *filename = path_txn_node_props(fs, noderev->id, pool); 7649 apr_file_t *file; 7650 svn_stream_t *out; 7651 7652 /* Dump the property list to the mutable property file. */ 7653 SVN_ERR(svn_io_file_open(&file, filename, 7654 APR_WRITE | APR_CREATE | APR_TRUNCATE 7655 | APR_BUFFERED, APR_OS_DEFAULT, pool)); 7656 out = svn_stream_from_aprfile2(file, TRUE, pool); 7657 SVN_ERR(svn_hash_write2(proplist, out, SVN_HASH_TERMINATOR, pool)); 7658 SVN_ERR(svn_io_file_close(file, pool)); 7659 7660 /* Mark the node-rev's prop rep as mutable, if not already done. */ 7661 if (!noderev->prop_rep || !noderev->prop_rep->txn_id) 7662 { 7663 noderev->prop_rep = apr_pcalloc(pool, sizeof(*noderev->prop_rep)); 7664 noderev->prop_rep->txn_id = svn_fs_fs__id_txn_id(noderev->id); 7665 SVN_ERR(svn_fs_fs__put_node_revision(fs, noderev->id, noderev, FALSE, pool)); 7666 } 7667 7668 return SVN_NO_ERROR; 7669} 7670 7671/* Read the 'current' file for filesystem FS and store the next 7672 available node id in *NODE_ID, and the next available copy id in 7673 *COPY_ID. Allocations are performed from POOL. */ 7674static svn_error_t * 7675get_next_revision_ids(const char **node_id, 7676 const char **copy_id, 7677 svn_fs_t *fs, 7678 apr_pool_t *pool) 7679{ 7680 char *buf; 7681 char *str; 7682 svn_stringbuf_t *content; 7683 7684 SVN_ERR(read_content(&content, svn_fs_fs__path_current(fs, pool), pool)); 7685 buf = content->data; 7686 7687 str = svn_cstring_tokenize(" ", &buf); 7688 if (! str) 7689 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 7690 _("Corrupt 'current' file")); 7691 7692 str = svn_cstring_tokenize(" ", &buf); 7693 if (! str) 7694 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 7695 _("Corrupt 'current' file")); 7696 7697 *node_id = apr_pstrdup(pool, str); 7698 7699 str = svn_cstring_tokenize(" \n", &buf); 7700 if (! str) 7701 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 7702 _("Corrupt 'current' file")); 7703 7704 *copy_id = apr_pstrdup(pool, str); 7705 7706 return SVN_NO_ERROR; 7707} 7708 7709/* This baton is used by the stream created for write_hash_rep. */ 7710struct write_hash_baton 7711{ 7712 svn_stream_t *stream; 7713 7714 apr_size_t size; 7715 7716 svn_checksum_ctx_t *md5_ctx; 7717 svn_checksum_ctx_t *sha1_ctx; 7718}; 7719 7720/* The handler for the write_hash_rep stream. BATON is a 7721 write_hash_baton, DATA has the data to write and *LEN is the number 7722 of bytes to write. */ 7723static svn_error_t * 7724write_hash_handler(void *baton, 7725 const char *data, 7726 apr_size_t *len) 7727{ 7728 struct write_hash_baton *whb = baton; 7729 7730 SVN_ERR(svn_checksum_update(whb->md5_ctx, data, *len)); 7731 SVN_ERR(svn_checksum_update(whb->sha1_ctx, data, *len)); 7732 7733 SVN_ERR(svn_stream_write(whb->stream, data, len)); 7734 whb->size += *len; 7735 7736 return SVN_NO_ERROR; 7737} 7738 7739/* Write out the hash HASH as a text representation to file FILE. In 7740 the process, record position, the total size of the dump and MD5 as 7741 well as SHA1 in REP. If rep sharing has been enabled and REPS_HASH 7742 is not NULL, it will be used in addition to the on-disk cache to find 7743 earlier reps with the same content. When such existing reps can be 7744 found, we will truncate the one just written from the file and return 7745 the existing rep. Perform temporary allocations in POOL. */ 7746static svn_error_t * 7747write_hash_rep(representation_t *rep, 7748 apr_file_t *file, 7749 apr_hash_t *hash, 7750 svn_fs_t *fs, 7751 apr_hash_t *reps_hash, 7752 apr_pool_t *pool) 7753{ 7754 svn_stream_t *stream; 7755 struct write_hash_baton *whb; 7756 representation_t *old_rep; 7757 7758 SVN_ERR(get_file_offset(&rep->offset, file, pool)); 7759 7760 whb = apr_pcalloc(pool, sizeof(*whb)); 7761 7762 whb->stream = svn_stream_from_aprfile2(file, TRUE, pool); 7763 whb->size = 0; 7764 whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); 7765 whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); 7766 7767 stream = svn_stream_create(whb, pool); 7768 svn_stream_set_write(stream, write_hash_handler); 7769 7770 SVN_ERR(svn_stream_puts(whb->stream, "PLAIN\n")); 7771 7772 SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool)); 7773 7774 /* Store the results. */ 7775 SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool)); 7776 SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool)); 7777 7778 /* Check and see if we already have a representation somewhere that's 7779 identical to the one we just wrote out. */ 7780 SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); 7781 7782 if (old_rep) 7783 { 7784 /* We need to erase from the protorev the data we just wrote. */ 7785 SVN_ERR(svn_io_file_trunc(file, rep->offset, pool)); 7786 7787 /* Use the old rep for this content. */ 7788 memcpy(rep, old_rep, sizeof (*rep)); 7789 } 7790 else 7791 { 7792 /* Write out our cosmetic end marker. */ 7793 SVN_ERR(svn_stream_puts(whb->stream, "ENDREP\n")); 7794 7795 /* update the representation */ 7796 rep->size = whb->size; 7797 rep->expanded_size = 0; 7798 } 7799 7800 return SVN_NO_ERROR; 7801} 7802 7803/* Write out the hash HASH pertaining to the NODEREV in FS as a deltified 7804 text representation to file FILE. In the process, record the total size 7805 and the md5 digest in REP. If rep sharing has been enabled and REPS_HASH 7806 is not NULL, it will be used in addition to the on-disk cache to find 7807 earlier reps with the same content. When such existing reps can be found, 7808 we will truncate the one just written from the file and return the existing 7809 rep. If PROPS is set, assume that we want to a props representation as 7810 the base for our delta. Perform temporary allocations in POOL. */ 7811static svn_error_t * 7812write_hash_delta_rep(representation_t *rep, 7813 apr_file_t *file, 7814 apr_hash_t *hash, 7815 svn_fs_t *fs, 7816 node_revision_t *noderev, 7817 apr_hash_t *reps_hash, 7818 svn_boolean_t props, 7819 apr_pool_t *pool) 7820{ 7821 svn_txdelta_window_handler_t diff_wh; 7822 void *diff_whb; 7823 7824 svn_stream_t *file_stream; 7825 svn_stream_t *stream; 7826 representation_t *base_rep; 7827 representation_t *old_rep; 7828 svn_stream_t *source; 7829 const char *header; 7830 7831 apr_off_t rep_end = 0; 7832 apr_off_t delta_start = 0; 7833 7834 struct write_hash_baton *whb; 7835 fs_fs_data_t *ffd = fs->fsap_data; 7836 int diff_version = ffd->format >= SVN_FS_FS__MIN_SVNDIFF1_FORMAT ? 1 : 0; 7837 7838 /* Get the base for this delta. */ 7839 SVN_ERR(choose_delta_base(&base_rep, fs, noderev, props, pool)); 7840 SVN_ERR(read_representation(&source, fs, base_rep, pool)); 7841 7842 SVN_ERR(get_file_offset(&rep->offset, file, pool)); 7843 7844 /* Write out the rep header. */ 7845 if (base_rep) 7846 { 7847 header = apr_psprintf(pool, REP_DELTA " %ld %" APR_OFF_T_FMT " %" 7848 SVN_FILESIZE_T_FMT "\n", 7849 base_rep->revision, base_rep->offset, 7850 base_rep->size); 7851 } 7852 else 7853 { 7854 header = REP_DELTA "\n"; 7855 } 7856 SVN_ERR(svn_io_file_write_full(file, header, strlen(header), NULL, 7857 pool)); 7858 7859 SVN_ERR(get_file_offset(&delta_start, file, pool)); 7860 file_stream = svn_stream_from_aprfile2(file, TRUE, pool); 7861 7862 /* Prepare to write the svndiff data. */ 7863 svn_txdelta_to_svndiff3(&diff_wh, 7864 &diff_whb, 7865 file_stream, 7866 diff_version, 7867 SVN_DELTA_COMPRESSION_LEVEL_DEFAULT, 7868 pool); 7869 7870 whb = apr_pcalloc(pool, sizeof(*whb)); 7871 whb->stream = svn_txdelta_target_push(diff_wh, diff_whb, source, pool); 7872 whb->size = 0; 7873 whb->md5_ctx = svn_checksum_ctx_create(svn_checksum_md5, pool); 7874 whb->sha1_ctx = svn_checksum_ctx_create(svn_checksum_sha1, pool); 7875 7876 /* serialize the hash */ 7877 stream = svn_stream_create(whb, pool); 7878 svn_stream_set_write(stream, write_hash_handler); 7879 7880 SVN_ERR(svn_hash_write2(hash, stream, SVN_HASH_TERMINATOR, pool)); 7881 SVN_ERR(svn_stream_close(whb->stream)); 7882 7883 /* Store the results. */ 7884 SVN_ERR(svn_checksum_final(&rep->md5_checksum, whb->md5_ctx, pool)); 7885 SVN_ERR(svn_checksum_final(&rep->sha1_checksum, whb->sha1_ctx, pool)); 7886 7887 /* Check and see if we already have a representation somewhere that's 7888 identical to the one we just wrote out. */ 7889 SVN_ERR(get_shared_rep(&old_rep, fs, rep, reps_hash, pool)); 7890 7891 if (old_rep) 7892 { 7893 /* We need to erase from the protorev the data we just wrote. */ 7894 SVN_ERR(svn_io_file_trunc(file, rep->offset, pool)); 7895 7896 /* Use the old rep for this content. */ 7897 memcpy(rep, old_rep, sizeof (*rep)); 7898 } 7899 else 7900 { 7901 /* Write out our cosmetic end marker. */ 7902 SVN_ERR(get_file_offset(&rep_end, file, pool)); 7903 SVN_ERR(svn_stream_puts(file_stream, "ENDREP\n")); 7904 7905 /* update the representation */ 7906 rep->expanded_size = whb->size; 7907 rep->size = rep_end - delta_start; 7908 } 7909 7910 return SVN_NO_ERROR; 7911} 7912 7913/* Sanity check ROOT_NODEREV, a candidate for being the root node-revision 7914 of (not yet committed) revision REV in FS. Use POOL for temporary 7915 allocations. 7916 7917 If you change this function, consider updating svn_fs_fs__verify() too. 7918 */ 7919static svn_error_t * 7920validate_root_noderev(svn_fs_t *fs, 7921 node_revision_t *root_noderev, 7922 svn_revnum_t rev, 7923 apr_pool_t *pool) 7924{ 7925 svn_revnum_t head_revnum = rev-1; 7926 int head_predecessor_count; 7927 7928 SVN_ERR_ASSERT(rev > 0); 7929 7930 /* Compute HEAD_PREDECESSOR_COUNT. */ 7931 { 7932 svn_fs_root_t *head_revision; 7933 const svn_fs_id_t *head_root_id; 7934 node_revision_t *head_root_noderev; 7935 7936 /* Get /@HEAD's noderev. */ 7937 SVN_ERR(svn_fs_fs__revision_root(&head_revision, fs, head_revnum, pool)); 7938 SVN_ERR(svn_fs_fs__node_id(&head_root_id, head_revision, "/", pool)); 7939 SVN_ERR(svn_fs_fs__get_node_revision(&head_root_noderev, fs, head_root_id, 7940 pool)); 7941 7942 head_predecessor_count = head_root_noderev->predecessor_count; 7943 } 7944 7945 /* Check that the root noderev's predecessor count equals REV. 7946 7947 This kind of corruption was seen on svn.apache.org (both on 7948 the root noderev and on other fspaths' noderevs); see 7949 issue #4129. 7950 7951 Normally (rev == root_noderev->predecessor_count), but here we 7952 use a more roundabout check that should only trigger on new instances 7953 of the corruption, rather then trigger on each and every new commit 7954 to a repository that has triggered the bug somewhere in its root 7955 noderev's history. 7956 */ 7957 if (root_noderev->predecessor_count != -1 7958 && (root_noderev->predecessor_count - head_predecessor_count) 7959 != (rev - head_revnum)) 7960 { 7961 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 7962 _("predecessor count for " 7963 "the root node-revision is wrong: " 7964 "found (%d+%ld != %d), committing r%ld"), 7965 head_predecessor_count, 7966 rev - head_revnum, /* This is equal to 1. */ 7967 root_noderev->predecessor_count, 7968 rev); 7969 } 7970 7971 return SVN_NO_ERROR; 7972} 7973 7974/* Copy a node-revision specified by id ID in fileystem FS from a 7975 transaction into the proto-rev-file FILE. Set *NEW_ID_P to a 7976 pointer to the new node-id which will be allocated in POOL. 7977 If this is a directory, copy all children as well. 7978 7979 START_NODE_ID and START_COPY_ID are 7980 the first available node and copy ids for this filesystem, for older 7981 FS formats. 7982 7983 REV is the revision number that this proto-rev-file will represent. 7984 7985 INITIAL_OFFSET is the offset of the proto-rev-file on entry to 7986 commit_body. 7987 7988 If REPS_TO_CACHE is not NULL, append to it a copy (allocated in 7989 REPS_POOL) of each data rep that is new in this revision. 7990 7991 If REPS_HASH is not NULL, append copies (allocated in REPS_POOL) 7992 of the representations of each property rep that is new in this 7993 revision. 7994 7995 AT_ROOT is true if the node revision being written is the root 7996 node-revision. It is only controls additional sanity checking 7997 logic. 7998 7999 Temporary allocations are also from POOL. */ 8000static svn_error_t * 8001write_final_rev(const svn_fs_id_t **new_id_p, 8002 apr_file_t *file, 8003 svn_revnum_t rev, 8004 svn_fs_t *fs, 8005 const svn_fs_id_t *id, 8006 const char *start_node_id, 8007 const char *start_copy_id, 8008 apr_off_t initial_offset, 8009 apr_array_header_t *reps_to_cache, 8010 apr_hash_t *reps_hash, 8011 apr_pool_t *reps_pool, 8012 svn_boolean_t at_root, 8013 apr_pool_t *pool) 8014{ 8015 node_revision_t *noderev; 8016 apr_off_t my_offset; 8017 char my_node_id_buf[MAX_KEY_SIZE + 2]; 8018 char my_copy_id_buf[MAX_KEY_SIZE + 2]; 8019 const svn_fs_id_t *new_id; 8020 const char *node_id, *copy_id, *my_node_id, *my_copy_id; 8021 fs_fs_data_t *ffd = fs->fsap_data; 8022 8023 *new_id_p = NULL; 8024 8025 /* Check to see if this is a transaction node. */ 8026 if (! svn_fs_fs__id_txn_id(id)) 8027 return SVN_NO_ERROR; 8028 8029 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, pool)); 8030 8031 if (noderev->kind == svn_node_dir) 8032 { 8033 apr_pool_t *subpool; 8034 apr_hash_t *entries, *str_entries; 8035 apr_array_header_t *sorted_entries; 8036 int i; 8037 8038 /* This is a directory. Write out all the children first. */ 8039 subpool = svn_pool_create(pool); 8040 8041 SVN_ERR(svn_fs_fs__rep_contents_dir(&entries, fs, noderev, pool)); 8042 /* For the sake of the repository administrator sort the entries 8043 so that the final file is deterministic and repeatable, 8044 however the rest of the FSFS code doesn't require any 8045 particular order here. */ 8046 sorted_entries = svn_sort__hash(entries, svn_sort_compare_items_lexically, 8047 pool); 8048 for (i = 0; i < sorted_entries->nelts; ++i) 8049 { 8050 svn_fs_dirent_t *dirent = APR_ARRAY_IDX(sorted_entries, i, 8051 svn_sort__item_t).value; 8052 8053 svn_pool_clear(subpool); 8054 SVN_ERR(write_final_rev(&new_id, file, rev, fs, dirent->id, 8055 start_node_id, start_copy_id, initial_offset, 8056 reps_to_cache, reps_hash, reps_pool, FALSE, 8057 subpool)); 8058 if (new_id && (svn_fs_fs__id_rev(new_id) == rev)) 8059 dirent->id = svn_fs_fs__id_copy(new_id, pool); 8060 } 8061 svn_pool_destroy(subpool); 8062 8063 if (noderev->data_rep && noderev->data_rep->txn_id) 8064 { 8065 /* Write out the contents of this directory as a text rep. */ 8066 SVN_ERR(unparse_dir_entries(&str_entries, entries, pool)); 8067 8068 noderev->data_rep->txn_id = NULL; 8069 noderev->data_rep->revision = rev; 8070 8071 if (ffd->deltify_directories) 8072 SVN_ERR(write_hash_delta_rep(noderev->data_rep, file, 8073 str_entries, fs, noderev, NULL, 8074 FALSE, pool)); 8075 else 8076 SVN_ERR(write_hash_rep(noderev->data_rep, file, str_entries, 8077 fs, NULL, pool)); 8078 } 8079 } 8080 else 8081 { 8082 /* This is a file. We should make sure the data rep, if it 8083 exists in a "this" state, gets rewritten to our new revision 8084 num. */ 8085 8086 if (noderev->data_rep && noderev->data_rep->txn_id) 8087 { 8088 noderev->data_rep->txn_id = NULL; 8089 noderev->data_rep->revision = rev; 8090 8091 /* See issue 3845. Some unknown mechanism caused the 8092 protorev file to get truncated, so check for that 8093 here. */ 8094 if (noderev->data_rep->offset + noderev->data_rep->size 8095 > initial_offset) 8096 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 8097 _("Truncated protorev file detected")); 8098 } 8099 } 8100 8101 /* Fix up the property reps. */ 8102 if (noderev->prop_rep && noderev->prop_rep->txn_id) 8103 { 8104 apr_hash_t *proplist; 8105 SVN_ERR(svn_fs_fs__get_proplist(&proplist, fs, noderev, pool)); 8106 8107 noderev->prop_rep->txn_id = NULL; 8108 noderev->prop_rep->revision = rev; 8109 8110 if (ffd->deltify_properties) 8111 SVN_ERR(write_hash_delta_rep(noderev->prop_rep, file, 8112 proplist, fs, noderev, reps_hash, 8113 TRUE, pool)); 8114 else 8115 SVN_ERR(write_hash_rep(noderev->prop_rep, file, proplist, 8116 fs, reps_hash, pool)); 8117 } 8118 8119 8120 /* Convert our temporary ID into a permanent revision one. */ 8121 SVN_ERR(get_file_offset(&my_offset, file, pool)); 8122 8123 node_id = svn_fs_fs__id_node_id(noderev->id); 8124 if (*node_id == '_') 8125 { 8126 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 8127 my_node_id = apr_psprintf(pool, "%s-%ld", node_id + 1, rev); 8128 else 8129 { 8130 svn_fs_fs__add_keys(start_node_id, node_id + 1, my_node_id_buf); 8131 my_node_id = my_node_id_buf; 8132 } 8133 } 8134 else 8135 my_node_id = node_id; 8136 8137 copy_id = svn_fs_fs__id_copy_id(noderev->id); 8138 if (*copy_id == '_') 8139 { 8140 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 8141 my_copy_id = apr_psprintf(pool, "%s-%ld", copy_id + 1, rev); 8142 else 8143 { 8144 svn_fs_fs__add_keys(start_copy_id, copy_id + 1, my_copy_id_buf); 8145 my_copy_id = my_copy_id_buf; 8146 } 8147 } 8148 else 8149 my_copy_id = copy_id; 8150 8151 if (noderev->copyroot_rev == SVN_INVALID_REVNUM) 8152 noderev->copyroot_rev = rev; 8153 8154 new_id = svn_fs_fs__id_rev_create(my_node_id, my_copy_id, rev, my_offset, 8155 pool); 8156 8157 noderev->id = new_id; 8158 8159 if (ffd->rep_sharing_allowed) 8160 { 8161 /* Save the data representation's hash in the rep cache. */ 8162 if ( noderev->data_rep && noderev->kind == svn_node_file 8163 && noderev->data_rep->revision == rev) 8164 { 8165 SVN_ERR_ASSERT(reps_to_cache && reps_pool); 8166 APR_ARRAY_PUSH(reps_to_cache, representation_t *) 8167 = svn_fs_fs__rep_copy(noderev->data_rep, reps_pool); 8168 } 8169 8170 if (noderev->prop_rep && noderev->prop_rep->revision == rev) 8171 { 8172 /* Add new property reps to hash and on-disk cache. */ 8173 representation_t *copy 8174 = svn_fs_fs__rep_copy(noderev->prop_rep, reps_pool); 8175 8176 SVN_ERR_ASSERT(reps_to_cache && reps_pool); 8177 APR_ARRAY_PUSH(reps_to_cache, representation_t *) = copy; 8178 8179 apr_hash_set(reps_hash, 8180 copy->sha1_checksum->digest, 8181 APR_SHA1_DIGESTSIZE, 8182 copy); 8183 } 8184 } 8185 8186 /* don't serialize SHA1 for dirs to disk (waste of space) */ 8187 if (noderev->data_rep && noderev->kind == svn_node_dir) 8188 noderev->data_rep->sha1_checksum = NULL; 8189 8190 /* don't serialize SHA1 for props to disk (waste of space) */ 8191 if (noderev->prop_rep) 8192 noderev->prop_rep->sha1_checksum = NULL; 8193 8194 /* Workaround issue #4031: is-fresh-txn-root in revision files. */ 8195 noderev->is_fresh_txn_root = FALSE; 8196 8197 /* Write out our new node-revision. */ 8198 if (at_root) 8199 SVN_ERR(validate_root_noderev(fs, noderev, rev, pool)); 8200 8201 SVN_ERR(svn_fs_fs__write_noderev(svn_stream_from_aprfile2(file, TRUE, pool), 8202 noderev, ffd->format, 8203 svn_fs_fs__fs_supports_mergeinfo(fs), 8204 pool)); 8205 8206 /* Return our ID that references the revision file. */ 8207 *new_id_p = noderev->id; 8208 8209 return SVN_NO_ERROR; 8210} 8211 8212/* Write the changed path info from transaction TXN_ID in filesystem 8213 FS to the permanent rev-file FILE. *OFFSET_P is set the to offset 8214 in the file of the beginning of this information. Perform 8215 temporary allocations in POOL. */ 8216static svn_error_t * 8217write_final_changed_path_info(apr_off_t *offset_p, 8218 apr_file_t *file, 8219 svn_fs_t *fs, 8220 const char *txn_id, 8221 apr_pool_t *pool) 8222{ 8223 apr_hash_t *changed_paths; 8224 apr_off_t offset; 8225 apr_pool_t *iterpool = svn_pool_create(pool); 8226 fs_fs_data_t *ffd = fs->fsap_data; 8227 svn_boolean_t include_node_kinds = 8228 ffd->format >= SVN_FS_FS__MIN_KIND_IN_CHANGED_FORMAT; 8229 apr_array_header_t *sorted_changed_paths; 8230 int i; 8231 8232 SVN_ERR(get_file_offset(&offset, file, pool)); 8233 8234 SVN_ERR(svn_fs_fs__txn_changes_fetch(&changed_paths, fs, txn_id, pool)); 8235 /* For the sake of the repository administrator sort the changes so 8236 that the final file is deterministic and repeatable, however the 8237 rest of the FSFS code doesn't require any particular order here. */ 8238 sorted_changed_paths = svn_sort__hash(changed_paths, 8239 svn_sort_compare_items_lexically, pool); 8240 8241 /* Iterate through the changed paths one at a time, and convert the 8242 temporary node-id into a permanent one for each change entry. */ 8243 for (i = 0; i < sorted_changed_paths->nelts; ++i) 8244 { 8245 node_revision_t *noderev; 8246 const svn_fs_id_t *id; 8247 svn_fs_path_change2_t *change; 8248 const char *path; 8249 8250 svn_pool_clear(iterpool); 8251 8252 change = APR_ARRAY_IDX(sorted_changed_paths, i, svn_sort__item_t).value; 8253 path = APR_ARRAY_IDX(sorted_changed_paths, i, svn_sort__item_t).key; 8254 8255 id = change->node_rev_id; 8256 8257 /* If this was a delete of a mutable node, then it is OK to 8258 leave the change entry pointing to the non-existent temporary 8259 node, since it will never be used. */ 8260 if ((change->change_kind != svn_fs_path_change_delete) && 8261 (! svn_fs_fs__id_txn_id(id))) 8262 { 8263 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, iterpool)); 8264 8265 /* noderev has the permanent node-id at this point, so we just 8266 substitute it for the temporary one. */ 8267 change->node_rev_id = noderev->id; 8268 } 8269 8270 /* Write out the new entry into the final rev-file. */ 8271 SVN_ERR(write_change_entry(file, path, change, include_node_kinds, 8272 iterpool)); 8273 } 8274 8275 svn_pool_destroy(iterpool); 8276 8277 *offset_p = offset; 8278 8279 return SVN_NO_ERROR; 8280} 8281 8282/* Atomically update the 'current' file to hold the specifed REV, 8283 NEXT_NODE_ID, and NEXT_COPY_ID. (The two next-ID parameters are 8284 ignored and may be NULL if the FS format does not use them.) 8285 Perform temporary allocations in POOL. */ 8286static svn_error_t * 8287write_current(svn_fs_t *fs, svn_revnum_t rev, const char *next_node_id, 8288 const char *next_copy_id, apr_pool_t *pool) 8289{ 8290 char *buf; 8291 const char *tmp_name, *name; 8292 fs_fs_data_t *ffd = fs->fsap_data; 8293 8294 /* Now we can just write out this line. */ 8295 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 8296 buf = apr_psprintf(pool, "%ld\n", rev); 8297 else 8298 buf = apr_psprintf(pool, "%ld %s %s\n", rev, next_node_id, next_copy_id); 8299 8300 name = svn_fs_fs__path_current(fs, pool); 8301 SVN_ERR(svn_io_write_unique(&tmp_name, 8302 svn_dirent_dirname(name, pool), 8303 buf, strlen(buf), 8304 svn_io_file_del_none, pool)); 8305 8306 return move_into_place(tmp_name, name, name, pool); 8307} 8308 8309/* Open a new svn_fs_t handle to FS, set that handle's concept of "current 8310 youngest revision" to NEW_REV, and call svn_fs_fs__verify_root() on 8311 NEW_REV's revision root. 8312 8313 Intended to be called as the very last step in a commit before 'current' 8314 is bumped. This implies that we are holding the write lock. */ 8315static svn_error_t * 8316verify_as_revision_before_current_plus_plus(svn_fs_t *fs, 8317 svn_revnum_t new_rev, 8318 apr_pool_t *pool) 8319{ 8320#ifdef SVN_DEBUG 8321 fs_fs_data_t *ffd = fs->fsap_data; 8322 svn_fs_t *ft; /* fs++ == ft */ 8323 svn_fs_root_t *root; 8324 fs_fs_data_t *ft_ffd; 8325 apr_hash_t *fs_config; 8326 8327 SVN_ERR_ASSERT(ffd->svn_fs_open_); 8328 8329 /* make sure FT does not simply return data cached by other instances 8330 * but actually retrieves it from disk at least once. 8331 */ 8332 fs_config = apr_hash_make(pool); 8333 svn_hash_sets(fs_config, SVN_FS_CONFIG_FSFS_CACHE_NS, 8334 svn_uuid_generate(pool)); 8335 SVN_ERR(ffd->svn_fs_open_(&ft, fs->path, 8336 fs_config, 8337 pool)); 8338 ft_ffd = ft->fsap_data; 8339 /* Don't let FT consult rep-cache.db, either. */ 8340 ft_ffd->rep_sharing_allowed = FALSE; 8341 8342 /* Time travel! */ 8343 ft_ffd->youngest_rev_cache = new_rev; 8344 8345 SVN_ERR(svn_fs_fs__revision_root(&root, ft, new_rev, pool)); 8346 SVN_ERR_ASSERT(root->is_txn_root == FALSE && root->rev == new_rev); 8347 SVN_ERR_ASSERT(ft_ffd->youngest_rev_cache == new_rev); 8348 SVN_ERR(svn_fs_fs__verify_root(root, pool)); 8349#endif /* SVN_DEBUG */ 8350 8351 return SVN_NO_ERROR; 8352} 8353 8354/* Update the 'current' file to hold the correct next node and copy_ids 8355 from transaction TXN_ID in filesystem FS. The current revision is 8356 set to REV. Perform temporary allocations in POOL. */ 8357static svn_error_t * 8358write_final_current(svn_fs_t *fs, 8359 const char *txn_id, 8360 svn_revnum_t rev, 8361 const char *start_node_id, 8362 const char *start_copy_id, 8363 apr_pool_t *pool) 8364{ 8365 const char *txn_node_id, *txn_copy_id; 8366 char new_node_id[MAX_KEY_SIZE + 2]; 8367 char new_copy_id[MAX_KEY_SIZE + 2]; 8368 fs_fs_data_t *ffd = fs->fsap_data; 8369 8370 if (ffd->format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 8371 return write_current(fs, rev, NULL, NULL, pool); 8372 8373 /* To find the next available ids, we add the id that used to be in 8374 the 'current' file, to the next ids from the transaction file. */ 8375 SVN_ERR(read_next_ids(&txn_node_id, &txn_copy_id, fs, txn_id, pool)); 8376 8377 svn_fs_fs__add_keys(start_node_id, txn_node_id, new_node_id); 8378 svn_fs_fs__add_keys(start_copy_id, txn_copy_id, new_copy_id); 8379 8380 return write_current(fs, rev, new_node_id, new_copy_id, pool); 8381} 8382 8383/* Verify that the user registed with FS has all the locks necessary to 8384 permit all the changes associate with TXN_NAME. 8385 The FS write lock is assumed to be held by the caller. */ 8386static svn_error_t * 8387verify_locks(svn_fs_t *fs, 8388 const char *txn_name, 8389 apr_pool_t *pool) 8390{ 8391 apr_pool_t *subpool = svn_pool_create(pool); 8392 apr_hash_t *changes; 8393 apr_hash_index_t *hi; 8394 apr_array_header_t *changed_paths; 8395 svn_stringbuf_t *last_recursed = NULL; 8396 int i; 8397 8398 /* Fetch the changes for this transaction. */ 8399 SVN_ERR(svn_fs_fs__txn_changes_fetch(&changes, fs, txn_name, pool)); 8400 8401 /* Make an array of the changed paths, and sort them depth-first-ily. */ 8402 changed_paths = apr_array_make(pool, apr_hash_count(changes) + 1, 8403 sizeof(const char *)); 8404 for (hi = apr_hash_first(pool, changes); hi; hi = apr_hash_next(hi)) 8405 APR_ARRAY_PUSH(changed_paths, const char *) = svn__apr_hash_index_key(hi); 8406 qsort(changed_paths->elts, changed_paths->nelts, 8407 changed_paths->elt_size, svn_sort_compare_paths); 8408 8409 /* Now, traverse the array of changed paths, verify locks. Note 8410 that if we need to do a recursive verification a path, we'll skip 8411 over children of that path when we get to them. */ 8412 for (i = 0; i < changed_paths->nelts; i++) 8413 { 8414 const char *path; 8415 svn_fs_path_change2_t *change; 8416 svn_boolean_t recurse = TRUE; 8417 8418 svn_pool_clear(subpool); 8419 path = APR_ARRAY_IDX(changed_paths, i, const char *); 8420 8421 /* If this path has already been verified as part of a recursive 8422 check of one of its parents, no need to do it again. */ 8423 if (last_recursed 8424 && svn_dirent_is_child(last_recursed->data, path, subpool)) 8425 continue; 8426 8427 /* Fetch the change associated with our path. */ 8428 change = svn_hash_gets(changes, path); 8429 8430 /* What does it mean to succeed at lock verification for a given 8431 path? For an existing file or directory getting modified 8432 (text, props), it means we hold the lock on the file or 8433 directory. For paths being added or removed, we need to hold 8434 the locks for that path and any children of that path. 8435 8436 WHEW! We have no reliable way to determine the node kind 8437 of deleted items, but fortunately we are going to do a 8438 recursive check on deleted paths regardless of their kind. */ 8439 if (change->change_kind == svn_fs_path_change_modify) 8440 recurse = FALSE; 8441 SVN_ERR(svn_fs_fs__allow_locked_operation(path, fs, recurse, TRUE, 8442 subpool)); 8443 8444 /* If we just did a recursive check, remember the path we 8445 checked (so children can be skipped). */ 8446 if (recurse) 8447 { 8448 if (! last_recursed) 8449 last_recursed = svn_stringbuf_create(path, pool); 8450 else 8451 svn_stringbuf_set(last_recursed, path); 8452 } 8453 } 8454 svn_pool_destroy(subpool); 8455 return SVN_NO_ERROR; 8456} 8457 8458/* Baton used for commit_body below. */ 8459struct commit_baton { 8460 svn_revnum_t *new_rev_p; 8461 svn_fs_t *fs; 8462 svn_fs_txn_t *txn; 8463 apr_array_header_t *reps_to_cache; 8464 apr_hash_t *reps_hash; 8465 apr_pool_t *reps_pool; 8466}; 8467 8468/* The work-horse for svn_fs_fs__commit, called with the FS write lock. 8469 This implements the svn_fs_fs__with_write_lock() 'body' callback 8470 type. BATON is a 'struct commit_baton *'. */ 8471static svn_error_t * 8472commit_body(void *baton, apr_pool_t *pool) 8473{ 8474 struct commit_baton *cb = baton; 8475 fs_fs_data_t *ffd = cb->fs->fsap_data; 8476 const char *old_rev_filename, *rev_filename, *proto_filename; 8477 const char *revprop_filename, *final_revprop; 8478 const svn_fs_id_t *root_id, *new_root_id; 8479 const char *start_node_id = NULL, *start_copy_id = NULL; 8480 svn_revnum_t old_rev, new_rev; 8481 apr_file_t *proto_file; 8482 void *proto_file_lockcookie; 8483 apr_off_t initial_offset, changed_path_offset; 8484 char *buf; 8485 apr_hash_t *txnprops; 8486 apr_array_header_t *txnprop_list; 8487 svn_prop_t prop; 8488 svn_string_t date; 8489 8490 /* Get the current youngest revision. */ 8491 SVN_ERR(svn_fs_fs__youngest_rev(&old_rev, cb->fs, pool)); 8492 8493 /* Check to make sure this transaction is based off the most recent 8494 revision. */ 8495 if (cb->txn->base_rev != old_rev) 8496 return svn_error_create(SVN_ERR_FS_TXN_OUT_OF_DATE, NULL, 8497 _("Transaction out of date")); 8498 8499 /* Locks may have been added (or stolen) between the calling of 8500 previous svn_fs.h functions and svn_fs_commit_txn(), so we need 8501 to re-examine every changed-path in the txn and re-verify all 8502 discovered locks. */ 8503 SVN_ERR(verify_locks(cb->fs, cb->txn->id, pool)); 8504 8505 /* Get the next node_id and copy_id to use. */ 8506 if (ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 8507 SVN_ERR(get_next_revision_ids(&start_node_id, &start_copy_id, cb->fs, 8508 pool)); 8509 8510 /* We are going to be one better than this puny old revision. */ 8511 new_rev = old_rev + 1; 8512 8513 /* Get a write handle on the proto revision file. */ 8514 SVN_ERR(get_writable_proto_rev(&proto_file, &proto_file_lockcookie, 8515 cb->fs, cb->txn->id, pool)); 8516 SVN_ERR(get_file_offset(&initial_offset, proto_file, pool)); 8517 8518 /* Write out all the node-revisions and directory contents. */ 8519 root_id = svn_fs_fs__id_txn_create("0", "0", cb->txn->id, pool); 8520 SVN_ERR(write_final_rev(&new_root_id, proto_file, new_rev, cb->fs, root_id, 8521 start_node_id, start_copy_id, initial_offset, 8522 cb->reps_to_cache, cb->reps_hash, cb->reps_pool, 8523 TRUE, pool)); 8524 8525 /* Write the changed-path information. */ 8526 SVN_ERR(write_final_changed_path_info(&changed_path_offset, proto_file, 8527 cb->fs, cb->txn->id, pool)); 8528 8529 /* Write the final line. */ 8530 buf = apr_psprintf(pool, "\n%" APR_OFF_T_FMT " %" APR_OFF_T_FMT "\n", 8531 svn_fs_fs__id_offset(new_root_id), 8532 changed_path_offset); 8533 SVN_ERR(svn_io_file_write_full(proto_file, buf, strlen(buf), NULL, 8534 pool)); 8535 SVN_ERR(svn_io_file_flush_to_disk(proto_file, pool)); 8536 SVN_ERR(svn_io_file_close(proto_file, pool)); 8537 8538 /* We don't unlock the prototype revision file immediately to avoid a 8539 race with another caller writing to the prototype revision file 8540 before we commit it. */ 8541 8542 /* Remove any temporary txn props representing 'flags'. */ 8543 SVN_ERR(svn_fs_fs__txn_proplist(&txnprops, cb->txn, pool)); 8544 txnprop_list = apr_array_make(pool, 3, sizeof(svn_prop_t)); 8545 prop.value = NULL; 8546 8547 if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_OOD)) 8548 { 8549 prop.name = SVN_FS__PROP_TXN_CHECK_OOD; 8550 APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; 8551 } 8552 8553 if (svn_hash_gets(txnprops, SVN_FS__PROP_TXN_CHECK_LOCKS)) 8554 { 8555 prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS; 8556 APR_ARRAY_PUSH(txnprop_list, svn_prop_t) = prop; 8557 } 8558 8559 if (! apr_is_empty_array(txnprop_list)) 8560 SVN_ERR(svn_fs_fs__change_txn_props(cb->txn, txnprop_list, pool)); 8561 8562 /* Create the shard for the rev and revprop file, if we're sharding and 8563 this is the first revision of a new shard. We don't care if this 8564 fails because the shard already existed for some reason. */ 8565 if (ffd->max_files_per_dir && new_rev % ffd->max_files_per_dir == 0) 8566 { 8567 /* Create the revs shard. */ 8568 { 8569 const char *new_dir = path_rev_shard(cb->fs, new_rev, pool); 8570 svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool); 8571 if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) 8572 return svn_error_trace(err); 8573 svn_error_clear(err); 8574 SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path, 8575 PATH_REVS_DIR, 8576 pool), 8577 new_dir, pool)); 8578 } 8579 8580 /* Create the revprops shard. */ 8581 SVN_ERR_ASSERT(! is_packed_revprop(cb->fs, new_rev)); 8582 { 8583 const char *new_dir = path_revprops_shard(cb->fs, new_rev, pool); 8584 svn_error_t *err = svn_io_dir_make(new_dir, APR_OS_DEFAULT, pool); 8585 if (err && !APR_STATUS_IS_EEXIST(err->apr_err)) 8586 return svn_error_trace(err); 8587 svn_error_clear(err); 8588 SVN_ERR(svn_io_copy_perms(svn_dirent_join(cb->fs->path, 8589 PATH_REVPROPS_DIR, 8590 pool), 8591 new_dir, pool)); 8592 } 8593 } 8594 8595 /* Move the finished rev file into place. */ 8596 SVN_ERR(svn_fs_fs__path_rev_absolute(&old_rev_filename, 8597 cb->fs, old_rev, pool)); 8598 rev_filename = path_rev(cb->fs, new_rev, pool); 8599 proto_filename = path_txn_proto_rev(cb->fs, cb->txn->id, pool); 8600 SVN_ERR(move_into_place(proto_filename, rev_filename, old_rev_filename, 8601 pool)); 8602 8603 /* Now that we've moved the prototype revision file out of the way, 8604 we can unlock it (since further attempts to write to the file 8605 will fail as it no longer exists). We must do this so that we can 8606 remove the transaction directory later. */ 8607 SVN_ERR(unlock_proto_rev(cb->fs, cb->txn->id, proto_file_lockcookie, pool)); 8608 8609 /* Update commit time to ensure that svn:date revprops remain ordered. */ 8610 date.data = svn_time_to_cstring(apr_time_now(), pool); 8611 date.len = strlen(date.data); 8612 8613 SVN_ERR(svn_fs_fs__change_txn_prop(cb->txn, SVN_PROP_REVISION_DATE, 8614 &date, pool)); 8615 8616 /* Move the revprops file into place. */ 8617 SVN_ERR_ASSERT(! is_packed_revprop(cb->fs, new_rev)); 8618 revprop_filename = path_txn_props(cb->fs, cb->txn->id, pool); 8619 final_revprop = path_revprops(cb->fs, new_rev, pool); 8620 SVN_ERR(move_into_place(revprop_filename, final_revprop, 8621 old_rev_filename, pool)); 8622 8623 /* Update the 'current' file. */ 8624 SVN_ERR(verify_as_revision_before_current_plus_plus(cb->fs, new_rev, pool)); 8625 SVN_ERR(write_final_current(cb->fs, cb->txn->id, new_rev, start_node_id, 8626 start_copy_id, pool)); 8627 8628 /* At this point the new revision is committed and globally visible 8629 so let the caller know it succeeded by giving it the new revision 8630 number, which fulfills svn_fs_commit_txn() contract. Any errors 8631 after this point do not change the fact that a new revision was 8632 created. */ 8633 *cb->new_rev_p = new_rev; 8634 8635 ffd->youngest_rev_cache = new_rev; 8636 8637 /* Remove this transaction directory. */ 8638 SVN_ERR(svn_fs_fs__purge_txn(cb->fs, cb->txn->id, pool)); 8639 8640 return SVN_NO_ERROR; 8641} 8642 8643/* Add the representations in REPS_TO_CACHE (an array of representation_t *) 8644 * to the rep-cache database of FS. */ 8645static svn_error_t * 8646write_reps_to_cache(svn_fs_t *fs, 8647 const apr_array_header_t *reps_to_cache, 8648 apr_pool_t *scratch_pool) 8649{ 8650 int i; 8651 8652 for (i = 0; i < reps_to_cache->nelts; i++) 8653 { 8654 representation_t *rep = APR_ARRAY_IDX(reps_to_cache, i, representation_t *); 8655 8656 /* FALSE because we don't care if another parallel commit happened to 8657 * collide with us. (Non-parallel collisions will not be detected.) */ 8658 SVN_ERR(svn_fs_fs__set_rep_reference(fs, rep, FALSE, scratch_pool)); 8659 } 8660 8661 return SVN_NO_ERROR; 8662} 8663 8664svn_error_t * 8665svn_fs_fs__commit(svn_revnum_t *new_rev_p, 8666 svn_fs_t *fs, 8667 svn_fs_txn_t *txn, 8668 apr_pool_t *pool) 8669{ 8670 struct commit_baton cb; 8671 fs_fs_data_t *ffd = fs->fsap_data; 8672 8673 cb.new_rev_p = new_rev_p; 8674 cb.fs = fs; 8675 cb.txn = txn; 8676 8677 if (ffd->rep_sharing_allowed) 8678 { 8679 cb.reps_to_cache = apr_array_make(pool, 5, sizeof(representation_t *)); 8680 cb.reps_hash = apr_hash_make(pool); 8681 cb.reps_pool = pool; 8682 } 8683 else 8684 { 8685 cb.reps_to_cache = NULL; 8686 cb.reps_hash = NULL; 8687 cb.reps_pool = NULL; 8688 } 8689 8690 SVN_ERR(svn_fs_fs__with_write_lock(fs, commit_body, &cb, pool)); 8691 8692 /* At this point, *NEW_REV_P has been set, so errors below won't affect 8693 the success of the commit. (See svn_fs_commit_txn().) */ 8694 8695 if (ffd->rep_sharing_allowed) 8696 { 8697 SVN_ERR(svn_fs_fs__open_rep_cache(fs, pool)); 8698 8699 /* Write new entries to the rep-sharing database. 8700 * 8701 * We use an sqlite transaction to speed things up; 8702 * see <http://www.sqlite.org/faq.html#q19>. 8703 */ 8704 SVN_SQLITE__WITH_TXN( 8705 write_reps_to_cache(fs, cb.reps_to_cache, pool), 8706 ffd->rep_cache_db); 8707 } 8708 8709 return SVN_NO_ERROR; 8710} 8711 8712 8713svn_error_t * 8714svn_fs_fs__reserve_copy_id(const char **copy_id_p, 8715 svn_fs_t *fs, 8716 const char *txn_id, 8717 apr_pool_t *pool) 8718{ 8719 const char *cur_node_id, *cur_copy_id; 8720 char *copy_id; 8721 apr_size_t len; 8722 8723 /* First read in the current next-ids file. */ 8724 SVN_ERR(read_next_ids(&cur_node_id, &cur_copy_id, fs, txn_id, pool)); 8725 8726 copy_id = apr_pcalloc(pool, strlen(cur_copy_id) + 2); 8727 8728 len = strlen(cur_copy_id); 8729 svn_fs_fs__next_key(cur_copy_id, &len, copy_id); 8730 8731 SVN_ERR(write_next_ids(fs, txn_id, cur_node_id, copy_id, pool)); 8732 8733 *copy_id_p = apr_pstrcat(pool, "_", cur_copy_id, (char *)NULL); 8734 8735 return SVN_NO_ERROR; 8736} 8737 8738/* Write out the zeroth revision for filesystem FS. */ 8739static svn_error_t * 8740write_revision_zero(svn_fs_t *fs) 8741{ 8742 const char *path_revision_zero = path_rev(fs, 0, fs->pool); 8743 apr_hash_t *proplist; 8744 svn_string_t date; 8745 8746 /* Write out a rev file for revision 0. */ 8747 SVN_ERR(svn_io_file_create(path_revision_zero, 8748 "PLAIN\nEND\nENDREP\n" 8749 "id: 0.0.r0/17\n" 8750 "type: dir\n" 8751 "count: 0\n" 8752 "text: 0 0 4 4 " 8753 "2d2977d1c96f487abe4a1e202dd03b4e\n" 8754 "cpath: /\n" 8755 "\n\n17 107\n", fs->pool)); 8756 SVN_ERR(svn_io_set_file_read_only(path_revision_zero, FALSE, fs->pool)); 8757 8758 /* Set a date on revision 0. */ 8759 date.data = svn_time_to_cstring(apr_time_now(), fs->pool); 8760 date.len = strlen(date.data); 8761 proplist = apr_hash_make(fs->pool); 8762 svn_hash_sets(proplist, SVN_PROP_REVISION_DATE, &date); 8763 return set_revision_proplist(fs, 0, proplist, fs->pool); 8764} 8765 8766svn_error_t * 8767svn_fs_fs__create(svn_fs_t *fs, 8768 const char *path, 8769 apr_pool_t *pool) 8770{ 8771 int format = SVN_FS_FS__FORMAT_NUMBER; 8772 fs_fs_data_t *ffd = fs->fsap_data; 8773 8774 fs->path = apr_pstrdup(pool, path); 8775 /* See if compatibility with older versions was explicitly requested. */ 8776 if (fs->config) 8777 { 8778 if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_4_COMPATIBLE)) 8779 format = 1; 8780 else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_5_COMPATIBLE)) 8781 format = 2; 8782 else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_6_COMPATIBLE)) 8783 format = 3; 8784 else if (svn_hash_gets(fs->config, SVN_FS_CONFIG_PRE_1_8_COMPATIBLE)) 8785 format = 4; 8786 } 8787 ffd->format = format; 8788 8789 /* Override the default linear layout if this is a new-enough format. */ 8790 if (format >= SVN_FS_FS__MIN_LAYOUT_FORMAT_OPTION_FORMAT) 8791 ffd->max_files_per_dir = SVN_FS_FS_DEFAULT_MAX_FILES_PER_DIR; 8792 8793 /* Create the revision data directories. */ 8794 if (ffd->max_files_per_dir) 8795 SVN_ERR(svn_io_make_dir_recursively(path_rev_shard(fs, 0, pool), pool)); 8796 else 8797 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_REVS_DIR, 8798 pool), 8799 pool)); 8800 8801 /* Create the revprops directory. */ 8802 if (ffd->max_files_per_dir) 8803 SVN_ERR(svn_io_make_dir_recursively(path_revprops_shard(fs, 0, pool), 8804 pool)); 8805 else 8806 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, 8807 PATH_REVPROPS_DIR, 8808 pool), 8809 pool)); 8810 8811 /* Create the transaction directory. */ 8812 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXNS_DIR, 8813 pool), 8814 pool)); 8815 8816 /* Create the protorevs directory. */ 8817 if (format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) 8818 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(path, PATH_TXN_PROTOS_DIR, 8819 pool), 8820 pool)); 8821 8822 /* Create the 'current' file. */ 8823 SVN_ERR(svn_io_file_create(svn_fs_fs__path_current(fs, pool), 8824 (format >= SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT 8825 ? "0\n" : "0 1 1\n"), 8826 pool)); 8827 SVN_ERR(svn_io_file_create(path_lock(fs, pool), "", pool)); 8828 SVN_ERR(svn_fs_fs__set_uuid(fs, NULL, pool)); 8829 8830 SVN_ERR(write_revision_zero(fs)); 8831 8832 SVN_ERR(write_config(fs, pool)); 8833 8834 SVN_ERR(read_config(ffd, fs->path, pool)); 8835 8836 /* Create the min unpacked rev file. */ 8837 if (ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 8838 SVN_ERR(svn_io_file_create(path_min_unpacked_rev(fs, pool), "0\n", pool)); 8839 8840 /* Create the txn-current file if the repository supports 8841 the transaction sequence file. */ 8842 if (format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) 8843 { 8844 SVN_ERR(svn_io_file_create(path_txn_current(fs, pool), 8845 "0\n", pool)); 8846 SVN_ERR(svn_io_file_create(path_txn_current_lock(fs, pool), 8847 "", pool)); 8848 } 8849 8850 /* This filesystem is ready. Stamp it with a format number. */ 8851 SVN_ERR(write_format(path_format(fs, pool), 8852 ffd->format, ffd->max_files_per_dir, FALSE, pool)); 8853 8854 ffd->youngest_rev_cache = 0; 8855 return SVN_NO_ERROR; 8856} 8857 8858/* Part of the recovery procedure. Return the largest revision *REV in 8859 filesystem FS. Use POOL for temporary allocation. */ 8860static svn_error_t * 8861recover_get_largest_revision(svn_fs_t *fs, svn_revnum_t *rev, apr_pool_t *pool) 8862{ 8863 /* Discovering the largest revision in the filesystem would be an 8864 expensive operation if we did a readdir() or searched linearly, 8865 so we'll do a form of binary search. left is a revision that we 8866 know exists, right a revision that we know does not exist. */ 8867 apr_pool_t *iterpool; 8868 svn_revnum_t left, right = 1; 8869 8870 iterpool = svn_pool_create(pool); 8871 /* Keep doubling right, until we find a revision that doesn't exist. */ 8872 while (1) 8873 { 8874 svn_error_t *err; 8875 apr_file_t *file; 8876 8877 err = open_pack_or_rev_file(&file, fs, right, iterpool); 8878 svn_pool_clear(iterpool); 8879 8880 if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) 8881 { 8882 svn_error_clear(err); 8883 break; 8884 } 8885 else 8886 SVN_ERR(err); 8887 8888 right <<= 1; 8889 } 8890 8891 left = right >> 1; 8892 8893 /* We know that left exists and right doesn't. Do a normal bsearch to find 8894 the last revision. */ 8895 while (left + 1 < right) 8896 { 8897 svn_revnum_t probe = left + ((right - left) / 2); 8898 svn_error_t *err; 8899 apr_file_t *file; 8900 8901 err = open_pack_or_rev_file(&file, fs, probe, iterpool); 8902 svn_pool_clear(iterpool); 8903 8904 if (err && err->apr_err == SVN_ERR_FS_NO_SUCH_REVISION) 8905 { 8906 svn_error_clear(err); 8907 right = probe; 8908 } 8909 else 8910 { 8911 SVN_ERR(err); 8912 left = probe; 8913 } 8914 } 8915 8916 svn_pool_destroy(iterpool); 8917 8918 /* left is now the largest revision that exists. */ 8919 *rev = left; 8920 return SVN_NO_ERROR; 8921} 8922 8923/* A baton for reading a fixed amount from an open file. For 8924 recover_find_max_ids() below. */ 8925struct recover_read_from_file_baton 8926{ 8927 apr_file_t *file; 8928 apr_pool_t *pool; 8929 apr_off_t remaining; 8930}; 8931 8932/* A stream read handler used by recover_find_max_ids() below. 8933 Read and return at most BATON->REMAINING bytes from the stream, 8934 returning nothing after that to indicate EOF. */ 8935static svn_error_t * 8936read_handler_recover(void *baton, char *buffer, apr_size_t *len) 8937{ 8938 struct recover_read_from_file_baton *b = baton; 8939 svn_filesize_t bytes_to_read = *len; 8940 8941 if (b->remaining == 0) 8942 { 8943 /* Return a successful read of zero bytes to signal EOF. */ 8944 *len = 0; 8945 return SVN_NO_ERROR; 8946 } 8947 8948 if (bytes_to_read > b->remaining) 8949 bytes_to_read = b->remaining; 8950 b->remaining -= bytes_to_read; 8951 8952 return svn_io_file_read_full2(b->file, buffer, (apr_size_t) bytes_to_read, 8953 len, NULL, b->pool); 8954} 8955 8956/* Part of the recovery procedure. Read the directory noderev at offset 8957 OFFSET of file REV_FILE (the revision file of revision REV of 8958 filesystem FS), and set MAX_NODE_ID and MAX_COPY_ID to be the node-id 8959 and copy-id of that node, if greater than the current value stored 8960 in either. Recurse into any child directories that were modified in 8961 this revision. 8962 8963 MAX_NODE_ID and MAX_COPY_ID must be arrays of at least MAX_KEY_SIZE. 8964 8965 Perform temporary allocation in POOL. */ 8966static svn_error_t * 8967recover_find_max_ids(svn_fs_t *fs, svn_revnum_t rev, 8968 apr_file_t *rev_file, apr_off_t offset, 8969 char *max_node_id, char *max_copy_id, 8970 apr_pool_t *pool) 8971{ 8972 apr_hash_t *headers; 8973 char *value; 8974 representation_t *data_rep; 8975 struct rep_args *ra; 8976 struct recover_read_from_file_baton baton; 8977 svn_stream_t *stream; 8978 apr_hash_t *entries; 8979 apr_hash_index_t *hi; 8980 apr_pool_t *iterpool; 8981 8982 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool)); 8983 SVN_ERR(read_header_block(&headers, svn_stream_from_aprfile2(rev_file, TRUE, 8984 pool), 8985 pool)); 8986 8987 /* Check that this is a directory. It should be. */ 8988 value = svn_hash_gets(headers, HEADER_TYPE); 8989 if (value == NULL || strcmp(value, KIND_DIR) != 0) 8990 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 8991 _("Recovery encountered a non-directory node")); 8992 8993 /* Get the data location. No data location indicates an empty directory. */ 8994 value = svn_hash_gets(headers, HEADER_TEXT); 8995 if (!value) 8996 return SVN_NO_ERROR; 8997 SVN_ERR(read_rep_offsets(&data_rep, value, NULL, FALSE, pool)); 8998 8999 /* If the directory's data representation wasn't changed in this revision, 9000 we've already scanned the directory's contents for noderevs, so we don't 9001 need to again. This will occur if a property is changed on a directory 9002 without changing the directory's contents. */ 9003 if (data_rep->revision != rev) 9004 return SVN_NO_ERROR; 9005 9006 /* We could use get_dir_contents(), but this is much cheaper. It does 9007 rely on directory entries being stored as PLAIN reps, though. */ 9008 offset = data_rep->offset; 9009 SVN_ERR(svn_io_file_seek(rev_file, APR_SET, &offset, pool)); 9010 SVN_ERR(read_rep_line(&ra, rev_file, pool)); 9011 if (ra->is_delta) 9012 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 9013 _("Recovery encountered a deltified directory " 9014 "representation")); 9015 9016 /* Now create a stream that's allowed to read only as much data as is 9017 stored in the representation. */ 9018 baton.file = rev_file; 9019 baton.pool = pool; 9020 baton.remaining = data_rep->expanded_size; 9021 stream = svn_stream_create(&baton, pool); 9022 svn_stream_set_read(stream, read_handler_recover); 9023 9024 /* Now read the entries from that stream. */ 9025 entries = apr_hash_make(pool); 9026 SVN_ERR(svn_hash_read2(entries, stream, SVN_HASH_TERMINATOR, pool)); 9027 SVN_ERR(svn_stream_close(stream)); 9028 9029 /* Now check each of the entries in our directory to find new node and 9030 copy ids, and recurse into new subdirectories. */ 9031 iterpool = svn_pool_create(pool); 9032 for (hi = apr_hash_first(pool, entries); hi; hi = apr_hash_next(hi)) 9033 { 9034 char *str_val; 9035 char *str; 9036 svn_node_kind_t kind; 9037 svn_fs_id_t *id; 9038 const char *node_id, *copy_id; 9039 apr_off_t child_dir_offset; 9040 const svn_string_t *path = svn__apr_hash_index_val(hi); 9041 9042 svn_pool_clear(iterpool); 9043 9044 str_val = apr_pstrdup(iterpool, path->data); 9045 9046 str = svn_cstring_tokenize(" ", &str_val); 9047 if (str == NULL) 9048 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 9049 _("Directory entry corrupt")); 9050 9051 if (strcmp(str, KIND_FILE) == 0) 9052 kind = svn_node_file; 9053 else if (strcmp(str, KIND_DIR) == 0) 9054 kind = svn_node_dir; 9055 else 9056 { 9057 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 9058 _("Directory entry corrupt")); 9059 } 9060 9061 str = svn_cstring_tokenize(" ", &str_val); 9062 if (str == NULL) 9063 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 9064 _("Directory entry corrupt")); 9065 9066 id = svn_fs_fs__id_parse(str, strlen(str), iterpool); 9067 9068 if (svn_fs_fs__id_rev(id) != rev) 9069 { 9070 /* If the node wasn't modified in this revision, we've already 9071 checked the node and copy id. */ 9072 continue; 9073 } 9074 9075 node_id = svn_fs_fs__id_node_id(id); 9076 copy_id = svn_fs_fs__id_copy_id(id); 9077 9078 if (svn_fs_fs__key_compare(node_id, max_node_id) > 0) 9079 { 9080 SVN_ERR_ASSERT(strlen(node_id) < MAX_KEY_SIZE); 9081 apr_cpystrn(max_node_id, node_id, MAX_KEY_SIZE); 9082 } 9083 if (svn_fs_fs__key_compare(copy_id, max_copy_id) > 0) 9084 { 9085 SVN_ERR_ASSERT(strlen(copy_id) < MAX_KEY_SIZE); 9086 apr_cpystrn(max_copy_id, copy_id, MAX_KEY_SIZE); 9087 } 9088 9089 if (kind == svn_node_file) 9090 continue; 9091 9092 child_dir_offset = svn_fs_fs__id_offset(id); 9093 SVN_ERR(recover_find_max_ids(fs, rev, rev_file, child_dir_offset, 9094 max_node_id, max_copy_id, iterpool)); 9095 } 9096 svn_pool_destroy(iterpool); 9097 9098 return SVN_NO_ERROR; 9099} 9100 9101/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. 9102 * Use POOL for temporary allocations. 9103 * Set *MISSING, if the reason is a missing manifest or pack file. 9104 */ 9105static svn_boolean_t 9106packed_revprop_available(svn_boolean_t *missing, 9107 svn_fs_t *fs, 9108 svn_revnum_t revision, 9109 apr_pool_t *pool) 9110{ 9111 fs_fs_data_t *ffd = fs->fsap_data; 9112 svn_stringbuf_t *content = NULL; 9113 9114 /* try to read the manifest file */ 9115 const char *folder = path_revprops_pack_shard(fs, revision, pool); 9116 const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool); 9117 9118 svn_error_t *err = try_stringbuf_from_file(&content, 9119 missing, 9120 manifest_path, 9121 FALSE, 9122 pool); 9123 9124 /* if the manifest cannot be read, consider the pack files inaccessible 9125 * even if the file itself exists. */ 9126 if (err) 9127 { 9128 svn_error_clear(err); 9129 return FALSE; 9130 } 9131 9132 if (*missing) 9133 return FALSE; 9134 9135 /* parse manifest content until we find the entry for REVISION. 9136 * Revision 0 is never packed. */ 9137 revision = revision < ffd->max_files_per_dir 9138 ? revision - 1 9139 : revision % ffd->max_files_per_dir; 9140 while (content->data) 9141 { 9142 char *next = strchr(content->data, '\n'); 9143 if (next) 9144 { 9145 *next = 0; 9146 ++next; 9147 } 9148 9149 if (revision-- == 0) 9150 { 9151 /* the respective pack file must exist (and be a file) */ 9152 svn_node_kind_t kind; 9153 err = svn_io_check_path(svn_dirent_join(folder, content->data, 9154 pool), 9155 &kind, pool); 9156 if (err) 9157 { 9158 svn_error_clear(err); 9159 return FALSE; 9160 } 9161 9162 *missing = kind == svn_node_none; 9163 return kind == svn_node_file; 9164 } 9165 9166 content->data = next; 9167 } 9168 9169 return FALSE; 9170} 9171 9172/* Baton used for recover_body below. */ 9173struct recover_baton { 9174 svn_fs_t *fs; 9175 svn_cancel_func_t cancel_func; 9176 void *cancel_baton; 9177}; 9178 9179/* The work-horse for svn_fs_fs__recover, called with the FS 9180 write lock. This implements the svn_fs_fs__with_write_lock() 9181 'body' callback type. BATON is a 'struct recover_baton *'. */ 9182static svn_error_t * 9183recover_body(void *baton, apr_pool_t *pool) 9184{ 9185 struct recover_baton *b = baton; 9186 svn_fs_t *fs = b->fs; 9187 fs_fs_data_t *ffd = fs->fsap_data; 9188 svn_revnum_t max_rev; 9189 char next_node_id_buf[MAX_KEY_SIZE], next_copy_id_buf[MAX_KEY_SIZE]; 9190 char *next_node_id = NULL, *next_copy_id = NULL; 9191 svn_revnum_t youngest_rev; 9192 svn_node_kind_t youngest_revprops_kind; 9193 9194 /* Lose potentially corrupted data in temp files */ 9195 SVN_ERR(cleanup_revprop_namespace(fs)); 9196 9197 /* We need to know the largest revision in the filesystem. */ 9198 SVN_ERR(recover_get_largest_revision(fs, &max_rev, pool)); 9199 9200 /* Get the expected youngest revision */ 9201 SVN_ERR(get_youngest(&youngest_rev, fs->path, pool)); 9202 9203 /* Policy note: 9204 9205 Since the revprops file is written after the revs file, the true 9206 maximum available revision is the youngest one for which both are 9207 present. That's probably the same as the max_rev we just found, 9208 but if it's not, we could, in theory, repeatedly decrement 9209 max_rev until we find a revision that has both a revs and 9210 revprops file, then write db/current with that. 9211 9212 But we choose not to. If a repository is so corrupt that it's 9213 missing at least one revprops file, we shouldn't assume that the 9214 youngest revision for which both the revs and revprops files are 9215 present is healthy. In other words, we're willing to recover 9216 from a missing or out-of-date db/current file, because db/current 9217 is truly redundant -- it's basically a cache so we don't have to 9218 find max_rev each time, albeit a cache with unusual semantics, 9219 since it also officially defines when a revision goes live. But 9220 if we're missing more than the cache, it's time to back out and 9221 let the admin reconstruct things by hand: correctness at that 9222 point may depend on external things like checking a commit email 9223 list, looking in particular working copies, etc. 9224 9225 This policy matches well with a typical naive backup scenario. 9226 Say you're rsyncing your FSFS repository nightly to the same 9227 location. Once revs and revprops are written, you've got the 9228 maximum rev; if the backup should bomb before db/current is 9229 written, then db/current could stay arbitrarily out-of-date, but 9230 we can still recover. It's a small window, but we might as well 9231 do what we can. */ 9232 9233 /* Even if db/current were missing, it would be created with 0 by 9234 get_youngest(), so this conditional remains valid. */ 9235 if (youngest_rev > max_rev) 9236 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 9237 _("Expected current rev to be <= %ld " 9238 "but found %ld"), max_rev, youngest_rev); 9239 9240 /* We only need to search for maximum IDs for old FS formats which 9241 se global ID counters. */ 9242 if (ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 9243 { 9244 /* Next we need to find the maximum node id and copy id in use across the 9245 filesystem. Unfortunately, the only way we can get this information 9246 is to scan all the noderevs of all the revisions and keep track as 9247 we go along. */ 9248 svn_revnum_t rev; 9249 apr_pool_t *iterpool = svn_pool_create(pool); 9250 char max_node_id[MAX_KEY_SIZE] = "0", max_copy_id[MAX_KEY_SIZE] = "0"; 9251 apr_size_t len; 9252 9253 for (rev = 0; rev <= max_rev; rev++) 9254 { 9255 apr_file_t *rev_file; 9256 apr_off_t root_offset; 9257 9258 svn_pool_clear(iterpool); 9259 9260 if (b->cancel_func) 9261 SVN_ERR(b->cancel_func(b->cancel_baton)); 9262 9263 SVN_ERR(open_pack_or_rev_file(&rev_file, fs, rev, iterpool)); 9264 SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file, fs, rev, 9265 iterpool)); 9266 SVN_ERR(recover_find_max_ids(fs, rev, rev_file, root_offset, 9267 max_node_id, max_copy_id, iterpool)); 9268 SVN_ERR(svn_io_file_close(rev_file, iterpool)); 9269 } 9270 svn_pool_destroy(iterpool); 9271 9272 /* Now that we finally have the maximum revision, node-id and copy-id, we 9273 can bump the two ids to get the next of each. */ 9274 len = strlen(max_node_id); 9275 svn_fs_fs__next_key(max_node_id, &len, next_node_id_buf); 9276 next_node_id = next_node_id_buf; 9277 len = strlen(max_copy_id); 9278 svn_fs_fs__next_key(max_copy_id, &len, next_copy_id_buf); 9279 next_copy_id = next_copy_id_buf; 9280 } 9281 9282 /* Before setting current, verify that there is a revprops file 9283 for the youngest revision. (Issue #2992) */ 9284 SVN_ERR(svn_io_check_path(path_revprops(fs, max_rev, pool), 9285 &youngest_revprops_kind, pool)); 9286 if (youngest_revprops_kind == svn_node_none) 9287 { 9288 svn_boolean_t missing = TRUE; 9289 if (!packed_revprop_available(&missing, fs, max_rev, pool)) 9290 { 9291 if (missing) 9292 { 9293 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 9294 _("Revision %ld has a revs file but no " 9295 "revprops file"), 9296 max_rev); 9297 } 9298 else 9299 { 9300 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 9301 _("Revision %ld has a revs file but the " 9302 "revprops file is inaccessible"), 9303 max_rev); 9304 } 9305 } 9306 } 9307 else if (youngest_revprops_kind != svn_node_file) 9308 { 9309 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 9310 _("Revision %ld has a non-file where its " 9311 "revprops file should be"), 9312 max_rev); 9313 } 9314 9315 /* Prune younger-than-(newfound-youngest) revisions from the rep 9316 cache if sharing is enabled taking care not to create the cache 9317 if it does not exist. */ 9318 if (ffd->rep_sharing_allowed) 9319 { 9320 svn_boolean_t rep_cache_exists; 9321 9322 SVN_ERR(svn_fs_fs__exists_rep_cache(&rep_cache_exists, fs, pool)); 9323 if (rep_cache_exists) 9324 SVN_ERR(svn_fs_fs__del_rep_reference(fs, max_rev, pool)); 9325 } 9326 9327 /* Now store the discovered youngest revision, and the next IDs if 9328 relevant, in a new 'current' file. */ 9329 return write_current(fs, max_rev, next_node_id, next_copy_id, pool); 9330} 9331 9332/* This implements the fs_library_vtable_t.recover() API. */ 9333svn_error_t * 9334svn_fs_fs__recover(svn_fs_t *fs, 9335 svn_cancel_func_t cancel_func, void *cancel_baton, 9336 apr_pool_t *pool) 9337{ 9338 struct recover_baton b; 9339 9340 /* We have no way to take out an exclusive lock in FSFS, so we're 9341 restricted as to the types of recovery we can do. Luckily, 9342 we just want to recreate the 'current' file, and we can do that just 9343 by blocking other writers. */ 9344 b.fs = fs; 9345 b.cancel_func = cancel_func; 9346 b.cancel_baton = cancel_baton; 9347 return svn_fs_fs__with_write_lock(fs, recover_body, &b, pool); 9348} 9349 9350svn_error_t * 9351svn_fs_fs__set_uuid(svn_fs_t *fs, 9352 const char *uuid, 9353 apr_pool_t *pool) 9354{ 9355 char *my_uuid; 9356 apr_size_t my_uuid_len; 9357 const char *tmp_path; 9358 const char *uuid_path = path_uuid(fs, pool); 9359 9360 if (! uuid) 9361 uuid = svn_uuid_generate(pool); 9362 9363 /* Make sure we have a copy in FS->POOL, and append a newline. */ 9364 my_uuid = apr_pstrcat(fs->pool, uuid, "\n", (char *)NULL); 9365 my_uuid_len = strlen(my_uuid); 9366 9367 SVN_ERR(svn_io_write_unique(&tmp_path, 9368 svn_dirent_dirname(uuid_path, pool), 9369 my_uuid, my_uuid_len, 9370 svn_io_file_del_none, pool)); 9371 9372 /* We use the permissions of the 'current' file, because the 'uuid' 9373 file does not exist during repository creation. */ 9374 SVN_ERR(move_into_place(tmp_path, uuid_path, 9375 svn_fs_fs__path_current(fs, pool), pool)); 9376 9377 /* Remove the newline we added, and stash the UUID. */ 9378 my_uuid[my_uuid_len - 1] = '\0'; 9379 fs->uuid = my_uuid; 9380 9381 return SVN_NO_ERROR; 9382} 9383 9384/** Node origin lazy cache. */ 9385 9386/* If directory PATH does not exist, create it and give it the same 9387 permissions as FS_path.*/ 9388svn_error_t * 9389svn_fs_fs__ensure_dir_exists(const char *path, 9390 const char *fs_path, 9391 apr_pool_t *pool) 9392{ 9393 svn_error_t *err = svn_io_dir_make(path, APR_OS_DEFAULT, pool); 9394 if (err && APR_STATUS_IS_EEXIST(err->apr_err)) 9395 { 9396 svn_error_clear(err); 9397 return SVN_NO_ERROR; 9398 } 9399 SVN_ERR(err); 9400 9401 /* We successfully created a new directory. Dup the permissions 9402 from FS->path. */ 9403 return svn_io_copy_perms(fs_path, path, pool); 9404} 9405 9406/* Set *NODE_ORIGINS to a hash mapping 'const char *' node IDs to 9407 'svn_string_t *' node revision IDs. Use POOL for allocations. */ 9408static svn_error_t * 9409get_node_origins_from_file(svn_fs_t *fs, 9410 apr_hash_t **node_origins, 9411 const char *node_origins_file, 9412 apr_pool_t *pool) 9413{ 9414 apr_file_t *fd; 9415 svn_error_t *err; 9416 svn_stream_t *stream; 9417 9418 *node_origins = NULL; 9419 err = svn_io_file_open(&fd, node_origins_file, 9420 APR_READ, APR_OS_DEFAULT, pool); 9421 if (err && APR_STATUS_IS_ENOENT(err->apr_err)) 9422 { 9423 svn_error_clear(err); 9424 return SVN_NO_ERROR; 9425 } 9426 SVN_ERR(err); 9427 9428 stream = svn_stream_from_aprfile2(fd, FALSE, pool); 9429 *node_origins = apr_hash_make(pool); 9430 SVN_ERR(svn_hash_read2(*node_origins, stream, SVN_HASH_TERMINATOR, pool)); 9431 return svn_stream_close(stream); 9432} 9433 9434svn_error_t * 9435svn_fs_fs__get_node_origin(const svn_fs_id_t **origin_id, 9436 svn_fs_t *fs, 9437 const char *node_id, 9438 apr_pool_t *pool) 9439{ 9440 apr_hash_t *node_origins; 9441 9442 *origin_id = NULL; 9443 SVN_ERR(get_node_origins_from_file(fs, &node_origins, 9444 path_node_origin(fs, node_id, pool), 9445 pool)); 9446 if (node_origins) 9447 { 9448 svn_string_t *origin_id_str = 9449 svn_hash_gets(node_origins, node_id); 9450 if (origin_id_str) 9451 *origin_id = svn_fs_fs__id_parse(origin_id_str->data, 9452 origin_id_str->len, pool); 9453 } 9454 return SVN_NO_ERROR; 9455} 9456 9457 9458/* Helper for svn_fs_fs__set_node_origin. Takes a NODE_ID/NODE_REV_ID 9459 pair and adds it to the NODE_ORIGINS_PATH file. */ 9460static svn_error_t * 9461set_node_origins_for_file(svn_fs_t *fs, 9462 const char *node_origins_path, 9463 const char *node_id, 9464 svn_string_t *node_rev_id, 9465 apr_pool_t *pool) 9466{ 9467 const char *path_tmp; 9468 svn_stream_t *stream; 9469 apr_hash_t *origins_hash; 9470 svn_string_t *old_node_rev_id; 9471 9472 SVN_ERR(svn_fs_fs__ensure_dir_exists(svn_dirent_join(fs->path, 9473 PATH_NODE_ORIGINS_DIR, 9474 pool), 9475 fs->path, pool)); 9476 9477 /* Read the previously existing origins (if any), and merge our 9478 update with it. */ 9479 SVN_ERR(get_node_origins_from_file(fs, &origins_hash, 9480 node_origins_path, pool)); 9481 if (! origins_hash) 9482 origins_hash = apr_hash_make(pool); 9483 9484 old_node_rev_id = svn_hash_gets(origins_hash, node_id); 9485 9486 if (old_node_rev_id && !svn_string_compare(node_rev_id, old_node_rev_id)) 9487 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 9488 _("Node origin for '%s' exists with a different " 9489 "value (%s) than what we were about to store " 9490 "(%s)"), 9491 node_id, old_node_rev_id->data, node_rev_id->data); 9492 9493 svn_hash_sets(origins_hash, node_id, node_rev_id); 9494 9495 /* Sure, there's a race condition here. Two processes could be 9496 trying to add different cache elements to the same file at the 9497 same time, and the entries added by the first one to write will 9498 be lost. But this is just a cache of reconstructible data, so 9499 we'll accept this problem in return for not having to deal with 9500 locking overhead. */ 9501 9502 /* Create a temporary file, write out our hash, and close the file. */ 9503 SVN_ERR(svn_stream_open_unique(&stream, &path_tmp, 9504 svn_dirent_dirname(node_origins_path, pool), 9505 svn_io_file_del_none, pool, pool)); 9506 SVN_ERR(svn_hash_write2(origins_hash, stream, SVN_HASH_TERMINATOR, pool)); 9507 SVN_ERR(svn_stream_close(stream)); 9508 9509 /* Rename the temp file as the real destination */ 9510 return svn_io_file_rename(path_tmp, node_origins_path, pool); 9511} 9512 9513 9514svn_error_t * 9515svn_fs_fs__set_node_origin(svn_fs_t *fs, 9516 const char *node_id, 9517 const svn_fs_id_t *node_rev_id, 9518 apr_pool_t *pool) 9519{ 9520 svn_error_t *err; 9521 const char *filename = path_node_origin(fs, node_id, pool); 9522 9523 err = set_node_origins_for_file(fs, filename, 9524 node_id, 9525 svn_fs_fs__id_unparse(node_rev_id, pool), 9526 pool); 9527 if (err && APR_STATUS_IS_EACCES(err->apr_err)) 9528 { 9529 /* It's just a cache; stop trying if I can't write. */ 9530 svn_error_clear(err); 9531 err = NULL; 9532 } 9533 return svn_error_trace(err); 9534} 9535 9536 9537svn_error_t * 9538svn_fs_fs__list_transactions(apr_array_header_t **names_p, 9539 svn_fs_t *fs, 9540 apr_pool_t *pool) 9541{ 9542 const char *txn_dir; 9543 apr_hash_t *dirents; 9544 apr_hash_index_t *hi; 9545 apr_array_header_t *names; 9546 apr_size_t ext_len = strlen(PATH_EXT_TXN); 9547 9548 names = apr_array_make(pool, 1, sizeof(const char *)); 9549 9550 /* Get the transactions directory. */ 9551 txn_dir = svn_dirent_join(fs->path, PATH_TXNS_DIR, pool); 9552 9553 /* Now find a listing of this directory. */ 9554 SVN_ERR(svn_io_get_dirents3(&dirents, txn_dir, TRUE, pool, pool)); 9555 9556 /* Loop through all the entries and return anything that ends with '.txn'. */ 9557 for (hi = apr_hash_first(pool, dirents); hi; hi = apr_hash_next(hi)) 9558 { 9559 const char *name = svn__apr_hash_index_key(hi); 9560 apr_ssize_t klen = svn__apr_hash_index_klen(hi); 9561 const char *id; 9562 9563 /* The name must end with ".txn" to be considered a transaction. */ 9564 if ((apr_size_t) klen <= ext_len 9565 || (strcmp(name + klen - ext_len, PATH_EXT_TXN)) != 0) 9566 continue; 9567 9568 /* Truncate the ".txn" extension and store the ID. */ 9569 id = apr_pstrndup(pool, name, strlen(name) - ext_len); 9570 APR_ARRAY_PUSH(names, const char *) = id; 9571 } 9572 9573 *names_p = names; 9574 9575 return SVN_NO_ERROR; 9576} 9577 9578svn_error_t * 9579svn_fs_fs__open_txn(svn_fs_txn_t **txn_p, 9580 svn_fs_t *fs, 9581 const char *name, 9582 apr_pool_t *pool) 9583{ 9584 svn_fs_txn_t *txn; 9585 svn_node_kind_t kind; 9586 transaction_t *local_txn; 9587 9588 /* First check to see if the directory exists. */ 9589 SVN_ERR(svn_io_check_path(path_txn_dir(fs, name, pool), &kind, pool)); 9590 9591 /* Did we find it? */ 9592 if (kind != svn_node_dir) 9593 return svn_error_createf(SVN_ERR_FS_NO_SUCH_TRANSACTION, NULL, 9594 _("No such transaction '%s'"), 9595 name); 9596 9597 txn = apr_pcalloc(pool, sizeof(*txn)); 9598 9599 /* Read in the root node of this transaction. */ 9600 txn->id = apr_pstrdup(pool, name); 9601 txn->fs = fs; 9602 9603 SVN_ERR(svn_fs_fs__get_txn(&local_txn, fs, name, pool)); 9604 9605 txn->base_rev = svn_fs_fs__id_rev(local_txn->base_id); 9606 9607 txn->vtable = &txn_vtable; 9608 *txn_p = txn; 9609 9610 return SVN_NO_ERROR; 9611} 9612 9613svn_error_t * 9614svn_fs_fs__txn_proplist(apr_hash_t **table_p, 9615 svn_fs_txn_t *txn, 9616 apr_pool_t *pool) 9617{ 9618 apr_hash_t *proplist = apr_hash_make(pool); 9619 SVN_ERR(get_txn_proplist(proplist, txn->fs, txn->id, pool)); 9620 *table_p = proplist; 9621 9622 return SVN_NO_ERROR; 9623} 9624 9625svn_error_t * 9626svn_fs_fs__delete_node_revision(svn_fs_t *fs, 9627 const svn_fs_id_t *id, 9628 apr_pool_t *pool) 9629{ 9630 node_revision_t *noderev; 9631 9632 SVN_ERR(svn_fs_fs__get_node_revision(&noderev, fs, id, pool)); 9633 9634 /* Delete any mutable property representation. */ 9635 if (noderev->prop_rep && noderev->prop_rep->txn_id) 9636 SVN_ERR(svn_io_remove_file2(path_txn_node_props(fs, id, pool), FALSE, 9637 pool)); 9638 9639 /* Delete any mutable data representation. */ 9640 if (noderev->data_rep && noderev->data_rep->txn_id 9641 && noderev->kind == svn_node_dir) 9642 { 9643 fs_fs_data_t *ffd = fs->fsap_data; 9644 SVN_ERR(svn_io_remove_file2(path_txn_node_children(fs, id, pool), FALSE, 9645 pool)); 9646 9647 /* remove the corresponding entry from the cache, if such exists */ 9648 if (ffd->txn_dir_cache) 9649 { 9650 const char *key = svn_fs_fs__id_unparse(id, pool)->data; 9651 SVN_ERR(svn_cache__set(ffd->txn_dir_cache, key, NULL, pool)); 9652 } 9653 } 9654 9655 return svn_io_remove_file2(path_txn_node_rev(fs, id, pool), FALSE, pool); 9656} 9657 9658 9659 9660/*** Revisions ***/ 9661 9662svn_error_t * 9663svn_fs_fs__revision_prop(svn_string_t **value_p, 9664 svn_fs_t *fs, 9665 svn_revnum_t rev, 9666 const char *propname, 9667 apr_pool_t *pool) 9668{ 9669 apr_hash_t *table; 9670 9671 SVN_ERR(svn_fs__check_fs(fs, TRUE)); 9672 SVN_ERR(svn_fs_fs__revision_proplist(&table, fs, rev, pool)); 9673 9674 *value_p = svn_hash_gets(table, propname); 9675 9676 return SVN_NO_ERROR; 9677} 9678 9679 9680/* Baton used for change_rev_prop_body below. */ 9681struct change_rev_prop_baton { 9682 svn_fs_t *fs; 9683 svn_revnum_t rev; 9684 const char *name; 9685 const svn_string_t *const *old_value_p; 9686 const svn_string_t *value; 9687}; 9688 9689/* The work-horse for svn_fs_fs__change_rev_prop, called with the FS 9690 write lock. This implements the svn_fs_fs__with_write_lock() 9691 'body' callback type. BATON is a 'struct change_rev_prop_baton *'. */ 9692static svn_error_t * 9693change_rev_prop_body(void *baton, apr_pool_t *pool) 9694{ 9695 struct change_rev_prop_baton *cb = baton; 9696 apr_hash_t *table; 9697 9698 SVN_ERR(svn_fs_fs__revision_proplist(&table, cb->fs, cb->rev, pool)); 9699 9700 if (cb->old_value_p) 9701 { 9702 const svn_string_t *wanted_value = *cb->old_value_p; 9703 const svn_string_t *present_value = svn_hash_gets(table, cb->name); 9704 if ((!wanted_value != !present_value) 9705 || (wanted_value && present_value 9706 && !svn_string_compare(wanted_value, present_value))) 9707 { 9708 /* What we expected isn't what we found. */ 9709 return svn_error_createf(SVN_ERR_FS_PROP_BASEVALUE_MISMATCH, NULL, 9710 _("revprop '%s' has unexpected value in " 9711 "filesystem"), 9712 cb->name); 9713 } 9714 /* Fall through. */ 9715 } 9716 svn_hash_sets(table, cb->name, cb->value); 9717 9718 return set_revision_proplist(cb->fs, cb->rev, table, pool); 9719} 9720 9721svn_error_t * 9722svn_fs_fs__change_rev_prop(svn_fs_t *fs, 9723 svn_revnum_t rev, 9724 const char *name, 9725 const svn_string_t *const *old_value_p, 9726 const svn_string_t *value, 9727 apr_pool_t *pool) 9728{ 9729 struct change_rev_prop_baton cb; 9730 9731 SVN_ERR(svn_fs__check_fs(fs, TRUE)); 9732 9733 cb.fs = fs; 9734 cb.rev = rev; 9735 cb.name = name; 9736 cb.old_value_p = old_value_p; 9737 cb.value = value; 9738 9739 return svn_fs_fs__with_write_lock(fs, change_rev_prop_body, &cb, pool); 9740} 9741 9742 9743 9744/*** Transactions ***/ 9745 9746svn_error_t * 9747svn_fs_fs__get_txn_ids(const svn_fs_id_t **root_id_p, 9748 const svn_fs_id_t **base_root_id_p, 9749 svn_fs_t *fs, 9750 const char *txn_name, 9751 apr_pool_t *pool) 9752{ 9753 transaction_t *txn; 9754 SVN_ERR(svn_fs_fs__get_txn(&txn, fs, txn_name, pool)); 9755 *root_id_p = txn->root_id; 9756 *base_root_id_p = txn->base_id; 9757 return SVN_NO_ERROR; 9758} 9759 9760 9761/* Generic transaction operations. */ 9762 9763svn_error_t * 9764svn_fs_fs__txn_prop(svn_string_t **value_p, 9765 svn_fs_txn_t *txn, 9766 const char *propname, 9767 apr_pool_t *pool) 9768{ 9769 apr_hash_t *table; 9770 svn_fs_t *fs = txn->fs; 9771 9772 SVN_ERR(svn_fs__check_fs(fs, TRUE)); 9773 SVN_ERR(svn_fs_fs__txn_proplist(&table, txn, pool)); 9774 9775 *value_p = svn_hash_gets(table, propname); 9776 9777 return SVN_NO_ERROR; 9778} 9779 9780svn_error_t * 9781svn_fs_fs__begin_txn(svn_fs_txn_t **txn_p, 9782 svn_fs_t *fs, 9783 svn_revnum_t rev, 9784 apr_uint32_t flags, 9785 apr_pool_t *pool) 9786{ 9787 svn_string_t date; 9788 svn_prop_t prop; 9789 apr_array_header_t *props = apr_array_make(pool, 3, sizeof(svn_prop_t)); 9790 9791 SVN_ERR(svn_fs__check_fs(fs, TRUE)); 9792 9793 SVN_ERR(svn_fs_fs__create_txn(txn_p, fs, rev, pool)); 9794 9795 /* Put a datestamp on the newly created txn, so we always know 9796 exactly how old it is. (This will help sysadmins identify 9797 long-abandoned txns that may need to be manually removed.) When 9798 a txn is promoted to a revision, this property will be 9799 automatically overwritten with a revision datestamp. */ 9800 date.data = svn_time_to_cstring(apr_time_now(), pool); 9801 date.len = strlen(date.data); 9802 9803 prop.name = SVN_PROP_REVISION_DATE; 9804 prop.value = &date; 9805 APR_ARRAY_PUSH(props, svn_prop_t) = prop; 9806 9807 /* Set temporary txn props that represent the requested 'flags' 9808 behaviors. */ 9809 if (flags & SVN_FS_TXN_CHECK_OOD) 9810 { 9811 prop.name = SVN_FS__PROP_TXN_CHECK_OOD; 9812 prop.value = svn_string_create("true", pool); 9813 APR_ARRAY_PUSH(props, svn_prop_t) = prop; 9814 } 9815 9816 if (flags & SVN_FS_TXN_CHECK_LOCKS) 9817 { 9818 prop.name = SVN_FS__PROP_TXN_CHECK_LOCKS; 9819 prop.value = svn_string_create("true", pool); 9820 APR_ARRAY_PUSH(props, svn_prop_t) = prop; 9821 } 9822 9823 return svn_fs_fs__change_txn_props(*txn_p, props, pool); 9824} 9825 9826 9827/****** Packing FSFS shards *********/ 9828 9829/* Write a file FILENAME in directory FS_PATH, containing a single line 9830 * with the number REVNUM in ASCII decimal. Move the file into place 9831 * atomically, overwriting any existing file. 9832 * 9833 * Similar to write_current(). */ 9834static svn_error_t * 9835write_revnum_file(const char *fs_path, 9836 const char *filename, 9837 svn_revnum_t revnum, 9838 apr_pool_t *scratch_pool) 9839{ 9840 const char *final_path, *tmp_path; 9841 svn_stream_t *tmp_stream; 9842 9843 final_path = svn_dirent_join(fs_path, filename, scratch_pool); 9844 SVN_ERR(svn_stream_open_unique(&tmp_stream, &tmp_path, fs_path, 9845 svn_io_file_del_none, 9846 scratch_pool, scratch_pool)); 9847 SVN_ERR(svn_stream_printf(tmp_stream, scratch_pool, "%ld\n", revnum)); 9848 SVN_ERR(svn_stream_close(tmp_stream)); 9849 SVN_ERR(move_into_place(tmp_path, final_path, final_path, scratch_pool)); 9850 return SVN_NO_ERROR; 9851} 9852 9853/* Pack the revision SHARD containing exactly MAX_FILES_PER_DIR revisions 9854 * from SHARD_PATH into the PACK_FILE_DIR, using POOL for allocations. 9855 * CANCEL_FUNC and CANCEL_BATON are what you think they are. 9856 * 9857 * If for some reason we detect a partial packing already performed, we 9858 * remove the pack file and start again. 9859 */ 9860static svn_error_t * 9861pack_rev_shard(const char *pack_file_dir, 9862 const char *shard_path, 9863 apr_int64_t shard, 9864 int max_files_per_dir, 9865 svn_cancel_func_t cancel_func, 9866 void *cancel_baton, 9867 apr_pool_t *pool) 9868{ 9869 const char *pack_file_path, *manifest_file_path; 9870 svn_stream_t *pack_stream, *manifest_stream; 9871 svn_revnum_t start_rev, end_rev, rev; 9872 apr_off_t next_offset; 9873 apr_pool_t *iterpool; 9874 9875 /* Some useful paths. */ 9876 pack_file_path = svn_dirent_join(pack_file_dir, PATH_PACKED, pool); 9877 manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, pool); 9878 9879 /* Remove any existing pack file for this shard, since it is incomplete. */ 9880 SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, 9881 pool)); 9882 9883 /* Create the new directory and pack and manifest files. */ 9884 SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, pool)); 9885 SVN_ERR(svn_stream_open_writable(&pack_stream, pack_file_path, pool, 9886 pool)); 9887 SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path, 9888 pool, pool)); 9889 9890 start_rev = (svn_revnum_t) (shard * max_files_per_dir); 9891 end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); 9892 next_offset = 0; 9893 iterpool = svn_pool_create(pool); 9894 9895 /* Iterate over the revisions in this shard, squashing them together. */ 9896 for (rev = start_rev; rev <= end_rev; rev++) 9897 { 9898 svn_stream_t *rev_stream; 9899 apr_finfo_t finfo; 9900 const char *path; 9901 9902 svn_pool_clear(iterpool); 9903 9904 /* Get the size of the file. */ 9905 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 9906 iterpool); 9907 SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); 9908 9909 /* Update the manifest. */ 9910 SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%" APR_OFF_T_FMT 9911 "\n", next_offset)); 9912 next_offset += finfo.size; 9913 9914 /* Copy all the bits from the rev file to the end of the pack file. */ 9915 SVN_ERR(svn_stream_open_readonly(&rev_stream, path, iterpool, iterpool)); 9916 SVN_ERR(svn_stream_copy3(rev_stream, svn_stream_disown(pack_stream, 9917 iterpool), 9918 cancel_func, cancel_baton, iterpool)); 9919 } 9920 9921 SVN_ERR(svn_stream_close(manifest_stream)); 9922 SVN_ERR(svn_stream_close(pack_stream)); 9923 SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); 9924 SVN_ERR(svn_io_set_file_read_only(pack_file_path, FALSE, iterpool)); 9925 SVN_ERR(svn_io_set_file_read_only(manifest_file_path, FALSE, iterpool)); 9926 9927 svn_pool_destroy(iterpool); 9928 9929 return SVN_NO_ERROR; 9930} 9931 9932/* Copy revprop files for revisions [START_REV, END_REV) from SHARD_PATH 9933 * to the pack file at PACK_FILE_NAME in PACK_FILE_DIR. 9934 * 9935 * The file sizes have already been determined and written to SIZES. 9936 * Please note that this function will be executed while the filesystem 9937 * has been locked and that revprops files will therefore not be modified 9938 * while the pack is in progress. 9939 * 9940 * COMPRESSION_LEVEL defines how well the resulting pack file shall be 9941 * compressed or whether is shall be compressed at all. TOTAL_SIZE is 9942 * a hint on which initial buffer size we should use to hold the pack file 9943 * content. 9944 * 9945 * CANCEL_FUNC and CANCEL_BATON are used as usual. Temporary allocations 9946 * are done in SCRATCH_POOL. 9947 */ 9948static svn_error_t * 9949copy_revprops(const char *pack_file_dir, 9950 const char *pack_filename, 9951 const char *shard_path, 9952 svn_revnum_t start_rev, 9953 svn_revnum_t end_rev, 9954 apr_array_header_t *sizes, 9955 apr_size_t total_size, 9956 int compression_level, 9957 svn_cancel_func_t cancel_func, 9958 void *cancel_baton, 9959 apr_pool_t *scratch_pool) 9960{ 9961 svn_stream_t *pack_stream; 9962 apr_file_t *pack_file; 9963 svn_revnum_t rev; 9964 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 9965 svn_stream_t *stream; 9966 9967 /* create empty data buffer and a write stream on top of it */ 9968 svn_stringbuf_t *uncompressed 9969 = svn_stringbuf_create_ensure(total_size, scratch_pool); 9970 svn_stringbuf_t *compressed 9971 = svn_stringbuf_create_empty(scratch_pool); 9972 pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 9973 9974 /* write the pack file header */ 9975 SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, 9976 sizes->nelts, iterpool)); 9977 9978 /* Some useful paths. */ 9979 SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, 9980 pack_filename, 9981 scratch_pool), 9982 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, 9983 scratch_pool)); 9984 9985 /* Iterate over the revisions in this shard, squashing them together. */ 9986 for (rev = start_rev; rev <= end_rev; rev++) 9987 { 9988 const char *path; 9989 9990 svn_pool_clear(iterpool); 9991 9992 /* Construct the file name. */ 9993 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 9994 iterpool); 9995 9996 /* Copy all the bits from the non-packed revprop file to the end of 9997 * the pack file. */ 9998 SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); 9999 SVN_ERR(svn_stream_copy3(stream, pack_stream, 10000 cancel_func, cancel_baton, iterpool)); 10001 } 10002 10003 /* flush stream buffers to content buffer */ 10004 SVN_ERR(svn_stream_close(pack_stream)); 10005 10006 /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ 10007 SVN_ERR(svn__compress(svn_stringbuf__morph_into_string(uncompressed), 10008 compressed, compression_level)); 10009 10010 /* write the pack file content to disk */ 10011 stream = svn_stream_from_aprfile2(pack_file, FALSE, scratch_pool); 10012 SVN_ERR(svn_stream_write(stream, compressed->data, &compressed->len)); 10013 SVN_ERR(svn_stream_close(stream)); 10014 10015 svn_pool_destroy(iterpool); 10016 10017 return SVN_NO_ERROR; 10018} 10019 10020/* For the revprop SHARD at SHARD_PATH with exactly MAX_FILES_PER_DIR 10021 * revprop files in it, create a packed shared at PACK_FILE_DIR. 10022 * 10023 * COMPRESSION_LEVEL defines how well the resulting pack file shall be 10024 * compressed or whether is shall be compressed at all. Individual pack 10025 * file containing more than one revision will be limited to a size of 10026 * MAX_PACK_SIZE bytes before compression. 10027 * 10028 * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary 10029 * allocations are done in SCRATCH_POOL. 10030 */ 10031static svn_error_t * 10032pack_revprops_shard(const char *pack_file_dir, 10033 const char *shard_path, 10034 apr_int64_t shard, 10035 int max_files_per_dir, 10036 apr_off_t max_pack_size, 10037 int compression_level, 10038 svn_cancel_func_t cancel_func, 10039 void *cancel_baton, 10040 apr_pool_t *scratch_pool) 10041{ 10042 const char *manifest_file_path, *pack_filename = NULL; 10043 svn_stream_t *manifest_stream; 10044 svn_revnum_t start_rev, end_rev, rev; 10045 apr_off_t total_size; 10046 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 10047 apr_array_header_t *sizes; 10048 10049 /* Some useful paths. */ 10050 manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, 10051 scratch_pool); 10052 10053 /* Remove any existing pack file for this shard, since it is incomplete. */ 10054 SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, 10055 scratch_pool)); 10056 10057 /* Create the new directory and manifest file stream. */ 10058 SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); 10059 SVN_ERR(svn_stream_open_writable(&manifest_stream, manifest_file_path, 10060 scratch_pool, scratch_pool)); 10061 10062 /* revisions to handle. Special case: revision 0 */ 10063 start_rev = (svn_revnum_t) (shard * max_files_per_dir); 10064 end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); 10065 if (start_rev == 0) 10066 ++start_rev; 10067 10068 /* initialize the revprop size info */ 10069 sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); 10070 total_size = 2 * SVN_INT64_BUFFER_SIZE; 10071 10072 /* Iterate over the revisions in this shard, determine their size and 10073 * squashing them together into pack files. */ 10074 for (rev = start_rev; rev <= end_rev; rev++) 10075 { 10076 apr_finfo_t finfo; 10077 const char *path; 10078 10079 svn_pool_clear(iterpool); 10080 10081 /* Get the size of the file. */ 10082 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 10083 iterpool); 10084 SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); 10085 10086 /* if we already have started a pack file and this revprop cannot be 10087 * appended to it, write the previous pack file. */ 10088 if (sizes->nelts != 0 && 10089 total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) 10090 { 10091 SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path, 10092 start_rev, rev-1, sizes, (apr_size_t)total_size, 10093 compression_level, cancel_func, cancel_baton, 10094 iterpool)); 10095 10096 /* next pack file starts empty again */ 10097 apr_array_clear(sizes); 10098 total_size = 2 * SVN_INT64_BUFFER_SIZE; 10099 start_rev = rev; 10100 } 10101 10102 /* Update the manifest. Allocate a file name for the current pack 10103 * file if it is a new one */ 10104 if (sizes->nelts == 0) 10105 pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); 10106 10107 SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", 10108 pack_filename)); 10109 10110 /* add to list of files to put into the current pack file */ 10111 APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; 10112 total_size += SVN_INT64_BUFFER_SIZE + finfo.size; 10113 } 10114 10115 /* write the last pack file */ 10116 if (sizes->nelts != 0) 10117 SVN_ERR(copy_revprops(pack_file_dir, pack_filename, shard_path, 10118 start_rev, rev-1, sizes, (apr_size_t)total_size, 10119 compression_level, cancel_func, cancel_baton, 10120 iterpool)); 10121 10122 /* flush the manifest file and update permissions */ 10123 SVN_ERR(svn_stream_close(manifest_stream)); 10124 SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); 10125 10126 svn_pool_destroy(iterpool); 10127 10128 return SVN_NO_ERROR; 10129} 10130 10131/* Delete the non-packed revprop SHARD at SHARD_PATH with exactly 10132 * MAX_FILES_PER_DIR revprop files in it. If this is shard 0, keep the 10133 * revprop file for revision 0. 10134 * 10135 * CANCEL_FUNC and CANCEL_BATON are used in the usual way. Temporary 10136 * allocations are done in SCRATCH_POOL. 10137 */ 10138static svn_error_t * 10139delete_revprops_shard(const char *shard_path, 10140 apr_int64_t shard, 10141 int max_files_per_dir, 10142 svn_cancel_func_t cancel_func, 10143 void *cancel_baton, 10144 apr_pool_t *scratch_pool) 10145{ 10146 if (shard == 0) 10147 { 10148 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 10149 int i; 10150 10151 /* delete all files except the one for revision 0 */ 10152 for (i = 1; i < max_files_per_dir; ++i) 10153 { 10154 const char *path = svn_dirent_join(shard_path, 10155 apr_psprintf(iterpool, "%d", i), 10156 iterpool); 10157 if (cancel_func) 10158 SVN_ERR((*cancel_func)(cancel_baton)); 10159 10160 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 10161 svn_pool_clear(iterpool); 10162 } 10163 10164 svn_pool_destroy(iterpool); 10165 } 10166 else 10167 SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, 10168 cancel_func, cancel_baton, scratch_pool)); 10169 10170 return SVN_NO_ERROR; 10171} 10172 10173/* In the file system at FS_PATH, pack the SHARD in REVS_DIR and 10174 * REVPROPS_DIR containing exactly MAX_FILES_PER_DIR revisions, using POOL 10175 * for allocations. REVPROPS_DIR will be NULL if revprop packing is not 10176 * supported. COMPRESSION_LEVEL and MAX_PACK_SIZE will be ignored in that 10177 * case. 10178 * 10179 * CANCEL_FUNC and CANCEL_BATON are what you think they are; similarly 10180 * NOTIFY_FUNC and NOTIFY_BATON. 10181 * 10182 * If for some reason we detect a partial packing already performed, we 10183 * remove the pack file and start again. 10184 */ 10185static svn_error_t * 10186pack_shard(const char *revs_dir, 10187 const char *revsprops_dir, 10188 const char *fs_path, 10189 apr_int64_t shard, 10190 int max_files_per_dir, 10191 apr_off_t max_pack_size, 10192 int compression_level, 10193 svn_fs_pack_notify_t notify_func, 10194 void *notify_baton, 10195 svn_cancel_func_t cancel_func, 10196 void *cancel_baton, 10197 apr_pool_t *pool) 10198{ 10199 const char *rev_shard_path, *rev_pack_file_dir; 10200 const char *revprops_shard_path, *revprops_pack_file_dir; 10201 10202 /* Notify caller we're starting to pack this shard. */ 10203 if (notify_func) 10204 SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_start, 10205 pool)); 10206 10207 /* Some useful paths. */ 10208 rev_pack_file_dir = svn_dirent_join(revs_dir, 10209 apr_psprintf(pool, 10210 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, 10211 shard), 10212 pool); 10213 rev_shard_path = svn_dirent_join(revs_dir, 10214 apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), 10215 pool); 10216 10217 /* pack the revision content */ 10218 SVN_ERR(pack_rev_shard(rev_pack_file_dir, rev_shard_path, 10219 shard, max_files_per_dir, 10220 cancel_func, cancel_baton, pool)); 10221 10222 /* if enabled, pack the revprops in an equivalent way */ 10223 if (revsprops_dir) 10224 { 10225 revprops_pack_file_dir = svn_dirent_join(revsprops_dir, 10226 apr_psprintf(pool, 10227 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, 10228 shard), 10229 pool); 10230 revprops_shard_path = svn_dirent_join(revsprops_dir, 10231 apr_psprintf(pool, "%" APR_INT64_T_FMT, shard), 10232 pool); 10233 10234 SVN_ERR(pack_revprops_shard(revprops_pack_file_dir, revprops_shard_path, 10235 shard, max_files_per_dir, 10236 (int)(0.9 * max_pack_size), 10237 compression_level, 10238 cancel_func, cancel_baton, pool)); 10239 } 10240 10241 /* Update the min-unpacked-rev file to reflect our newly packed shard. 10242 * (This doesn't update ffd->min_unpacked_rev. That will be updated by 10243 * update_min_unpacked_rev() when necessary.) */ 10244 SVN_ERR(write_revnum_file(fs_path, PATH_MIN_UNPACKED_REV, 10245 (svn_revnum_t)((shard + 1) * max_files_per_dir), 10246 pool)); 10247 10248 /* Finally, remove the existing shard directories. */ 10249 SVN_ERR(svn_io_remove_dir2(rev_shard_path, TRUE, 10250 cancel_func, cancel_baton, pool)); 10251 if (revsprops_dir) 10252 SVN_ERR(delete_revprops_shard(revprops_shard_path, 10253 shard, max_files_per_dir, 10254 cancel_func, cancel_baton, pool)); 10255 10256 /* Notify caller we're starting to pack this shard. */ 10257 if (notify_func) 10258 SVN_ERR(notify_func(notify_baton, shard, svn_fs_pack_notify_end, 10259 pool)); 10260 10261 return SVN_NO_ERROR; 10262} 10263 10264struct pack_baton 10265{ 10266 svn_fs_t *fs; 10267 svn_fs_pack_notify_t notify_func; 10268 void *notify_baton; 10269 svn_cancel_func_t cancel_func; 10270 void *cancel_baton; 10271}; 10272 10273 10274/* The work-horse for svn_fs_fs__pack, called with the FS write lock. 10275 This implements the svn_fs_fs__with_write_lock() 'body' callback 10276 type. BATON is a 'struct pack_baton *'. 10277 10278 WARNING: if you add a call to this function, please note: 10279 The code currently assumes that any piece of code running with 10280 the write-lock set can rely on the ffd->min_unpacked_rev and 10281 ffd->min_unpacked_revprop caches to be up-to-date (and, by 10282 extension, on not having to use a retry when calling 10283 svn_fs_fs__path_rev_absolute() and friends). If you add a call 10284 to this function, consider whether you have to call 10285 update_min_unpacked_rev(). 10286 See this thread: http://thread.gmane.org/1291206765.3782.3309.camel@edith 10287 */ 10288static svn_error_t * 10289pack_body(void *baton, 10290 apr_pool_t *pool) 10291{ 10292 struct pack_baton *pb = baton; 10293 fs_fs_data_t ffd = {0}; 10294 apr_int64_t completed_shards; 10295 apr_int64_t i; 10296 svn_revnum_t youngest; 10297 apr_pool_t *iterpool; 10298 const char *rev_data_path; 10299 const char *revprops_data_path = NULL; 10300 10301 /* read repository settings */ 10302 SVN_ERR(read_format(&ffd.format, &ffd.max_files_per_dir, 10303 path_format(pb->fs, pool), pool)); 10304 SVN_ERR(check_format(ffd.format)); 10305 SVN_ERR(read_config(&ffd, pb->fs->path, pool)); 10306 10307 /* If the repository isn't a new enough format, we don't support packing. 10308 Return a friendly error to that effect. */ 10309 if (ffd.format < SVN_FS_FS__MIN_PACKED_FORMAT) 10310 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 10311 _("FSFS format (%d) too old to pack; please upgrade the filesystem."), 10312 ffd.format); 10313 10314 /* If we aren't using sharding, we can't do any packing, so quit. */ 10315 if (!ffd.max_files_per_dir) 10316 return SVN_NO_ERROR; 10317 10318 SVN_ERR(read_min_unpacked_rev(&ffd.min_unpacked_rev, 10319 path_min_unpacked_rev(pb->fs, pool), 10320 pool)); 10321 10322 SVN_ERR(get_youngest(&youngest, pb->fs->path, pool)); 10323 completed_shards = (youngest + 1) / ffd.max_files_per_dir; 10324 10325 /* See if we've already completed all possible shards thus far. */ 10326 if (ffd.min_unpacked_rev == (completed_shards * ffd.max_files_per_dir)) 10327 return SVN_NO_ERROR; 10328 10329 rev_data_path = svn_dirent_join(pb->fs->path, PATH_REVS_DIR, pool); 10330 if (ffd.format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 10331 revprops_data_path = svn_dirent_join(pb->fs->path, PATH_REVPROPS_DIR, 10332 pool); 10333 10334 iterpool = svn_pool_create(pool); 10335 for (i = ffd.min_unpacked_rev / ffd.max_files_per_dir; 10336 i < completed_shards; 10337 i++) 10338 { 10339 svn_pool_clear(iterpool); 10340 10341 if (pb->cancel_func) 10342 SVN_ERR(pb->cancel_func(pb->cancel_baton)); 10343 10344 SVN_ERR(pack_shard(rev_data_path, revprops_data_path, 10345 pb->fs->path, i, ffd.max_files_per_dir, 10346 ffd.revprop_pack_size, 10347 ffd.compress_packed_revprops 10348 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 10349 : SVN_DELTA_COMPRESSION_LEVEL_NONE, 10350 pb->notify_func, pb->notify_baton, 10351 pb->cancel_func, pb->cancel_baton, iterpool)); 10352 } 10353 10354 svn_pool_destroy(iterpool); 10355 return SVN_NO_ERROR; 10356} 10357 10358svn_error_t * 10359svn_fs_fs__pack(svn_fs_t *fs, 10360 svn_fs_pack_notify_t notify_func, 10361 void *notify_baton, 10362 svn_cancel_func_t cancel_func, 10363 void *cancel_baton, 10364 apr_pool_t *pool) 10365{ 10366 struct pack_baton pb = { 0 }; 10367 pb.fs = fs; 10368 pb.notify_func = notify_func; 10369 pb.notify_baton = notify_baton; 10370 pb.cancel_func = cancel_func; 10371 pb.cancel_baton = cancel_baton; 10372 return svn_fs_fs__with_write_lock(fs, pack_body, &pb, pool); 10373} 10374 10375 10376/** Verifying. **/ 10377 10378/* Baton type expected by verify_walker(). The purpose is to reuse open 10379 * rev / pack file handles between calls. Its contents need to be cleaned 10380 * periodically to limit resource usage. 10381 */ 10382typedef struct verify_walker_baton_t 10383{ 10384 /* number of calls to verify_walker() since the last clean */ 10385 int iteration_count; 10386 10387 /* number of files opened since the last clean */ 10388 int file_count; 10389 10390 /* progress notification callback to invoke periodically (may be NULL) */ 10391 svn_fs_progress_notify_func_t notify_func; 10392 10393 /* baton to use with NOTIFY_FUNC */ 10394 void *notify_baton; 10395 10396 /* remember the last revision for which we called notify_func */ 10397 svn_revnum_t last_notified_revision; 10398 10399 /* current file handle (or NULL) */ 10400 apr_file_t *file_hint; 10401 10402 /* corresponding revision (or SVN_INVALID_REVNUM) */ 10403 svn_revnum_t rev_hint; 10404 10405 /* pool to use for the file handles etc. */ 10406 apr_pool_t *pool; 10407} verify_walker_baton_t; 10408 10409/* Used by svn_fs_fs__verify(). 10410 Implements svn_fs_fs__walk_rep_reference().walker. */ 10411static svn_error_t * 10412verify_walker(representation_t *rep, 10413 void *baton, 10414 svn_fs_t *fs, 10415 apr_pool_t *scratch_pool) 10416{ 10417 struct rep_state *rs; 10418 struct rep_args *rep_args; 10419 10420 if (baton) 10421 { 10422 verify_walker_baton_t *walker_baton = baton; 10423 apr_file_t * previous_file; 10424 10425 /* notify and free resources periodically */ 10426 if ( walker_baton->iteration_count > 1000 10427 || walker_baton->file_count > 16) 10428 { 10429 if ( walker_baton->notify_func 10430 && rep->revision != walker_baton->last_notified_revision) 10431 { 10432 walker_baton->notify_func(rep->revision, 10433 walker_baton->notify_baton, 10434 scratch_pool); 10435 walker_baton->last_notified_revision = rep->revision; 10436 } 10437 10438 svn_pool_clear(walker_baton->pool); 10439 10440 walker_baton->iteration_count = 0; 10441 walker_baton->file_count = 0; 10442 walker_baton->file_hint = NULL; 10443 walker_baton->rev_hint = SVN_INVALID_REVNUM; 10444 } 10445 10446 /* access the repo data */ 10447 previous_file = walker_baton->file_hint; 10448 SVN_ERR(create_rep_state(&rs, &rep_args, &walker_baton->file_hint, 10449 &walker_baton->rev_hint, rep, fs, 10450 walker_baton->pool)); 10451 10452 /* update resource usage counters */ 10453 walker_baton->iteration_count++; 10454 if (previous_file != walker_baton->file_hint) 10455 walker_baton->file_count++; 10456 } 10457 else 10458 { 10459 /* ### Should this be using read_rep_line() directly? */ 10460 SVN_ERR(create_rep_state(&rs, &rep_args, NULL, NULL, rep, fs, 10461 scratch_pool)); 10462 } 10463 10464 return SVN_NO_ERROR; 10465} 10466 10467svn_error_t * 10468svn_fs_fs__verify(svn_fs_t *fs, 10469 svn_revnum_t start, 10470 svn_revnum_t end, 10471 svn_fs_progress_notify_func_t notify_func, 10472 void *notify_baton, 10473 svn_cancel_func_t cancel_func, 10474 void *cancel_baton, 10475 apr_pool_t *pool) 10476{ 10477 fs_fs_data_t *ffd = fs->fsap_data; 10478 svn_boolean_t exists; 10479 svn_revnum_t youngest = ffd->youngest_rev_cache; /* cache is current */ 10480 10481 if (ffd->format < SVN_FS_FS__MIN_REP_SHARING_FORMAT) 10482 return SVN_NO_ERROR; 10483 10484 /* Input validation. */ 10485 if (! SVN_IS_VALID_REVNUM(start)) 10486 start = 0; 10487 if (! SVN_IS_VALID_REVNUM(end)) 10488 end = youngest; 10489 SVN_ERR(ensure_revision_exists(fs, start, pool)); 10490 SVN_ERR(ensure_revision_exists(fs, end, pool)); 10491 10492 /* rep-cache verification. */ 10493 SVN_ERR(svn_fs_fs__exists_rep_cache(&exists, fs, pool)); 10494 if (exists) 10495 { 10496 /* provide a baton to allow the reuse of open file handles between 10497 iterations (saves 2/3 of OS level file operations). */ 10498 verify_walker_baton_t *baton = apr_pcalloc(pool, sizeof(*baton)); 10499 baton->rev_hint = SVN_INVALID_REVNUM; 10500 baton->pool = svn_pool_create(pool); 10501 baton->last_notified_revision = SVN_INVALID_REVNUM; 10502 baton->notify_func = notify_func; 10503 baton->notify_baton = notify_baton; 10504 10505 /* tell the user that we are now ready to do *something* */ 10506 if (notify_func) 10507 notify_func(SVN_INVALID_REVNUM, notify_baton, baton->pool); 10508 10509 /* Do not attempt to walk the rep-cache database if its file does 10510 not exist, since doing so would create it --- which may confuse 10511 the administrator. Don't take any lock. */ 10512 SVN_ERR(svn_fs_fs__walk_rep_reference(fs, start, end, 10513 verify_walker, baton, 10514 cancel_func, cancel_baton, 10515 pool)); 10516 10517 /* walker resource cleanup */ 10518 svn_pool_destroy(baton->pool); 10519 } 10520 10521 return SVN_NO_ERROR; 10522} 10523 10524 10525/** Hotcopy. **/ 10526 10527/* Like svn_io_dir_file_copy(), but doesn't copy files that exist at 10528 * the destination and do not differ in terms of kind, size, and mtime. */ 10529static svn_error_t * 10530hotcopy_io_dir_file_copy(const char *src_path, 10531 const char *dst_path, 10532 const char *file, 10533 apr_pool_t *scratch_pool) 10534{ 10535 const svn_io_dirent2_t *src_dirent; 10536 const svn_io_dirent2_t *dst_dirent; 10537 const char *src_target; 10538 const char *dst_target; 10539 10540 /* Does the destination already exist? If not, we must copy it. */ 10541 dst_target = svn_dirent_join(dst_path, file, scratch_pool); 10542 SVN_ERR(svn_io_stat_dirent2(&dst_dirent, dst_target, FALSE, TRUE, 10543 scratch_pool, scratch_pool)); 10544 if (dst_dirent->kind != svn_node_none) 10545 { 10546 /* If the destination's stat information indicates that the file 10547 * is equal to the source, don't bother copying the file again. */ 10548 src_target = svn_dirent_join(src_path, file, scratch_pool); 10549 SVN_ERR(svn_io_stat_dirent2(&src_dirent, src_target, FALSE, FALSE, 10550 scratch_pool, scratch_pool)); 10551 if (src_dirent->kind == dst_dirent->kind && 10552 src_dirent->special == dst_dirent->special && 10553 src_dirent->filesize == dst_dirent->filesize && 10554 src_dirent->mtime <= dst_dirent->mtime) 10555 return SVN_NO_ERROR; 10556 } 10557 10558 return svn_error_trace(svn_io_dir_file_copy(src_path, dst_path, file, 10559 scratch_pool)); 10560} 10561 10562/* Set *NAME_P to the UTF-8 representation of directory entry NAME. 10563 * NAME is in the internal encoding used by APR; PARENT is in 10564 * UTF-8 and in internal (not local) style. 10565 * 10566 * Use PARENT only for generating an error string if the conversion 10567 * fails because NAME could not be represented in UTF-8. In that 10568 * case, return a two-level error in which the outer error's message 10569 * mentions PARENT, but the inner error's message does not mention 10570 * NAME (except possibly in hex) since NAME may not be printable. 10571 * Such a compound error at least allows the user to go looking in the 10572 * right directory for the problem. 10573 * 10574 * If there is any other error, just return that error directly. 10575 * 10576 * If there is any error, the effect on *NAME_P is undefined. 10577 * 10578 * *NAME_P and NAME may refer to the same storage. 10579 */ 10580static svn_error_t * 10581entry_name_to_utf8(const char **name_p, 10582 const char *name, 10583 const char *parent, 10584 apr_pool_t *pool) 10585{ 10586 svn_error_t *err = svn_path_cstring_to_utf8(name_p, name, pool); 10587 if (err && err->apr_err == APR_EINVAL) 10588 { 10589 return svn_error_createf(err->apr_err, err, 10590 _("Error converting entry " 10591 "in directory '%s' to UTF-8"), 10592 svn_dirent_local_style(parent, pool)); 10593 } 10594 return err; 10595} 10596 10597/* Like svn_io_copy_dir_recursively() but doesn't copy regular files that 10598 * exist in the destination and do not differ from the source in terms of 10599 * kind, size, and mtime. */ 10600static svn_error_t * 10601hotcopy_io_copy_dir_recursively(const char *src, 10602 const char *dst_parent, 10603 const char *dst_basename, 10604 svn_boolean_t copy_perms, 10605 svn_cancel_func_t cancel_func, 10606 void *cancel_baton, 10607 apr_pool_t *pool) 10608{ 10609 svn_node_kind_t kind; 10610 apr_status_t status; 10611 const char *dst_path; 10612 apr_dir_t *this_dir; 10613 apr_finfo_t this_entry; 10614 apr_int32_t flags = APR_FINFO_TYPE | APR_FINFO_NAME; 10615 10616 /* Make a subpool for recursion */ 10617 apr_pool_t *subpool = svn_pool_create(pool); 10618 10619 /* The 'dst_path' is simply dst_parent/dst_basename */ 10620 dst_path = svn_dirent_join(dst_parent, dst_basename, pool); 10621 10622 /* Sanity checks: SRC and DST_PARENT are directories, and 10623 DST_BASENAME doesn't already exist in DST_PARENT. */ 10624 SVN_ERR(svn_io_check_path(src, &kind, subpool)); 10625 if (kind != svn_node_dir) 10626 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, 10627 _("Source '%s' is not a directory"), 10628 svn_dirent_local_style(src, pool)); 10629 10630 SVN_ERR(svn_io_check_path(dst_parent, &kind, subpool)); 10631 if (kind != svn_node_dir) 10632 return svn_error_createf(SVN_ERR_NODE_UNEXPECTED_KIND, NULL, 10633 _("Destination '%s' is not a directory"), 10634 svn_dirent_local_style(dst_parent, pool)); 10635 10636 SVN_ERR(svn_io_check_path(dst_path, &kind, subpool)); 10637 10638 /* Create the new directory. */ 10639 /* ### TODO: copy permissions (needs apr_file_attrs_get()) */ 10640 SVN_ERR(svn_io_make_dir_recursively(dst_path, pool)); 10641 10642 /* Loop over the dirents in SRC. ('.' and '..' are auto-excluded) */ 10643 SVN_ERR(svn_io_dir_open(&this_dir, src, subpool)); 10644 10645 for (status = apr_dir_read(&this_entry, flags, this_dir); 10646 status == APR_SUCCESS; 10647 status = apr_dir_read(&this_entry, flags, this_dir)) 10648 { 10649 if ((this_entry.name[0] == '.') 10650 && ((this_entry.name[1] == '\0') 10651 || ((this_entry.name[1] == '.') 10652 && (this_entry.name[2] == '\0')))) 10653 { 10654 continue; 10655 } 10656 else 10657 { 10658 const char *entryname_utf8; 10659 10660 if (cancel_func) 10661 SVN_ERR(cancel_func(cancel_baton)); 10662 10663 SVN_ERR(entry_name_to_utf8(&entryname_utf8, this_entry.name, 10664 src, subpool)); 10665 if (this_entry.filetype == APR_REG) /* regular file */ 10666 { 10667 SVN_ERR(hotcopy_io_dir_file_copy(src, dst_path, entryname_utf8, 10668 subpool)); 10669 } 10670 else if (this_entry.filetype == APR_LNK) /* symlink */ 10671 { 10672 const char *src_target = svn_dirent_join(src, entryname_utf8, 10673 subpool); 10674 const char *dst_target = svn_dirent_join(dst_path, 10675 entryname_utf8, 10676 subpool); 10677 SVN_ERR(svn_io_copy_link(src_target, dst_target, 10678 subpool)); 10679 } 10680 else if (this_entry.filetype == APR_DIR) /* recurse */ 10681 { 10682 const char *src_target; 10683 10684 /* Prevent infinite recursion by filtering off our 10685 newly created destination path. */ 10686 if (strcmp(src, dst_parent) == 0 10687 && strcmp(entryname_utf8, dst_basename) == 0) 10688 continue; 10689 10690 src_target = svn_dirent_join(src, entryname_utf8, subpool); 10691 SVN_ERR(hotcopy_io_copy_dir_recursively(src_target, 10692 dst_path, 10693 entryname_utf8, 10694 copy_perms, 10695 cancel_func, 10696 cancel_baton, 10697 subpool)); 10698 } 10699 /* ### support other APR node types someday?? */ 10700 10701 } 10702 } 10703 10704 if (! (APR_STATUS_IS_ENOENT(status))) 10705 return svn_error_wrap_apr(status, _("Can't read directory '%s'"), 10706 svn_dirent_local_style(src, pool)); 10707 10708 status = apr_dir_close(this_dir); 10709 if (status) 10710 return svn_error_wrap_apr(status, _("Error closing directory '%s'"), 10711 svn_dirent_local_style(src, pool)); 10712 10713 /* Free any memory used by recursion */ 10714 svn_pool_destroy(subpool); 10715 10716 return SVN_NO_ERROR; 10717} 10718 10719/* Copy an un-packed revision or revprop file for revision REV from SRC_SUBDIR 10720 * to DST_SUBDIR. Assume a sharding layout based on MAX_FILES_PER_DIR. 10721 * Use SCRATCH_POOL for temporary allocations. */ 10722static svn_error_t * 10723hotcopy_copy_shard_file(const char *src_subdir, 10724 const char *dst_subdir, 10725 svn_revnum_t rev, 10726 int max_files_per_dir, 10727 apr_pool_t *scratch_pool) 10728{ 10729 const char *src_subdir_shard = src_subdir, 10730 *dst_subdir_shard = dst_subdir; 10731 10732 if (max_files_per_dir) 10733 { 10734 const char *shard = apr_psprintf(scratch_pool, "%ld", 10735 rev / max_files_per_dir); 10736 src_subdir_shard = svn_dirent_join(src_subdir, shard, scratch_pool); 10737 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); 10738 10739 if (rev % max_files_per_dir == 0) 10740 { 10741 SVN_ERR(svn_io_make_dir_recursively(dst_subdir_shard, scratch_pool)); 10742 SVN_ERR(svn_io_copy_perms(dst_subdir, dst_subdir_shard, 10743 scratch_pool)); 10744 } 10745 } 10746 10747 SVN_ERR(hotcopy_io_dir_file_copy(src_subdir_shard, dst_subdir_shard, 10748 apr_psprintf(scratch_pool, "%ld", rev), 10749 scratch_pool)); 10750 return SVN_NO_ERROR; 10751} 10752 10753 10754/* Copy a packed shard containing revision REV, and which contains 10755 * MAX_FILES_PER_DIR revisions, from SRC_FS to DST_FS. 10756 * Update *DST_MIN_UNPACKED_REV in case the shard is new in DST_FS. 10757 * Do not re-copy data which already exists in DST_FS. 10758 * Use SCRATCH_POOL for temporary allocations. */ 10759static svn_error_t * 10760hotcopy_copy_packed_shard(svn_revnum_t *dst_min_unpacked_rev, 10761 svn_fs_t *src_fs, 10762 svn_fs_t *dst_fs, 10763 svn_revnum_t rev, 10764 int max_files_per_dir, 10765 apr_pool_t *scratch_pool) 10766{ 10767 const char *src_subdir; 10768 const char *dst_subdir; 10769 const char *packed_shard; 10770 const char *src_subdir_packed_shard; 10771 svn_revnum_t revprop_rev; 10772 apr_pool_t *iterpool; 10773 fs_fs_data_t *src_ffd = src_fs->fsap_data; 10774 10775 /* Copy the packed shard. */ 10776 src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, scratch_pool); 10777 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); 10778 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, 10779 rev / max_files_per_dir); 10780 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, 10781 scratch_pool); 10782 SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, 10783 dst_subdir, packed_shard, 10784 TRUE /* copy_perms */, 10785 NULL /* cancel_func */, NULL, 10786 scratch_pool)); 10787 10788 /* Copy revprops belonging to revisions in this pack. */ 10789 src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, scratch_pool); 10790 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, scratch_pool); 10791 10792 if ( src_ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT 10793 || src_ffd->min_unpacked_rev < rev + max_files_per_dir) 10794 { 10795 /* copy unpacked revprops rev by rev */ 10796 iterpool = svn_pool_create(scratch_pool); 10797 for (revprop_rev = rev; 10798 revprop_rev < rev + max_files_per_dir; 10799 revprop_rev++) 10800 { 10801 svn_pool_clear(iterpool); 10802 10803 SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, 10804 revprop_rev, max_files_per_dir, 10805 iterpool)); 10806 } 10807 svn_pool_destroy(iterpool); 10808 } 10809 else 10810 { 10811 /* revprop for revision 0 will never be packed */ 10812 if (rev == 0) 10813 SVN_ERR(hotcopy_copy_shard_file(src_subdir, dst_subdir, 10814 0, max_files_per_dir, 10815 scratch_pool)); 10816 10817 /* packed revprops folder */ 10818 packed_shard = apr_psprintf(scratch_pool, "%ld" PATH_EXT_PACKED_SHARD, 10819 rev / max_files_per_dir); 10820 src_subdir_packed_shard = svn_dirent_join(src_subdir, packed_shard, 10821 scratch_pool); 10822 SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir_packed_shard, 10823 dst_subdir, packed_shard, 10824 TRUE /* copy_perms */, 10825 NULL /* cancel_func */, NULL, 10826 scratch_pool)); 10827 } 10828 10829 /* If necessary, update the min-unpacked rev file in the hotcopy. */ 10830 if (*dst_min_unpacked_rev < rev + max_files_per_dir) 10831 { 10832 *dst_min_unpacked_rev = rev + max_files_per_dir; 10833 SVN_ERR(write_revnum_file(dst_fs->path, PATH_MIN_UNPACKED_REV, 10834 *dst_min_unpacked_rev, 10835 scratch_pool)); 10836 } 10837 10838 return SVN_NO_ERROR; 10839} 10840 10841/* If NEW_YOUNGEST is younger than *DST_YOUNGEST, update the 'current' 10842 * file in DST_FS and set *DST_YOUNGEST to NEW_YOUNGEST. 10843 * Use SCRATCH_POOL for temporary allocations. */ 10844static svn_error_t * 10845hotcopy_update_current(svn_revnum_t *dst_youngest, 10846 svn_fs_t *dst_fs, 10847 svn_revnum_t new_youngest, 10848 apr_pool_t *scratch_pool) 10849{ 10850 char next_node_id[MAX_KEY_SIZE] = "0"; 10851 char next_copy_id[MAX_KEY_SIZE] = "0"; 10852 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 10853 10854 if (*dst_youngest >= new_youngest) 10855 return SVN_NO_ERROR; 10856 10857 /* If necessary, get new current next_node and next_copy IDs. */ 10858 if (dst_ffd->format < SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT) 10859 { 10860 apr_off_t root_offset; 10861 apr_file_t *rev_file; 10862 10863 if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 10864 SVN_ERR(update_min_unpacked_rev(dst_fs, scratch_pool)); 10865 10866 SVN_ERR(open_pack_or_rev_file(&rev_file, dst_fs, new_youngest, 10867 scratch_pool)); 10868 SVN_ERR(get_root_changes_offset(&root_offset, NULL, rev_file, 10869 dst_fs, new_youngest, scratch_pool)); 10870 SVN_ERR(recover_find_max_ids(dst_fs, new_youngest, rev_file, 10871 root_offset, next_node_id, next_copy_id, 10872 scratch_pool)); 10873 SVN_ERR(svn_io_file_close(rev_file, scratch_pool)); 10874 } 10875 10876 /* Update 'current'. */ 10877 SVN_ERR(write_current(dst_fs, new_youngest, next_node_id, next_copy_id, 10878 scratch_pool)); 10879 10880 *dst_youngest = new_youngest; 10881 10882 return SVN_NO_ERROR; 10883} 10884 10885 10886/* Remove revisions between START_REV (inclusive) and END_REV (non-inclusive) 10887 * from DST_FS. Assume sharding as per MAX_FILES_PER_DIR. 10888 * Use SCRATCH_POOL for temporary allocations. */ 10889static svn_error_t * 10890hotcopy_remove_rev_files(svn_fs_t *dst_fs, 10891 svn_revnum_t start_rev, 10892 svn_revnum_t end_rev, 10893 int max_files_per_dir, 10894 apr_pool_t *scratch_pool) 10895{ 10896 const char *dst_subdir; 10897 const char *shard; 10898 const char *dst_subdir_shard; 10899 svn_revnum_t rev; 10900 apr_pool_t *iterpool; 10901 10902 SVN_ERR_ASSERT(start_rev <= end_rev); 10903 10904 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, scratch_pool); 10905 10906 /* Pre-compute paths for initial shard. */ 10907 shard = apr_psprintf(scratch_pool, "%ld", start_rev / max_files_per_dir); 10908 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); 10909 10910 iterpool = svn_pool_create(scratch_pool); 10911 for (rev = start_rev; rev < end_rev; rev++) 10912 { 10913 const char *rev_path; 10914 10915 svn_pool_clear(iterpool); 10916 10917 /* If necessary, update paths for shard. */ 10918 if (rev != start_rev && rev % max_files_per_dir == 0) 10919 { 10920 shard = apr_psprintf(iterpool, "%ld", rev / max_files_per_dir); 10921 dst_subdir_shard = svn_dirent_join(dst_subdir, shard, scratch_pool); 10922 } 10923 10924 rev_path = svn_dirent_join(dst_subdir_shard, 10925 apr_psprintf(iterpool, "%ld", rev), 10926 iterpool); 10927 10928 /* Make the rev file writable and remove it. */ 10929 SVN_ERR(svn_io_set_file_read_write(rev_path, TRUE, iterpool)); 10930 SVN_ERR(svn_io_remove_file2(rev_path, TRUE, iterpool)); 10931 } 10932 svn_pool_destroy(iterpool); 10933 10934 return SVN_NO_ERROR; 10935} 10936 10937/* Verify that DST_FS is a suitable destination for an incremental 10938 * hotcopy from SRC_FS. */ 10939static svn_error_t * 10940hotcopy_incremental_check_preconditions(svn_fs_t *src_fs, 10941 svn_fs_t *dst_fs, 10942 apr_pool_t *pool) 10943{ 10944 fs_fs_data_t *src_ffd = src_fs->fsap_data; 10945 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 10946 10947 /* We only support incremental hotcopy between the same format. */ 10948 if (src_ffd->format != dst_ffd->format) 10949 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 10950 _("The FSFS format (%d) of the hotcopy source does not match the " 10951 "FSFS format (%d) of the hotcopy destination; please upgrade " 10952 "both repositories to the same format"), 10953 src_ffd->format, dst_ffd->format); 10954 10955 /* Make sure the UUID of source and destination match up. 10956 * We don't want to copy over a different repository. */ 10957 if (strcmp(src_fs->uuid, dst_fs->uuid) != 0) 10958 return svn_error_create(SVN_ERR_RA_UUID_MISMATCH, NULL, 10959 _("The UUID of the hotcopy source does " 10960 "not match the UUID of the hotcopy " 10961 "destination")); 10962 10963 /* Also require same shard size. */ 10964 if (src_ffd->max_files_per_dir != dst_ffd->max_files_per_dir) 10965 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 10966 _("The sharding layout configuration " 10967 "of the hotcopy source does not match " 10968 "the sharding layout configuration of " 10969 "the hotcopy destination")); 10970 return SVN_NO_ERROR; 10971} 10972 10973 10974/* Baton for hotcopy_body(). */ 10975struct hotcopy_body_baton { 10976 svn_fs_t *src_fs; 10977 svn_fs_t *dst_fs; 10978 svn_boolean_t incremental; 10979 svn_cancel_func_t cancel_func; 10980 void *cancel_baton; 10981} hotcopy_body_baton; 10982 10983/* Perform a hotcopy, either normal or incremental. 10984 * 10985 * Normal hotcopy assumes that the destination exists as an empty 10986 * directory. It behaves like an incremental hotcopy except that 10987 * none of the copied files already exist in the destination. 10988 * 10989 * An incremental hotcopy copies only changed or new files to the destination, 10990 * and removes files from the destination no longer present in the source. 10991 * While the incremental hotcopy is running, readers should still be able 10992 * to access the destintation repository without error and should not see 10993 * revisions currently in progress of being copied. Readers are able to see 10994 * new fully copied revisions even if the entire incremental hotcopy procedure 10995 * has not yet completed. 10996 * 10997 * Writers are blocked out completely during the entire incremental hotcopy 10998 * process to ensure consistency. This function assumes that the repository 10999 * write-lock is held. 11000 */ 11001static svn_error_t * 11002hotcopy_body(void *baton, apr_pool_t *pool) 11003{ 11004 struct hotcopy_body_baton *hbb = baton; 11005 svn_fs_t *src_fs = hbb->src_fs; 11006 fs_fs_data_t *src_ffd = src_fs->fsap_data; 11007 svn_fs_t *dst_fs = hbb->dst_fs; 11008 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 11009 int max_files_per_dir = src_ffd->max_files_per_dir; 11010 svn_boolean_t incremental = hbb->incremental; 11011 svn_cancel_func_t cancel_func = hbb->cancel_func; 11012 void* cancel_baton = hbb->cancel_baton; 11013 svn_revnum_t src_youngest; 11014 svn_revnum_t dst_youngest; 11015 svn_revnum_t rev; 11016 svn_revnum_t src_min_unpacked_rev; 11017 svn_revnum_t dst_min_unpacked_rev; 11018 const char *src_subdir; 11019 const char *dst_subdir; 11020 const char *revprop_src_subdir; 11021 const char *revprop_dst_subdir; 11022 apr_pool_t *iterpool; 11023 svn_node_kind_t kind; 11024 11025 /* Try to copy the config. 11026 * 11027 * ### We try copying the config file before doing anything else, 11028 * ### because higher layers will abort the hotcopy if we throw 11029 * ### an error from this function, and that renders the hotcopy 11030 * ### unusable anyway. */ 11031 if (src_ffd->format >= SVN_FS_FS__MIN_CONFIG_FILE) 11032 { 11033 svn_error_t *err; 11034 11035 err = svn_io_dir_file_copy(src_fs->path, dst_fs->path, PATH_CONFIG, 11036 pool); 11037 if (err) 11038 { 11039 if (APR_STATUS_IS_ENOENT(err->apr_err)) 11040 { 11041 /* 1.6.0 to 1.6.11 did not copy the configuration file during 11042 * hotcopy. So if we're hotcopying a repository which has been 11043 * created as a hotcopy itself, it's possible that fsfs.conf 11044 * does not exist. Ask the user to re-create it. 11045 * 11046 * ### It would be nice to make this a non-fatal error, 11047 * ### but this function does not get an svn_fs_t object 11048 * ### so we have no way of just printing a warning via 11049 * ### the fs->warning() callback. */ 11050 11051 const char *msg; 11052 const char *src_abspath; 11053 const char *dst_abspath; 11054 const char *config_relpath; 11055 svn_error_t *err2; 11056 11057 config_relpath = svn_dirent_join(src_fs->path, PATH_CONFIG, pool); 11058 err2 = svn_dirent_get_absolute(&src_abspath, src_fs->path, pool); 11059 if (err2) 11060 return svn_error_trace(svn_error_compose_create(err, err2)); 11061 err2 = svn_dirent_get_absolute(&dst_abspath, dst_fs->path, pool); 11062 if (err2) 11063 return svn_error_trace(svn_error_compose_create(err, err2)); 11064 11065 /* ### hack: strip off the 'db/' directory from paths so 11066 * ### they make sense to the user */ 11067 src_abspath = svn_dirent_dirname(src_abspath, pool); 11068 dst_abspath = svn_dirent_dirname(dst_abspath, pool); 11069 11070 msg = apr_psprintf(pool, 11071 _("Failed to create hotcopy at '%s'. " 11072 "The file '%s' is missing from the source " 11073 "repository. Please create this file, for " 11074 "instance by running 'svnadmin upgrade %s'"), 11075 dst_abspath, config_relpath, src_abspath); 11076 return svn_error_quick_wrap(err, msg); 11077 } 11078 else 11079 return svn_error_trace(err); 11080 } 11081 } 11082 11083 if (cancel_func) 11084 SVN_ERR(cancel_func(cancel_baton)); 11085 11086 /* Find the youngest revision in the source and destination. 11087 * We only support hotcopies from sources with an equal or greater amount 11088 * of revisions than the destination. 11089 * This also catches the case where users accidentally swap the 11090 * source and destination arguments. */ 11091 SVN_ERR(get_youngest(&src_youngest, src_fs->path, pool)); 11092 if (incremental) 11093 { 11094 SVN_ERR(get_youngest(&dst_youngest, dst_fs->path, pool)); 11095 if (src_youngest < dst_youngest) 11096 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 11097 _("The hotcopy destination already contains more revisions " 11098 "(%lu) than the hotcopy source contains (%lu); are source " 11099 "and destination swapped?"), 11100 dst_youngest, src_youngest); 11101 } 11102 else 11103 dst_youngest = 0; 11104 11105 if (cancel_func) 11106 SVN_ERR(cancel_func(cancel_baton)); 11107 11108 /* Copy the min unpacked rev, and read its value. */ 11109 if (src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 11110 { 11111 const char *min_unpacked_rev_path; 11112 11113 min_unpacked_rev_path = svn_dirent_join(src_fs->path, 11114 PATH_MIN_UNPACKED_REV, 11115 pool); 11116 SVN_ERR(read_min_unpacked_rev(&src_min_unpacked_rev, 11117 min_unpacked_rev_path, 11118 pool)); 11119 11120 min_unpacked_rev_path = svn_dirent_join(dst_fs->path, 11121 PATH_MIN_UNPACKED_REV, 11122 pool); 11123 SVN_ERR(read_min_unpacked_rev(&dst_min_unpacked_rev, 11124 min_unpacked_rev_path, 11125 pool)); 11126 11127 /* We only support packs coming from the hotcopy source. 11128 * The destination should not be packed independently from 11129 * the source. This also catches the case where users accidentally 11130 * swap the source and destination arguments. */ 11131 if (src_min_unpacked_rev < dst_min_unpacked_rev) 11132 return svn_error_createf(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 11133 _("The hotcopy destination already contains " 11134 "more packed revisions (%lu) than the " 11135 "hotcopy source contains (%lu)"), 11136 dst_min_unpacked_rev - 1, 11137 src_min_unpacked_rev - 1); 11138 11139 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, 11140 PATH_MIN_UNPACKED_REV, pool)); 11141 } 11142 else 11143 { 11144 src_min_unpacked_rev = 0; 11145 dst_min_unpacked_rev = 0; 11146 } 11147 11148 if (cancel_func) 11149 SVN_ERR(cancel_func(cancel_baton)); 11150 11151 /* 11152 * Copy the necessary rev files. 11153 */ 11154 11155 src_subdir = svn_dirent_join(src_fs->path, PATH_REVS_DIR, pool); 11156 dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVS_DIR, pool); 11157 SVN_ERR(svn_io_make_dir_recursively(dst_subdir, pool)); 11158 11159 iterpool = svn_pool_create(pool); 11160 /* First, copy packed shards. */ 11161 for (rev = 0; rev < src_min_unpacked_rev; rev += max_files_per_dir) 11162 { 11163 svn_error_t *err; 11164 11165 svn_pool_clear(iterpool); 11166 11167 if (cancel_func) 11168 SVN_ERR(cancel_func(cancel_baton)); 11169 11170 /* Copy the packed shard. */ 11171 SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, 11172 src_fs, dst_fs, 11173 rev, max_files_per_dir, 11174 iterpool)); 11175 11176 /* If necessary, update 'current' to the most recent packed rev, 11177 * so readers can see new revisions which arrived in this pack. */ 11178 SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, 11179 rev + max_files_per_dir - 1, 11180 iterpool)); 11181 11182 /* Remove revision files which are now packed. */ 11183 if (incremental) 11184 SVN_ERR(hotcopy_remove_rev_files(dst_fs, rev, rev + max_files_per_dir, 11185 max_files_per_dir, iterpool)); 11186 11187 /* Now that all revisions have moved into the pack, the original 11188 * rev dir can be removed. */ 11189 err = svn_io_remove_dir2(path_rev_shard(dst_fs, rev, iterpool), 11190 TRUE, cancel_func, cancel_baton, iterpool); 11191 if (err) 11192 { 11193 if (APR_STATUS_IS_ENOTEMPTY(err->apr_err)) 11194 svn_error_clear(err); 11195 else 11196 return svn_error_trace(err); 11197 } 11198 } 11199 11200 if (cancel_func) 11201 SVN_ERR(cancel_func(cancel_baton)); 11202 11203 /* Now, copy pairs of non-packed revisions and revprop files. 11204 * If necessary, update 'current' after copying all files from a shard. */ 11205 SVN_ERR_ASSERT(rev == src_min_unpacked_rev); 11206 SVN_ERR_ASSERT(src_min_unpacked_rev == dst_min_unpacked_rev); 11207 revprop_src_subdir = svn_dirent_join(src_fs->path, PATH_REVPROPS_DIR, pool); 11208 revprop_dst_subdir = svn_dirent_join(dst_fs->path, PATH_REVPROPS_DIR, pool); 11209 SVN_ERR(svn_io_make_dir_recursively(revprop_dst_subdir, pool)); 11210 for (; rev <= src_youngest; rev++) 11211 { 11212 svn_error_t *err; 11213 11214 svn_pool_clear(iterpool); 11215 11216 if (cancel_func) 11217 SVN_ERR(cancel_func(cancel_baton)); 11218 11219 /* Copy the rev file. */ 11220 err = hotcopy_copy_shard_file(src_subdir, dst_subdir, 11221 rev, max_files_per_dir, 11222 iterpool); 11223 if (err) 11224 { 11225 if (APR_STATUS_IS_ENOENT(err->apr_err) && 11226 src_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 11227 { 11228 svn_error_clear(err); 11229 11230 /* The source rev file does not exist. This can happen if the 11231 * source repository is being packed concurrently with this 11232 * hotcopy operation. 11233 * 11234 * If the new revision is now packed, and the youngest revision 11235 * we're interested in is not inside this pack, try to copy the 11236 * pack instead. 11237 * 11238 * If the youngest revision ended up being packed, don't try 11239 * to be smart and work around this. Just abort the hotcopy. */ 11240 SVN_ERR(update_min_unpacked_rev(src_fs, pool)); 11241 if (is_packed_rev(src_fs, rev)) 11242 { 11243 if (is_packed_rev(src_fs, src_youngest)) 11244 return svn_error_createf( 11245 SVN_ERR_FS_NO_SUCH_REVISION, NULL, 11246 _("The assumed HEAD revision (%lu) of the " 11247 "hotcopy source has been packed while the " 11248 "hotcopy was in progress; please restart " 11249 "the hotcopy operation"), 11250 src_youngest); 11251 11252 SVN_ERR(hotcopy_copy_packed_shard(&dst_min_unpacked_rev, 11253 src_fs, dst_fs, 11254 rev, max_files_per_dir, 11255 iterpool)); 11256 rev = dst_min_unpacked_rev; 11257 continue; 11258 } 11259 else 11260 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 11261 _("Revision %lu disappeared from the " 11262 "hotcopy source while hotcopy was " 11263 "in progress"), rev); 11264 } 11265 else 11266 return svn_error_trace(err); 11267 } 11268 11269 /* Copy the revprop file. */ 11270 SVN_ERR(hotcopy_copy_shard_file(revprop_src_subdir, 11271 revprop_dst_subdir, 11272 rev, max_files_per_dir, 11273 iterpool)); 11274 11275 /* After completing a full shard, update 'current'. */ 11276 if (max_files_per_dir && rev % max_files_per_dir == 0) 11277 SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, rev, iterpool)); 11278 } 11279 svn_pool_destroy(iterpool); 11280 11281 if (cancel_func) 11282 SVN_ERR(cancel_func(cancel_baton)); 11283 11284 /* We assume that all revisions were copied now, i.e. we didn't exit the 11285 * above loop early. 'rev' was last incremented during exit of the loop. */ 11286 SVN_ERR_ASSERT(rev == src_youngest + 1); 11287 11288 /* All revisions were copied. Update 'current'. */ 11289 SVN_ERR(hotcopy_update_current(&dst_youngest, dst_fs, src_youngest, pool)); 11290 11291 /* Replace the locks tree. 11292 * This is racy in case readers are currently trying to list locks in 11293 * the destination. However, we need to get rid of stale locks. 11294 * This is the simplest way of doing this, so we accept this small race. */ 11295 dst_subdir = svn_dirent_join(dst_fs->path, PATH_LOCKS_DIR, pool); 11296 SVN_ERR(svn_io_remove_dir2(dst_subdir, TRUE, cancel_func, cancel_baton, 11297 pool)); 11298 src_subdir = svn_dirent_join(src_fs->path, PATH_LOCKS_DIR, pool); 11299 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); 11300 if (kind == svn_node_dir) 11301 SVN_ERR(svn_io_copy_dir_recursively(src_subdir, dst_fs->path, 11302 PATH_LOCKS_DIR, TRUE, 11303 cancel_func, cancel_baton, pool)); 11304 11305 /* Now copy the node-origins cache tree. */ 11306 src_subdir = svn_dirent_join(src_fs->path, PATH_NODE_ORIGINS_DIR, pool); 11307 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); 11308 if (kind == svn_node_dir) 11309 SVN_ERR(hotcopy_io_copy_dir_recursively(src_subdir, dst_fs->path, 11310 PATH_NODE_ORIGINS_DIR, TRUE, 11311 cancel_func, cancel_baton, pool)); 11312 11313 /* 11314 * NB: Data copied below is only read by writers, not readers. 11315 * Writers are still locked out at this point. 11316 */ 11317 11318 if (dst_ffd->format >= SVN_FS_FS__MIN_REP_SHARING_FORMAT) 11319 { 11320 /* Copy the rep cache and then remove entries for revisions 11321 * younger than the destination's youngest revision. */ 11322 src_subdir = svn_dirent_join(src_fs->path, REP_CACHE_DB_NAME, pool); 11323 dst_subdir = svn_dirent_join(dst_fs->path, REP_CACHE_DB_NAME, pool); 11324 SVN_ERR(svn_io_check_path(src_subdir, &kind, pool)); 11325 if (kind == svn_node_file) 11326 { 11327 SVN_ERR(svn_sqlite__hotcopy(src_subdir, dst_subdir, pool)); 11328 SVN_ERR(svn_fs_fs__del_rep_reference(dst_fs, dst_youngest, pool)); 11329 } 11330 } 11331 11332 /* Copy the txn-current file. */ 11333 if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) 11334 SVN_ERR(svn_io_dir_file_copy(src_fs->path, dst_fs->path, 11335 PATH_TXN_CURRENT, pool)); 11336 11337 /* If a revprop generation file exists in the source filesystem, 11338 * reset it to zero (since this is on a different path, it will not 11339 * overlap with data already in cache). Also, clean up stale files 11340 * used for the named atomics implementation. */ 11341 SVN_ERR(svn_io_check_path(path_revprop_generation(src_fs, pool), 11342 &kind, pool)); 11343 if (kind == svn_node_file) 11344 SVN_ERR(write_revprop_generation_file(dst_fs, 0, pool)); 11345 11346 SVN_ERR(cleanup_revprop_namespace(dst_fs)); 11347 11348 /* Hotcopied FS is complete. Stamp it with a format file. */ 11349 SVN_ERR(write_format(svn_dirent_join(dst_fs->path, PATH_FORMAT, pool), 11350 dst_ffd->format, max_files_per_dir, TRUE, pool)); 11351 11352 return SVN_NO_ERROR; 11353} 11354 11355 11356/* Set up shared data between SRC_FS and DST_FS. */ 11357static void 11358hotcopy_setup_shared_fs_data(svn_fs_t *src_fs, svn_fs_t *dst_fs) 11359{ 11360 fs_fs_data_t *src_ffd = src_fs->fsap_data; 11361 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 11362 11363 /* The common pool and mutexes are shared between src and dst filesystems. 11364 * During hotcopy we only grab the mutexes for the destination, so there 11365 * is no risk of dead-lock. We don't write to the src filesystem. Shared 11366 * data for the src_fs has already been initialised in fs_hotcopy(). */ 11367 dst_ffd->shared = src_ffd->shared; 11368} 11369 11370/* Create an empty filesystem at DST_FS at DST_PATH with the same 11371 * configuration as SRC_FS (uuid, format, and other parameters). 11372 * After creation DST_FS has no revisions, not even revision zero. */ 11373static svn_error_t * 11374hotcopy_create_empty_dest(svn_fs_t *src_fs, 11375 svn_fs_t *dst_fs, 11376 const char *dst_path, 11377 apr_pool_t *pool) 11378{ 11379 fs_fs_data_t *src_ffd = src_fs->fsap_data; 11380 fs_fs_data_t *dst_ffd = dst_fs->fsap_data; 11381 11382 dst_fs->path = apr_pstrdup(pool, dst_path); 11383 11384 dst_ffd->max_files_per_dir = src_ffd->max_files_per_dir; 11385 dst_ffd->config = src_ffd->config; 11386 dst_ffd->format = src_ffd->format; 11387 11388 /* Create the revision data directories. */ 11389 if (dst_ffd->max_files_per_dir) 11390 SVN_ERR(svn_io_make_dir_recursively(path_rev_shard(dst_fs, 0, pool), 11391 pool)); 11392 else 11393 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, 11394 PATH_REVS_DIR, pool), 11395 pool)); 11396 11397 /* Create the revprops directory. */ 11398 if (src_ffd->max_files_per_dir) 11399 SVN_ERR(svn_io_make_dir_recursively(path_revprops_shard(dst_fs, 0, pool), 11400 pool)); 11401 else 11402 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, 11403 PATH_REVPROPS_DIR, 11404 pool), 11405 pool)); 11406 11407 /* Create the transaction directory. */ 11408 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, PATH_TXNS_DIR, 11409 pool), 11410 pool)); 11411 11412 /* Create the protorevs directory. */ 11413 if (dst_ffd->format >= SVN_FS_FS__MIN_PROTOREVS_DIR_FORMAT) 11414 SVN_ERR(svn_io_make_dir_recursively(svn_dirent_join(dst_path, 11415 PATH_TXN_PROTOS_DIR, 11416 pool), 11417 pool)); 11418 11419 /* Create the 'current' file. */ 11420 SVN_ERR(svn_io_file_create(svn_fs_fs__path_current(dst_fs, pool), 11421 (dst_ffd->format >= 11422 SVN_FS_FS__MIN_NO_GLOBAL_IDS_FORMAT 11423 ? "0\n" : "0 1 1\n"), 11424 pool)); 11425 11426 /* Create lock file and UUID. */ 11427 SVN_ERR(svn_io_file_create(path_lock(dst_fs, pool), "", pool)); 11428 SVN_ERR(svn_fs_fs__set_uuid(dst_fs, src_fs->uuid, pool)); 11429 11430 /* Create the min unpacked rev file. */ 11431 if (dst_ffd->format >= SVN_FS_FS__MIN_PACKED_FORMAT) 11432 SVN_ERR(svn_io_file_create(path_min_unpacked_rev(dst_fs, pool), 11433 "0\n", pool)); 11434 /* Create the txn-current file if the repository supports 11435 the transaction sequence file. */ 11436 if (dst_ffd->format >= SVN_FS_FS__MIN_TXN_CURRENT_FORMAT) 11437 { 11438 SVN_ERR(svn_io_file_create(path_txn_current(dst_fs, pool), 11439 "0\n", pool)); 11440 SVN_ERR(svn_io_file_create(path_txn_current_lock(dst_fs, pool), 11441 "", pool)); 11442 } 11443 11444 dst_ffd->youngest_rev_cache = 0; 11445 11446 hotcopy_setup_shared_fs_data(src_fs, dst_fs); 11447 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); 11448 11449 return SVN_NO_ERROR; 11450} 11451 11452svn_error_t * 11453svn_fs_fs__hotcopy(svn_fs_t *src_fs, 11454 svn_fs_t *dst_fs, 11455 const char *src_path, 11456 const char *dst_path, 11457 svn_boolean_t incremental, 11458 svn_cancel_func_t cancel_func, 11459 void *cancel_baton, 11460 apr_pool_t *pool) 11461{ 11462 struct hotcopy_body_baton hbb; 11463 11464 if (cancel_func) 11465 SVN_ERR(cancel_func(cancel_baton)); 11466 11467 SVN_ERR(svn_fs_fs__open(src_fs, src_path, pool)); 11468 11469 if (incremental) 11470 { 11471 const char *dst_format_abspath; 11472 svn_node_kind_t dst_format_kind; 11473 11474 /* Check destination format to be sure we know how to incrementally 11475 * hotcopy to the destination FS. */ 11476 dst_format_abspath = svn_dirent_join(dst_path, PATH_FORMAT, pool); 11477 SVN_ERR(svn_io_check_path(dst_format_abspath, &dst_format_kind, pool)); 11478 if (dst_format_kind == svn_node_none) 11479 { 11480 /* Destination doesn't exist yet. Perform a normal hotcopy to a 11481 * empty destination using the same configuration as the source. */ 11482 SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); 11483 } 11484 else 11485 { 11486 /* Check the existing repository. */ 11487 SVN_ERR(svn_fs_fs__open(dst_fs, dst_path, pool)); 11488 SVN_ERR(hotcopy_incremental_check_preconditions(src_fs, dst_fs, 11489 pool)); 11490 hotcopy_setup_shared_fs_data(src_fs, dst_fs); 11491 SVN_ERR(svn_fs_fs__initialize_caches(dst_fs, pool)); 11492 } 11493 } 11494 else 11495 { 11496 /* Start out with an empty destination using the same configuration 11497 * as the source. */ 11498 SVN_ERR(hotcopy_create_empty_dest(src_fs, dst_fs, dst_path, pool)); 11499 } 11500 11501 if (cancel_func) 11502 SVN_ERR(cancel_func(cancel_baton)); 11503 11504 hbb.src_fs = src_fs; 11505 hbb.dst_fs = dst_fs; 11506 hbb.incremental = incremental; 11507 hbb.cancel_func = cancel_func; 11508 hbb.cancel_baton = cancel_baton; 11509 SVN_ERR(svn_fs_fs__with_write_lock(dst_fs, hotcopy_body, &hbb, pool)); 11510 11511 return SVN_NO_ERROR; 11512} 11513