revprops.c revision 299742
1254721Semaste/* revprops.c --- everything needed to handle revprops in FSFS 2254721Semaste * 3353358Sdim * ==================================================================== 4353358Sdim * Licensed to the Apache Software Foundation (ASF) under one 5353358Sdim * or more contributor license agreements. See the NOTICE file 6254721Semaste * distributed with this work for additional information 7254721Semaste * regarding copyright ownership. The ASF licenses this file 8254721Semaste * to you under the Apache License, Version 2.0 (the 9254721Semaste * "License"); you may not use this file except in compliance 10254721Semaste * with the License. You may obtain a copy of the License at 11254721Semaste * 12254721Semaste * http://www.apache.org/licenses/LICENSE-2.0 13254721Semaste * 14254721Semaste * Unless required by applicable law or agreed to in writing, 15254721Semaste * software distributed under the License is distributed on an 16314564Sdim * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17314564Sdim * KIND, either express or implied. See the License for the 18314564Sdim * specific language governing permissions and limitations 19254721Semaste * under the License. 20314564Sdim * ==================================================================== 21314564Sdim */ 22314564Sdim 23314564Sdim#include <assert.h> 24314564Sdim 25254721Semaste#include "svn_pools.h" 26254721Semaste#include "svn_hash.h" 27314564Sdim#include "svn_dirent_uri.h" 28288943Sdim 29314564Sdim#include "fs_fs.h" 30314564Sdim#include "revprops.h" 31314564Sdim#include "util.h" 32288943Sdim 33#include "private/svn_subr_private.h" 34#include "private/svn_string_private.h" 35#include "../libsvn_fs/fs-loader.h" 36 37#include "svn_private_config.h" 38 39/* Give writing processes 10 seconds to replace an existing revprop 40 file with a new one. After that time, we assume that the writing 41 process got aborted and that we have re-read revprops. */ 42#define REVPROP_CHANGE_TIMEOUT (10 * 1000000) 43 44svn_error_t * 45svn_fs_fs__upgrade_pack_revprops(svn_fs_t *fs, 46 svn_fs_upgrade_notify_t notify_func, 47 void *notify_baton, 48 svn_cancel_func_t cancel_func, 49 void *cancel_baton, 50 apr_pool_t *scratch_pool) 51{ 52 fs_fs_data_t *ffd = fs->fsap_data; 53 const char *revprops_shard_path; 54 const char *revprops_pack_file_dir; 55 apr_int64_t shard; 56 apr_int64_t first_unpacked_shard 57 = ffd->min_unpacked_rev / ffd->max_files_per_dir; 58 59 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 60 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 61 scratch_pool); 62 int compression_level = ffd->compress_packed_revprops 63 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 64 : SVN_DELTA_COMPRESSION_LEVEL_NONE; 65 66 /* first, pack all revprops shards to match the packed revision shards */ 67 for (shard = 0; shard < first_unpacked_shard; ++shard) 68 { 69 svn_pool_clear(iterpool); 70 71 revprops_pack_file_dir = svn_dirent_join(revsprops_dir, 72 apr_psprintf(iterpool, 73 "%" APR_INT64_T_FMT PATH_EXT_PACKED_SHARD, 74 shard), 75 iterpool); 76 revprops_shard_path = svn_dirent_join(revsprops_dir, 77 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 78 iterpool); 79 80 SVN_ERR(svn_fs_fs__pack_revprops_shard(revprops_pack_file_dir, 81 revprops_shard_path, 82 shard, ffd->max_files_per_dir, 83 (int)(0.9 * ffd->revprop_pack_size), 84 compression_level, 85 cancel_func, cancel_baton, 86 iterpool)); 87 if (notify_func) 88 SVN_ERR(notify_func(notify_baton, shard, 89 svn_fs_upgrade_pack_revprops, iterpool)); 90 } 91 92 svn_pool_destroy(iterpool); 93 94 return SVN_NO_ERROR; 95} 96 97svn_error_t * 98svn_fs_fs__upgrade_cleanup_pack_revprops(svn_fs_t *fs, 99 svn_fs_upgrade_notify_t notify_func, 100 void *notify_baton, 101 svn_cancel_func_t cancel_func, 102 void *cancel_baton, 103 apr_pool_t *scratch_pool) 104{ 105 fs_fs_data_t *ffd = fs->fsap_data; 106 const char *revprops_shard_path; 107 apr_int64_t shard; 108 apr_int64_t first_unpacked_shard 109 = ffd->min_unpacked_rev / ffd->max_files_per_dir; 110 111 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 112 const char *revsprops_dir = svn_dirent_join(fs->path, PATH_REVPROPS_DIR, 113 scratch_pool); 114 115 /* delete the non-packed revprops shards afterwards */ 116 for (shard = 0; shard < first_unpacked_shard; ++shard) 117 { 118 svn_pool_clear(iterpool); 119 120 revprops_shard_path = svn_dirent_join(revsprops_dir, 121 apr_psprintf(iterpool, "%" APR_INT64_T_FMT, shard), 122 iterpool); 123 SVN_ERR(svn_fs_fs__delete_revprops_shard(revprops_shard_path, 124 shard, 125 ffd->max_files_per_dir, 126 cancel_func, cancel_baton, 127 iterpool)); 128 if (notify_func) 129 SVN_ERR(notify_func(notify_baton, shard, 130 svn_fs_upgrade_cleanup_revprops, iterpool)); 131 } 132 133 svn_pool_destroy(iterpool); 134 135 return SVN_NO_ERROR; 136} 137 138/* Container for all data required to access the packed revprop file 139 * for a given REVISION. This structure will be filled incrementally 140 * by read_pack_revprops() its sub-routines. 141 */ 142typedef struct packed_revprops_t 143{ 144 /* revision number to read (not necessarily the first in the pack) */ 145 svn_revnum_t revision; 146 147 /* current revprop generation. Used when populating the revprop cache */ 148 apr_int64_t generation; 149 150 /* the actual revision properties */ 151 apr_hash_t *properties; 152 153 /* their size when serialized to a single string 154 * (as found in PACKED_REVPROPS) */ 155 apr_size_t serialized_size; 156 157 158 /* name of the pack file (without folder path) */ 159 const char *filename; 160 161 /* packed shard folder path */ 162 const char *folder; 163 164 /* sum of values in SIZES */ 165 apr_size_t total_size; 166 167 /* first revision in the pack (>= MANIFEST_START) */ 168 svn_revnum_t start_revision; 169 170 /* size of the revprops in PACKED_REVPROPS */ 171 apr_array_header_t *sizes; 172 173 /* offset of the revprops in PACKED_REVPROPS */ 174 apr_array_header_t *offsets; 175 176 177 /* concatenation of the serialized representation of all revprops 178 * in the pack, i.e. the pack content without header and compression */ 179 svn_stringbuf_t *packed_revprops; 180 181 /* First revision covered by MANIFEST. 182 * Will equal the shard start revision or 1, for the 1st shard. */ 183 svn_revnum_t manifest_start; 184 185 /* content of the manifest. 186 * Maps long(rev - MANIFEST_START) to const char* pack file name */ 187 apr_array_header_t *manifest; 188} packed_revprops_t; 189 190/* Parse the serialized revprops in CONTENT and return them in *PROPERTIES. 191 * Also, put them into the revprop cache, if activated, for future use. 192 * Three more parameters are being used to update the revprop cache: FS is 193 * our file system, the revprops belong to REVISION and the global revprop 194 * GENERATION is used as well. 195 * 196 * The returned hash will be allocated in POOL, SCRATCH_POOL is being used 197 * for temporary allocations. 198 */ 199static svn_error_t * 200parse_revprop(apr_hash_t **properties, 201 svn_fs_t *fs, 202 svn_revnum_t revision, 203 apr_int64_t generation, 204 svn_string_t *content, 205 apr_pool_t *pool, 206 apr_pool_t *scratch_pool) 207{ 208 svn_stream_t *stream = svn_stream_from_string(content, scratch_pool); 209 *properties = apr_hash_make(pool); 210 211 SVN_ERR_W(svn_hash_read2(*properties, stream, SVN_HASH_TERMINATOR, pool), 212 apr_psprintf(scratch_pool, "Failed to parse revprops for r%ld.", 213 revision)); 214 215 return SVN_NO_ERROR; 216} 217 218/* Read the non-packed revprops for revision REV in FS, put them into the 219 * revprop cache if activated and return them in *PROPERTIES. GENERATION 220 * is the current revprop generation. 221 * 222 * If the data could not be read due to an otherwise recoverable error, 223 * leave *PROPERTIES unchanged. No error will be returned in that case. 224 * 225 * Allocations will be done in POOL. 226 */ 227static svn_error_t * 228read_non_packed_revprop(apr_hash_t **properties, 229 svn_fs_t *fs, 230 svn_revnum_t rev, 231 apr_int64_t generation, 232 apr_pool_t *pool) 233{ 234 svn_stringbuf_t *content = NULL; 235 apr_pool_t *iterpool = svn_pool_create(pool); 236 svn_boolean_t missing = FALSE; 237 int i; 238 239 for (i = 0; 240 i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !missing && !content; 241 ++i) 242 { 243 svn_pool_clear(iterpool); 244 SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&content, 245 &missing, 246 svn_fs_fs__path_revprops(fs, rev, iterpool), 247 i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT , 248 iterpool)); 249 } 250 251 if (content) 252 SVN_ERR(parse_revprop(properties, fs, rev, generation, 253 svn_stringbuf__morph_into_string(content), 254 pool, iterpool)); 255 256 svn_pool_clear(iterpool); 257 258 return SVN_NO_ERROR; 259} 260 261/* Return the minimum length of any packed revprop file name in REVPROPS. */ 262static apr_size_t 263get_min_filename_len(packed_revprops_t *revprops) 264{ 265 char number_buffer[SVN_INT64_BUFFER_SIZE]; 266 267 /* The revprop filenames have the format <REV>.<COUNT> - with <REV> being 268 * at least the first rev in the shard and <COUNT> having at least one 269 * digit. Thus, the minimum is 2 + #decimal places in the start rev. 270 */ 271 return svn__i64toa(number_buffer, revprops->manifest_start) + 2; 272} 273 274/* Given FS and REVPROPS->REVISION, fill the FILENAME, FOLDER and MANIFEST 275 * members. Use POOL for allocating results and SCRATCH_POOL for temporaries. 276 */ 277static svn_error_t * 278get_revprop_packname(svn_fs_t *fs, 279 packed_revprops_t *revprops, 280 apr_pool_t *pool, 281 apr_pool_t *scratch_pool) 282{ 283 fs_fs_data_t *ffd = fs->fsap_data; 284 svn_stringbuf_t *content = NULL; 285 const char *manifest_file_path; 286 int idx, rev_count; 287 char *buffer, *buffer_end; 288 const char **filenames, **filenames_end; 289 apr_size_t min_filename_len; 290 291 /* Determine the dimensions. Rev 0 is excluded from the first shard. */ 292 rev_count = ffd->max_files_per_dir; 293 revprops->manifest_start 294 = revprops->revision - (revprops->revision % rev_count); 295 if (revprops->manifest_start == 0) 296 { 297 ++revprops->manifest_start; 298 --rev_count; 299 } 300 301 revprops->manifest = apr_array_make(pool, rev_count, sizeof(const char*)); 302 303 /* No line in the file can be less than this number of chars long. */ 304 min_filename_len = get_min_filename_len(revprops); 305 306 /* Read the content of the manifest file */ 307 revprops->folder 308 = svn_fs_fs__path_revprops_pack_shard(fs, revprops->revision, pool); 309 manifest_file_path 310 = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); 311 312 SVN_ERR(svn_fs_fs__read_content(&content, manifest_file_path, pool)); 313 314 /* There CONTENT must have a certain minimal size and there no 315 * unterminated lines at the end of the file. Both guarantees also 316 * simplify the parser loop below. 317 */ 318 if ( content->len < rev_count * (min_filename_len + 1) 319 || content->data[content->len - 1] != '\n') 320 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 321 _("Packed revprop manifest for r%ld not " 322 "properly terminated"), revprops->revision); 323 324 /* Chop (parse) the manifest CONTENT into filenames, one per line. 325 * We only have to replace all newlines with NUL and add all line 326 * starts to REVPROPS->MANIFEST. 327 * 328 * There must be exactly REV_COUNT lines and that is the number of 329 * lines we parse from BUFFER to FILENAMES. Set the end pointer for 330 * the source BUFFER such that BUFFER+MIN_FILENAME_LEN is still valid 331 * BUFFER_END is always valid due to CONTENT->LEN > MIN_FILENAME_LEN. 332 * 333 * Please note that this loop is performance critical for e.g. 'svn log'. 334 * It is run 1000x per revprop access, i.e. per revision and about 335 * 50 million times per sec (and CPU core). 336 */ 337 for (filenames = (const char **)revprops->manifest->elts, 338 filenames_end = filenames + rev_count, 339 buffer = content->data, 340 buffer_end = buffer + content->len - min_filename_len; 341 (filenames < filenames_end) && (buffer < buffer_end); 342 ++filenames) 343 { 344 /* BUFFER always points to the start of the next line / filename. */ 345 *filenames = buffer; 346 347 /* Find the next EOL. This is guaranteed to stay within the CONTENT 348 * buffer because we left enough room after BUFFER_END and we know 349 * we will always see a newline as the last non-NUL char. */ 350 buffer += min_filename_len; 351 while (*buffer != '\n') 352 ++buffer; 353 354 /* Found EOL. Turn it into the filename terminator and move BUFFER 355 * to the start of the next line or CONTENT buffer end. */ 356 *buffer = '\0'; 357 ++buffer; 358 } 359 360 /* We must have reached the end of both buffers. */ 361 if (buffer < content->data + content->len) 362 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 363 _("Packed revprop manifest for r%ld " 364 "has too many entries"), revprops->revision); 365 366 if (filenames < filenames_end) 367 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 368 _("Packed revprop manifest for r%ld " 369 "has too few entries"), revprops->revision); 370 371 /* The target array has now exactly one entry per revision. */ 372 revprops->manifest->nelts = rev_count; 373 374 /* Now get the file name */ 375 idx = (int)(revprops->revision - revprops->manifest_start); 376 revprops->filename = APR_ARRAY_IDX(revprops->manifest, idx, const char*); 377 378 return SVN_NO_ERROR; 379} 380 381/* Return TRUE, if revision R1 and R2 refer to the same shard in FS. 382 */ 383static svn_boolean_t 384same_shard(svn_fs_t *fs, 385 svn_revnum_t r1, 386 svn_revnum_t r2) 387{ 388 fs_fs_data_t *ffd = fs->fsap_data; 389 return (r1 / ffd->max_files_per_dir) == (r2 / ffd->max_files_per_dir); 390} 391 392/* Given FS and the full packed file content in REVPROPS->PACKED_REVPROPS, 393 * fill the START_REVISION member, and make PACKED_REVPROPS point to the 394 * first serialized revprop. If READ_ALL is set, initialize the SIZES 395 * and OFFSETS members as well. 396 * 397 * Parse the revprops for REVPROPS->REVISION and set the PROPERTIES as 398 * well as the SERIALIZED_SIZE member. If revprop caching has been 399 * enabled, parse all revprops in the pack and cache them. 400 */ 401static svn_error_t * 402parse_packed_revprops(svn_fs_t *fs, 403 packed_revprops_t *revprops, 404 svn_boolean_t read_all, 405 apr_pool_t *pool, 406 apr_pool_t *scratch_pool) 407{ 408 svn_stream_t *stream; 409 apr_int64_t first_rev, count, i; 410 apr_off_t offset; 411 const char *header_end; 412 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 413 414 /* decompress (even if the data is only "stored", there is still a 415 * length header to remove) */ 416 svn_stringbuf_t *compressed = revprops->packed_revprops; 417 svn_stringbuf_t *uncompressed = svn_stringbuf_create_empty(pool); 418 SVN_ERR(svn__decompress(compressed, uncompressed, APR_SIZE_MAX)); 419 420 /* read first revision number and number of revisions in the pack */ 421 stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 422 SVN_ERR(svn_fs_fs__read_number_from_stream(&first_rev, NULL, stream, 423 iterpool)); 424 SVN_ERR(svn_fs_fs__read_number_from_stream(&count, NULL, stream, 425 iterpool)); 426 427 /* Check revision range for validity. */ 428 if ( !same_shard(fs, revprops->revision, first_rev) 429 || !same_shard(fs, revprops->revision, first_rev + count - 1) 430 || count < 1) 431 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 432 _("Revprop pack for revision r%ld" 433 " contains revprops for r%ld .. r%ld"), 434 revprops->revision, 435 (svn_revnum_t)first_rev, 436 (svn_revnum_t)(first_rev + count -1)); 437 438 /* Since start & end are in the same shard, it is enough to just test 439 * the FIRST_REV for being actually packed. That will also cover the 440 * special case of rev 0 never being packed. */ 441 if (!svn_fs_fs__is_packed_revprop(fs, first_rev)) 442 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 443 _("Revprop pack for revision r%ld" 444 " starts at non-packed revisions r%ld"), 445 revprops->revision, (svn_revnum_t)first_rev); 446 447 /* make PACKED_REVPROPS point to the first char after the header. 448 * This is where the serialized revprops are. */ 449 header_end = strstr(uncompressed->data, "\n\n"); 450 if (header_end == NULL) 451 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 452 _("Header end not found")); 453 454 offset = header_end - uncompressed->data + 2; 455 456 revprops->packed_revprops = svn_stringbuf_create_empty(pool); 457 revprops->packed_revprops->data = uncompressed->data + offset; 458 revprops->packed_revprops->len = (apr_size_t)(uncompressed->len - offset); 459 revprops->packed_revprops->blocksize = (apr_size_t)(uncompressed->blocksize - offset); 460 461 /* STREAM still points to the first entry in the sizes list. */ 462 revprops->start_revision = (svn_revnum_t)first_rev; 463 if (read_all) 464 { 465 /* Init / construct REVPROPS members. */ 466 revprops->sizes = apr_array_make(pool, (int)count, sizeof(offset)); 467 revprops->offsets = apr_array_make(pool, (int)count, sizeof(offset)); 468 } 469 470 /* Now parse, revision by revision, the size and content of each 471 * revisions' revprops. */ 472 for (i = 0, offset = 0, revprops->total_size = 0; i < count; ++i) 473 { 474 apr_int64_t size; 475 svn_string_t serialized; 476 svn_revnum_t revision = (svn_revnum_t)(first_rev + i); 477 svn_pool_clear(iterpool); 478 479 /* read & check the serialized size */ 480 SVN_ERR(svn_fs_fs__read_number_from_stream(&size, NULL, stream, 481 iterpool)); 482 if (size + offset > (apr_int64_t)revprops->packed_revprops->len) 483 return svn_error_create(SVN_ERR_FS_CORRUPT, NULL, 484 _("Packed revprop size exceeds pack file size")); 485 486 /* Parse this revprops list, if necessary */ 487 serialized.data = revprops->packed_revprops->data + offset; 488 serialized.len = (apr_size_t)size; 489 490 if (revision == revprops->revision) 491 { 492 SVN_ERR(parse_revprop(&revprops->properties, fs, revision, 493 revprops->generation, &serialized, 494 pool, iterpool)); 495 revprops->serialized_size = serialized.len; 496 497 /* If we only wanted the revprops for REVISION then we are done. */ 498 if (!read_all) 499 break; 500 } 501 502 if (read_all) 503 { 504 /* fill REVPROPS data structures */ 505 APR_ARRAY_PUSH(revprops->sizes, apr_off_t) = serialized.len; 506 APR_ARRAY_PUSH(revprops->offsets, apr_off_t) = offset; 507 } 508 revprops->total_size += serialized.len; 509 510 offset += serialized.len; 511 } 512 513 return SVN_NO_ERROR; 514} 515 516/* In filesystem FS, read the packed revprops for revision REV into 517 * *REVPROPS. Use GENERATION to populate the revprop cache, if enabled. 518 * If you want to modify revprop contents / update REVPROPS, READ_ALL 519 * must be set. Otherwise, only the properties of REV are being provided. 520 * Allocate data in POOL. 521 */ 522static svn_error_t * 523read_pack_revprop(packed_revprops_t **revprops, 524 svn_fs_t *fs, 525 svn_revnum_t rev, 526 apr_int64_t generation, 527 svn_boolean_t read_all, 528 apr_pool_t *pool) 529{ 530 apr_pool_t *iterpool = svn_pool_create(pool); 531 svn_boolean_t missing = FALSE; 532 svn_error_t *err; 533 packed_revprops_t *result; 534 int i; 535 536 /* someone insisted that REV is packed. Double-check if necessary */ 537 if (!svn_fs_fs__is_packed_revprop(fs, rev)) 538 SVN_ERR(svn_fs_fs__update_min_unpacked_rev(fs, iterpool)); 539 540 if (!svn_fs_fs__is_packed_revprop(fs, rev)) 541 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 542 _("No such packed revision %ld"), rev); 543 544 /* initialize the result data structure */ 545 result = apr_pcalloc(pool, sizeof(*result)); 546 result->revision = rev; 547 result->generation = generation; 548 549 /* try to read the packed revprops. This may require retries if we have 550 * concurrent writers. */ 551 for (i = 0; 552 i < SVN_FS_FS__RECOVERABLE_RETRY_COUNT && !result->packed_revprops; 553 ++i) 554 { 555 const char *file_path; 556 svn_pool_clear(iterpool); 557 558 /* there might have been concurrent writes. 559 * Re-read the manifest and the pack file. 560 */ 561 SVN_ERR(get_revprop_packname(fs, result, pool, iterpool)); 562 file_path = svn_dirent_join(result->folder, 563 result->filename, 564 iterpool); 565 SVN_ERR(svn_fs_fs__try_stringbuf_from_file(&result->packed_revprops, 566 &missing, 567 file_path, 568 i + 1 < SVN_FS_FS__RECOVERABLE_RETRY_COUNT, 569 pool)); 570 } 571 572 /* the file content should be available now */ 573 if (!result->packed_revprops) 574 return svn_error_createf(SVN_ERR_FS_PACKED_REVPROP_READ_FAILURE, NULL, 575 _("Failed to read revprop pack file for r%ld"), rev); 576 577 /* parse it. RESULT will be complete afterwards. */ 578 err = parse_packed_revprops(fs, result, read_all, pool, iterpool); 579 svn_pool_destroy(iterpool); 580 if (err) 581 return svn_error_createf(SVN_ERR_FS_CORRUPT, err, 582 _("Revprop pack file for r%ld is corrupt"), rev); 583 584 *revprops = result; 585 586 return SVN_NO_ERROR; 587} 588 589/* Read the revprops for revision REV in FS and return them in *PROPERTIES_P. 590 * 591 * Allocations will be done in POOL. 592 */ 593svn_error_t * 594svn_fs_fs__get_revision_proplist(apr_hash_t **proplist_p, 595 svn_fs_t *fs, 596 svn_revnum_t rev, 597 apr_pool_t *pool) 598{ 599 fs_fs_data_t *ffd = fs->fsap_data; 600 apr_int64_t generation = 0; 601 602 /* not found, yet */ 603 *proplist_p = NULL; 604 605 /* should they be available at all? */ 606 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool)); 607 608 /* if REV had not been packed when we began, try reading it from the 609 * non-packed shard. If that fails, we will fall through to packed 610 * shard reads. */ 611 if (!svn_fs_fs__is_packed_revprop(fs, rev)) 612 { 613 svn_error_t *err = read_non_packed_revprop(proplist_p, fs, rev, 614 generation, pool); 615 if (err) 616 { 617 if (!APR_STATUS_IS_ENOENT(err->apr_err) 618 || ffd->format < SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT) 619 return svn_error_trace(err); 620 621 svn_error_clear(err); 622 *proplist_p = NULL; /* in case read_non_packed_revprop changed it */ 623 } 624 } 625 626 /* if revprop packing is available and we have not read the revprops, yet, 627 * try reading them from a packed shard. If that fails, REV is most 628 * likely invalid (or its revprops highly contested). */ 629 if (ffd->format >= SVN_FS_FS__MIN_PACKED_REVPROP_FORMAT && !*proplist_p) 630 { 631 packed_revprops_t *revprops; 632 SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, FALSE, pool)); 633 *proplist_p = revprops->properties; 634 } 635 636 /* The revprops should have been there. Did we get them? */ 637 if (!*proplist_p) 638 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION, NULL, 639 _("Could not read revprops for revision %ld"), 640 rev); 641 642 return SVN_NO_ERROR; 643} 644 645/* Serialize the revision property list PROPLIST of revision REV in 646 * filesystem FS to a non-packed file. Return the name of that temporary 647 * file in *TMP_PATH and the file path that it must be moved to in 648 * *FINAL_PATH. 649 * 650 * Use POOL for allocations. 651 */ 652static svn_error_t * 653write_non_packed_revprop(const char **final_path, 654 const char **tmp_path, 655 svn_fs_t *fs, 656 svn_revnum_t rev, 657 apr_hash_t *proplist, 658 apr_pool_t *pool) 659{ 660 apr_file_t *file; 661 svn_stream_t *stream; 662 *final_path = svn_fs_fs__path_revprops(fs, rev, pool); 663 664 /* ### do we have a directory sitting around already? we really shouldn't 665 ### have to get the dirname here. */ 666 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, 667 svn_dirent_dirname(*final_path, pool), 668 svn_io_file_del_none, pool, pool)); 669 stream = svn_stream_from_aprfile2(file, TRUE, pool); 670 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 671 SVN_ERR(svn_stream_close(stream)); 672 673 /* Flush temporary file to disk and close it. */ 674 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 675 SVN_ERR(svn_io_file_close(file, pool)); 676 677 return SVN_NO_ERROR; 678} 679 680/* After writing the new revprop file(s), call this function to move the 681 * file at TMP_PATH to FINAL_PATH and give it the permissions from 682 * PERMS_REFERENCE. 683 * 684 * Finally, delete all the temporary files given in FILES_TO_DELETE. 685 * The latter may be NULL. 686 * 687 * Use POOL for temporary allocations. 688 */ 689static svn_error_t * 690switch_to_new_revprop(svn_fs_t *fs, 691 const char *final_path, 692 const char *tmp_path, 693 const char *perms_reference, 694 apr_array_header_t *files_to_delete, 695 apr_pool_t *pool) 696{ 697 SVN_ERR(svn_fs_fs__move_into_place(tmp_path, final_path, perms_reference, 698 pool)); 699 700 /* Clean up temporary files, if necessary. */ 701 if (files_to_delete) 702 { 703 apr_pool_t *iterpool = svn_pool_create(pool); 704 int i; 705 706 for (i = 0; i < files_to_delete->nelts; ++i) 707 { 708 const char *path = APR_ARRAY_IDX(files_to_delete, i, const char*); 709 710 svn_pool_clear(iterpool); 711 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 712 } 713 714 svn_pool_destroy(iterpool); 715 } 716 return SVN_NO_ERROR; 717} 718 719/* Write a pack file header to STREAM that starts at revision START_REVISION 720 * and contains the indexes [START,END) of SIZES. 721 */ 722static svn_error_t * 723serialize_revprops_header(svn_stream_t *stream, 724 svn_revnum_t start_revision, 725 apr_array_header_t *sizes, 726 int start, 727 int end, 728 apr_pool_t *pool) 729{ 730 apr_pool_t *iterpool = svn_pool_create(pool); 731 int i; 732 733 SVN_ERR_ASSERT(start < end); 734 735 /* start revision and entry count */ 736 SVN_ERR(svn_stream_printf(stream, pool, "%ld\n", start_revision)); 737 SVN_ERR(svn_stream_printf(stream, pool, "%d\n", end - start)); 738 739 /* the sizes array */ 740 for (i = start; i < end; ++i) 741 { 742 /* Non-standard pool usage. 743 * 744 * We only allocate a few bytes each iteration -- even with a 745 * million iterations we would still be in good shape memory-wise. 746 */ 747 apr_off_t size = APR_ARRAY_IDX(sizes, i, apr_off_t); 748 SVN_ERR(svn_stream_printf(stream, iterpool, "%" APR_OFF_T_FMT "\n", 749 size)); 750 } 751 752 /* the double newline char indicates the end of the header */ 753 SVN_ERR(svn_stream_printf(stream, iterpool, "\n")); 754 755 svn_pool_destroy(iterpool); 756 return SVN_NO_ERROR; 757} 758 759/* Writes the a pack file to FILE. It copies the serialized data 760 * from REVPROPS for the indexes [START,END) except for index CHANGED_INDEX. 761 * 762 * The data for the latter is taken from NEW_SERIALIZED. Note, that 763 * CHANGED_INDEX may be outside the [START,END) range, i.e. no new data is 764 * taken in that case but only a subset of the old data will be copied. 765 * 766 * NEW_TOTAL_SIZE is a hint for pre-allocating buffers of appropriate size. 767 * POOL is used for temporary allocations. 768 */ 769static svn_error_t * 770repack_revprops(svn_fs_t *fs, 771 packed_revprops_t *revprops, 772 int start, 773 int end, 774 int changed_index, 775 svn_stringbuf_t *new_serialized, 776 apr_off_t new_total_size, 777 apr_file_t *file, 778 apr_pool_t *pool) 779{ 780 fs_fs_data_t *ffd = fs->fsap_data; 781 svn_stream_t *stream; 782 int i; 783 784 /* create data empty buffers and the stream object */ 785 svn_stringbuf_t *uncompressed 786 = svn_stringbuf_create_ensure((apr_size_t)new_total_size, pool); 787 svn_stringbuf_t *compressed 788 = svn_stringbuf_create_empty(pool); 789 stream = svn_stream_from_stringbuf(uncompressed, pool); 790 791 /* write the header*/ 792 SVN_ERR(serialize_revprops_header(stream, revprops->start_revision + start, 793 revprops->sizes, start, end, pool)); 794 795 /* append the serialized revprops */ 796 for (i = start; i < end; ++i) 797 if (i == changed_index) 798 { 799 SVN_ERR(svn_stream_write(stream, 800 new_serialized->data, 801 &new_serialized->len)); 802 } 803 else 804 { 805 apr_size_t size 806 = (apr_size_t)APR_ARRAY_IDX(revprops->sizes, i, apr_off_t); 807 apr_size_t offset 808 = (apr_size_t)APR_ARRAY_IDX(revprops->offsets, i, apr_off_t); 809 810 SVN_ERR(svn_stream_write(stream, 811 revprops->packed_revprops->data + offset, 812 &size)); 813 } 814 815 /* flush the stream buffer (if any) to our underlying data buffer */ 816 SVN_ERR(svn_stream_close(stream)); 817 818 /* compress / store the data */ 819 SVN_ERR(svn__compress(uncompressed, 820 compressed, 821 ffd->compress_packed_revprops 822 ? SVN_DELTA_COMPRESSION_LEVEL_DEFAULT 823 : SVN_DELTA_COMPRESSION_LEVEL_NONE)); 824 825 /* finally, write the content to the target file, flush and close it */ 826 SVN_ERR(svn_io_file_write_full(file, compressed->data, compressed->len, 827 NULL, pool)); 828 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 829 SVN_ERR(svn_io_file_close(file, pool)); 830 831 return SVN_NO_ERROR; 832} 833 834/* Allocate a new pack file name for revisions 835 * [REVPROPS->START_REVISION + START, REVPROPS->START_REVISION + END - 1] 836 * of REVPROPS->MANIFEST. Add the name of old file to FILES_TO_DELETE, 837 * auto-create that array if necessary. Return an open file *FILE that is 838 * allocated in POOL. 839 */ 840static svn_error_t * 841repack_file_open(apr_file_t **file, 842 svn_fs_t *fs, 843 packed_revprops_t *revprops, 844 int start, 845 int end, 846 apr_array_header_t **files_to_delete, 847 apr_pool_t *pool) 848{ 849 apr_int64_t tag; 850 const char *tag_string; 851 svn_string_t *new_filename; 852 int i; 853 int manifest_offset 854 = (int)(revprops->start_revision - revprops->manifest_start); 855 856 /* get the old (= current) file name and enlist it for later deletion */ 857 const char *old_filename = APR_ARRAY_IDX(revprops->manifest, 858 start + manifest_offset, 859 const char*); 860 861 if (*files_to_delete == NULL) 862 *files_to_delete = apr_array_make(pool, 3, sizeof(const char*)); 863 864 APR_ARRAY_PUSH(*files_to_delete, const char*) 865 = svn_dirent_join(revprops->folder, old_filename, pool); 866 867 /* increase the tag part, i.e. the counter after the dot */ 868 tag_string = strchr(old_filename, '.'); 869 if (tag_string == NULL) 870 return svn_error_createf(SVN_ERR_FS_CORRUPT, NULL, 871 _("Packed file '%s' misses a tag"), 872 old_filename); 873 874 SVN_ERR(svn_cstring_atoi64(&tag, tag_string + 1)); 875 new_filename = svn_string_createf(pool, "%ld.%" APR_INT64_T_FMT, 876 revprops->start_revision + start, 877 ++tag); 878 879 /* update the manifest to point to the new file */ 880 for (i = start; i < end; ++i) 881 APR_ARRAY_IDX(revprops->manifest, i + manifest_offset, const char*) 882 = new_filename->data; 883 884 /* open the file */ 885 SVN_ERR(svn_io_file_open(file, svn_dirent_join(revprops->folder, 886 new_filename->data, 887 pool), 888 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, pool)); 889 890 return SVN_NO_ERROR; 891} 892 893/* For revision REV in filesystem FS, set the revision properties to 894 * PROPLIST. Return a new file in *TMP_PATH that the caller shall move 895 * to *FINAL_PATH to make the change visible. Files to be deleted will 896 * be listed in *FILES_TO_DELETE which may remain unchanged / unallocated. 897 * Use POOL for allocations. 898 */ 899static svn_error_t * 900write_packed_revprop(const char **final_path, 901 const char **tmp_path, 902 apr_array_header_t **files_to_delete, 903 svn_fs_t *fs, 904 svn_revnum_t rev, 905 apr_hash_t *proplist, 906 apr_pool_t *pool) 907{ 908 fs_fs_data_t *ffd = fs->fsap_data; 909 packed_revprops_t *revprops; 910 apr_int64_t generation = 0; 911 svn_stream_t *stream; 912 apr_file_t *file; 913 svn_stringbuf_t *serialized; 914 apr_off_t new_total_size; 915 int changed_index; 916 917 /* read contents of the current pack file */ 918 SVN_ERR(read_pack_revprop(&revprops, fs, rev, generation, TRUE, pool)); 919 920 /* serialize the new revprops */ 921 serialized = svn_stringbuf_create_empty(pool); 922 stream = svn_stream_from_stringbuf(serialized, pool); 923 SVN_ERR(svn_hash_write2(proplist, stream, SVN_HASH_TERMINATOR, pool)); 924 SVN_ERR(svn_stream_close(stream)); 925 926 /* calculate the size of the new data */ 927 changed_index = (int)(rev - revprops->start_revision); 928 new_total_size = revprops->total_size - revprops->serialized_size 929 + serialized->len 930 + (revprops->offsets->nelts + 2) * SVN_INT64_BUFFER_SIZE; 931 932 APR_ARRAY_IDX(revprops->sizes, changed_index, apr_off_t) = serialized->len; 933 934 /* can we put the new data into the same pack as the before? */ 935 if ( new_total_size < ffd->revprop_pack_size 936 || revprops->sizes->nelts == 1) 937 { 938 /* simply replace the old pack file with new content as we do it 939 * in the non-packed case */ 940 941 *final_path = svn_dirent_join(revprops->folder, revprops->filename, 942 pool); 943 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder, 944 svn_io_file_del_none, pool, pool)); 945 SVN_ERR(repack_revprops(fs, revprops, 0, revprops->sizes->nelts, 946 changed_index, serialized, new_total_size, 947 file, pool)); 948 } 949 else 950 { 951 /* split the pack file into two of roughly equal size */ 952 int right_count, left_count, i; 953 954 int left = 0; 955 int right = revprops->sizes->nelts - 1; 956 apr_off_t left_size = 2 * SVN_INT64_BUFFER_SIZE; 957 apr_off_t right_size = 2 * SVN_INT64_BUFFER_SIZE; 958 959 /* let left and right side grow such that their size difference 960 * is minimal after each step. */ 961 while (left <= right) 962 if ( left_size + APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) 963 < right_size + APR_ARRAY_IDX(revprops->sizes, right, apr_off_t)) 964 { 965 left_size += APR_ARRAY_IDX(revprops->sizes, left, apr_off_t) 966 + SVN_INT64_BUFFER_SIZE; 967 ++left; 968 } 969 else 970 { 971 right_size += APR_ARRAY_IDX(revprops->sizes, right, apr_off_t) 972 + SVN_INT64_BUFFER_SIZE; 973 --right; 974 } 975 976 /* since the items need much less than SVN_INT64_BUFFER_SIZE 977 * bytes to represent their length, the split may not be optimal */ 978 left_count = left; 979 right_count = revprops->sizes->nelts - left; 980 981 /* if new_size is large, one side may exceed the pack size limit. 982 * In that case, split before and after the modified revprop.*/ 983 if ( left_size > ffd->revprop_pack_size 984 || right_size > ffd->revprop_pack_size) 985 { 986 left_count = changed_index; 987 right_count = revprops->sizes->nelts - left_count - 1; 988 } 989 990 /* write the new, split files */ 991 if (left_count) 992 { 993 SVN_ERR(repack_file_open(&file, fs, revprops, 0, 994 left_count, files_to_delete, pool)); 995 SVN_ERR(repack_revprops(fs, revprops, 0, left_count, 996 changed_index, serialized, new_total_size, 997 file, pool)); 998 } 999 1000 if (left_count + right_count < revprops->sizes->nelts) 1001 { 1002 SVN_ERR(repack_file_open(&file, fs, revprops, changed_index, 1003 changed_index + 1, files_to_delete, 1004 pool)); 1005 SVN_ERR(repack_revprops(fs, revprops, changed_index, 1006 changed_index + 1, 1007 changed_index, serialized, new_total_size, 1008 file, pool)); 1009 } 1010 1011 if (right_count) 1012 { 1013 SVN_ERR(repack_file_open(&file, fs, revprops, 1014 revprops->sizes->nelts - right_count, 1015 revprops->sizes->nelts, 1016 files_to_delete, pool)); 1017 SVN_ERR(repack_revprops(fs, revprops, 1018 revprops->sizes->nelts - right_count, 1019 revprops->sizes->nelts, changed_index, 1020 serialized, new_total_size, file, 1021 pool)); 1022 } 1023 1024 /* write the new manifest */ 1025 *final_path = svn_dirent_join(revprops->folder, PATH_MANIFEST, pool); 1026 SVN_ERR(svn_io_open_unique_file3(&file, tmp_path, revprops->folder, 1027 svn_io_file_del_none, pool, pool)); 1028 1029 for (i = 0; i < revprops->manifest->nelts; ++i) 1030 { 1031 const char *filename = APR_ARRAY_IDX(revprops->manifest, i, 1032 const char*); 1033 SVN_ERR(svn_io_file_write_full(file, filename, strlen(filename), 1034 NULL, pool)); 1035 SVN_ERR(svn_io_file_putc('\n', file, pool)); 1036 } 1037 1038 SVN_ERR(svn_io_file_flush_to_disk(file, pool)); 1039 SVN_ERR(svn_io_file_close(file, pool)); 1040 } 1041 1042 return SVN_NO_ERROR; 1043} 1044 1045/* Set the revision property list of revision REV in filesystem FS to 1046 PROPLIST. Use POOL for temporary allocations. */ 1047svn_error_t * 1048svn_fs_fs__set_revision_proplist(svn_fs_t *fs, 1049 svn_revnum_t rev, 1050 apr_hash_t *proplist, 1051 apr_pool_t *pool) 1052{ 1053 svn_boolean_t is_packed; 1054 const char *final_path; 1055 const char *tmp_path; 1056 const char *perms_reference; 1057 apr_array_header_t *files_to_delete = NULL; 1058 1059 SVN_ERR(svn_fs_fs__ensure_revision_exists(rev, fs, pool)); 1060 1061 /* this info will not change while we hold the global FS write lock */ 1062 is_packed = svn_fs_fs__is_packed_revprop(fs, rev); 1063 1064 /* Serialize the new revprop data */ 1065 if (is_packed) 1066 SVN_ERR(write_packed_revprop(&final_path, &tmp_path, &files_to_delete, 1067 fs, rev, proplist, pool)); 1068 else 1069 SVN_ERR(write_non_packed_revprop(&final_path, &tmp_path, 1070 fs, rev, proplist, pool)); 1071 1072 /* We use the rev file of this revision as the perms reference, 1073 * because when setting revprops for the first time, the revprop 1074 * file won't exist and therefore can't serve as its own reference. 1075 * (Whereas the rev file should already exist at this point.) 1076 */ 1077 perms_reference = svn_fs_fs__path_rev_absolute(fs, rev, pool); 1078 1079 /* Now, switch to the new revprop data. */ 1080 SVN_ERR(switch_to_new_revprop(fs, final_path, tmp_path, perms_reference, 1081 files_to_delete, pool)); 1082 1083 return SVN_NO_ERROR; 1084} 1085 1086/* Return TRUE, if for REVISION in FS, we can find the revprop pack file. 1087 * Use POOL for temporary allocations. 1088 * Set *MISSING, if the reason is a missing manifest or pack file. 1089 */ 1090svn_boolean_t 1091svn_fs_fs__packed_revprop_available(svn_boolean_t *missing, 1092 svn_fs_t *fs, 1093 svn_revnum_t revision, 1094 apr_pool_t *pool) 1095{ 1096 fs_fs_data_t *ffd = fs->fsap_data; 1097 svn_stringbuf_t *content = NULL; 1098 1099 /* try to read the manifest file */ 1100 const char *folder 1101 = svn_fs_fs__path_revprops_pack_shard(fs, revision, pool); 1102 const char *manifest_path = svn_dirent_join(folder, PATH_MANIFEST, pool); 1103 1104 svn_error_t *err = svn_fs_fs__try_stringbuf_from_file(&content, 1105 missing, 1106 manifest_path, 1107 FALSE, 1108 pool); 1109 1110 /* if the manifest cannot be read, consider the pack files inaccessible 1111 * even if the file itself exists. */ 1112 if (err) 1113 { 1114 svn_error_clear(err); 1115 return FALSE; 1116 } 1117 1118 if (*missing) 1119 return FALSE; 1120 1121 /* parse manifest content until we find the entry for REVISION. 1122 * Revision 0 is never packed. */ 1123 revision = revision < ffd->max_files_per_dir 1124 ? revision - 1 1125 : revision % ffd->max_files_per_dir; 1126 while (content->data) 1127 { 1128 char *next = strchr(content->data, '\n'); 1129 if (next) 1130 { 1131 *next = 0; 1132 ++next; 1133 } 1134 1135 if (revision-- == 0) 1136 { 1137 /* the respective pack file must exist (and be a file) */ 1138 svn_node_kind_t kind; 1139 err = svn_io_check_path(svn_dirent_join(folder, content->data, 1140 pool), 1141 &kind, pool); 1142 if (err) 1143 { 1144 svn_error_clear(err); 1145 return FALSE; 1146 } 1147 1148 *missing = kind == svn_node_none; 1149 return kind == svn_node_file; 1150 } 1151 1152 content->data = next; 1153 } 1154 1155 return FALSE; 1156} 1157 1158 1159/****** Packing FSFS shards *********/ 1160 1161svn_error_t * 1162svn_fs_fs__copy_revprops(const char *pack_file_dir, 1163 const char *pack_filename, 1164 const char *shard_path, 1165 svn_revnum_t start_rev, 1166 svn_revnum_t end_rev, 1167 apr_array_header_t *sizes, 1168 apr_size_t total_size, 1169 int compression_level, 1170 svn_cancel_func_t cancel_func, 1171 void *cancel_baton, 1172 apr_pool_t *scratch_pool) 1173{ 1174 svn_stream_t *pack_stream; 1175 apr_file_t *pack_file; 1176 svn_revnum_t rev; 1177 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1178 1179 /* create empty data buffer and a write stream on top of it */ 1180 svn_stringbuf_t *uncompressed 1181 = svn_stringbuf_create_ensure(total_size, scratch_pool); 1182 svn_stringbuf_t *compressed 1183 = svn_stringbuf_create_empty(scratch_pool); 1184 pack_stream = svn_stream_from_stringbuf(uncompressed, scratch_pool); 1185 1186 /* write the pack file header */ 1187 SVN_ERR(serialize_revprops_header(pack_stream, start_rev, sizes, 0, 1188 sizes->nelts, iterpool)); 1189 1190 /* Some useful paths. */ 1191 SVN_ERR(svn_io_file_open(&pack_file, svn_dirent_join(pack_file_dir, 1192 pack_filename, 1193 scratch_pool), 1194 APR_WRITE | APR_CREATE, APR_OS_DEFAULT, 1195 scratch_pool)); 1196 1197 /* Iterate over the revisions in this shard, squashing them together. */ 1198 for (rev = start_rev; rev <= end_rev; rev++) 1199 { 1200 const char *path; 1201 svn_stream_t *stream; 1202 1203 svn_pool_clear(iterpool); 1204 1205 /* Construct the file name. */ 1206 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 1207 iterpool); 1208 1209 /* Copy all the bits from the non-packed revprop file to the end of 1210 * the pack file. */ 1211 SVN_ERR(svn_stream_open_readonly(&stream, path, iterpool, iterpool)); 1212 SVN_ERR(svn_stream_copy3(stream, pack_stream, 1213 cancel_func, cancel_baton, iterpool)); 1214 } 1215 1216 /* flush stream buffers to content buffer */ 1217 SVN_ERR(svn_stream_close(pack_stream)); 1218 1219 /* compress the content (or just store it for COMPRESSION_LEVEL 0) */ 1220 SVN_ERR(svn__compress(uncompressed, compressed, compression_level)); 1221 1222 /* write the pack file content to disk */ 1223 SVN_ERR(svn_io_file_write_full(pack_file, compressed->data, compressed->len, 1224 NULL, scratch_pool)); 1225 SVN_ERR(svn_io_file_flush_to_disk(pack_file, scratch_pool)); 1226 SVN_ERR(svn_io_file_close(pack_file, scratch_pool)); 1227 1228 svn_pool_destroy(iterpool); 1229 1230 return SVN_NO_ERROR; 1231} 1232 1233svn_error_t * 1234svn_fs_fs__pack_revprops_shard(const char *pack_file_dir, 1235 const char *shard_path, 1236 apr_int64_t shard, 1237 int max_files_per_dir, 1238 apr_off_t max_pack_size, 1239 int compression_level, 1240 svn_cancel_func_t cancel_func, 1241 void *cancel_baton, 1242 apr_pool_t *scratch_pool) 1243{ 1244 const char *manifest_file_path, *pack_filename = NULL; 1245 apr_file_t *manifest_file; 1246 svn_stream_t *manifest_stream; 1247 svn_revnum_t start_rev, end_rev, rev; 1248 apr_off_t total_size; 1249 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1250 apr_array_header_t *sizes; 1251 1252 /* Some useful paths. */ 1253 manifest_file_path = svn_dirent_join(pack_file_dir, PATH_MANIFEST, 1254 scratch_pool); 1255 1256 /* Remove any existing pack file for this shard, since it is incomplete. */ 1257 SVN_ERR(svn_io_remove_dir2(pack_file_dir, TRUE, cancel_func, cancel_baton, 1258 scratch_pool)); 1259 1260 /* Create the new directory and manifest file stream. */ 1261 SVN_ERR(svn_io_dir_make(pack_file_dir, APR_OS_DEFAULT, scratch_pool)); 1262 1263 SVN_ERR(svn_io_file_open(&manifest_file, manifest_file_path, 1264 APR_WRITE | APR_BUFFERED | APR_CREATE | APR_EXCL, 1265 APR_OS_DEFAULT, scratch_pool)); 1266 manifest_stream = svn_stream_from_aprfile2(manifest_file, TRUE, 1267 scratch_pool); 1268 1269 /* revisions to handle. Special case: revision 0 */ 1270 start_rev = (svn_revnum_t) (shard * max_files_per_dir); 1271 end_rev = (svn_revnum_t) ((shard + 1) * (max_files_per_dir) - 1); 1272 if (start_rev == 0) 1273 ++start_rev; 1274 /* Special special case: if max_files_per_dir is 1, then at this point 1275 start_rev == 1 and end_rev == 0 (!). Fortunately, everything just 1276 works. */ 1277 1278 /* initialize the revprop size info */ 1279 sizes = apr_array_make(scratch_pool, max_files_per_dir, sizeof(apr_off_t)); 1280 total_size = 2 * SVN_INT64_BUFFER_SIZE; 1281 1282 /* Iterate over the revisions in this shard, determine their size and 1283 * squashing them together into pack files. */ 1284 for (rev = start_rev; rev <= end_rev; rev++) 1285 { 1286 apr_finfo_t finfo; 1287 const char *path; 1288 1289 svn_pool_clear(iterpool); 1290 1291 /* Get the size of the file. */ 1292 path = svn_dirent_join(shard_path, apr_psprintf(iterpool, "%ld", rev), 1293 iterpool); 1294 SVN_ERR(svn_io_stat(&finfo, path, APR_FINFO_SIZE, iterpool)); 1295 1296 /* if we already have started a pack file and this revprop cannot be 1297 * appended to it, write the previous pack file. */ 1298 if (sizes->nelts != 0 && 1299 total_size + SVN_INT64_BUFFER_SIZE + finfo.size > max_pack_size) 1300 { 1301 SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, 1302 shard_path, start_rev, rev-1, 1303 sizes, (apr_size_t)total_size, 1304 compression_level, cancel_func, 1305 cancel_baton, iterpool)); 1306 1307 /* next pack file starts empty again */ 1308 apr_array_clear(sizes); 1309 total_size = 2 * SVN_INT64_BUFFER_SIZE; 1310 start_rev = rev; 1311 } 1312 1313 /* Update the manifest. Allocate a file name for the current pack 1314 * file if it is a new one */ 1315 if (sizes->nelts == 0) 1316 pack_filename = apr_psprintf(scratch_pool, "%ld.0", rev); 1317 1318 SVN_ERR(svn_stream_printf(manifest_stream, iterpool, "%s\n", 1319 pack_filename)); 1320 1321 /* add to list of files to put into the current pack file */ 1322 APR_ARRAY_PUSH(sizes, apr_off_t) = finfo.size; 1323 total_size += SVN_INT64_BUFFER_SIZE + finfo.size; 1324 } 1325 1326 /* write the last pack file */ 1327 if (sizes->nelts != 0) 1328 SVN_ERR(svn_fs_fs__copy_revprops(pack_file_dir, pack_filename, 1329 shard_path, start_rev, rev-1, 1330 sizes, (apr_size_t)total_size, 1331 compression_level, cancel_func, 1332 cancel_baton, iterpool)); 1333 1334 /* flush the manifest file to disk and update permissions */ 1335 SVN_ERR(svn_stream_close(manifest_stream)); 1336 SVN_ERR(svn_io_file_flush_to_disk(manifest_file, iterpool)); 1337 SVN_ERR(svn_io_file_close(manifest_file, iterpool)); 1338 SVN_ERR(svn_io_copy_perms(shard_path, pack_file_dir, iterpool)); 1339 1340 svn_pool_destroy(iterpool); 1341 1342 return SVN_NO_ERROR; 1343} 1344 1345svn_error_t * 1346svn_fs_fs__delete_revprops_shard(const char *shard_path, 1347 apr_int64_t shard, 1348 int max_files_per_dir, 1349 svn_cancel_func_t cancel_func, 1350 void *cancel_baton, 1351 apr_pool_t *scratch_pool) 1352{ 1353 if (shard == 0) 1354 { 1355 apr_pool_t *iterpool = svn_pool_create(scratch_pool); 1356 int i; 1357 1358 /* delete all files except the one for revision 0 */ 1359 for (i = 1; i < max_files_per_dir; ++i) 1360 { 1361 const char *path; 1362 svn_pool_clear(iterpool); 1363 1364 path = svn_dirent_join(shard_path, 1365 apr_psprintf(iterpool, "%d", i), 1366 iterpool); 1367 if (cancel_func) 1368 SVN_ERR((*cancel_func)(cancel_baton)); 1369 1370 SVN_ERR(svn_io_remove_file2(path, TRUE, iterpool)); 1371 } 1372 1373 svn_pool_destroy(iterpool); 1374 } 1375 else 1376 SVN_ERR(svn_io_remove_dir2(shard_path, TRUE, 1377 cancel_func, cancel_baton, scratch_pool)); 1378 1379 return SVN_NO_ERROR; 1380} 1381 1382