sync.c revision 272461
1/* 2 * ==================================================================== 3 * Licensed to the Apache Software Foundation (ASF) under one 4 * or more contributor license agreements. See the NOTICE file 5 * distributed with this work for additional information 6 * regarding copyright ownership. The ASF licenses this file 7 * to you under the Apache License, Version 2.0 (the 8 * "License"); you may not use this file except in compliance 9 * with the License. You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, 14 * software distributed under the License is distributed on an 15 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 * KIND, either express or implied. See the License for the 17 * specific language governing permissions and limitations 18 * under the License. 19 * ==================================================================== 20 */ 21 22#include "svn_hash.h" 23#include "svn_cmdline.h" 24#include "svn_config.h" 25#include "svn_pools.h" 26#include "svn_delta.h" 27#include "svn_dirent_uri.h" 28#include "svn_path.h" 29#include "svn_props.h" 30#include "svn_auth.h" 31#include "svn_opt.h" 32#include "svn_ra.h" 33#include "svn_utf.h" 34#include "svn_subst.h" 35#include "svn_string.h" 36 37#include "sync.h" 38 39#include "svn_private_config.h" 40 41#include <apr_network_io.h> 42#include <apr_signal.h> 43#include <apr_uuid.h> 44 45 46/* Normalize the encoding and line ending style of *STR, so that it contains 47 * only LF (\n) line endings and is encoded in UTF-8. After return, *STR may 48 * point at a new svn_string_t* allocated in RESULT_POOL. 49 * 50 * If SOURCE_PROP_ENCODING is NULL, then *STR is presumed to be encoded in 51 * UTF-8. 52 * 53 * *WAS_NORMALIZED is set to TRUE when *STR needed line ending normalization. 54 * Otherwise it is set to FALSE. 55 * 56 * SCRATCH_POOL is used for temporary allocations. 57 */ 58static svn_error_t * 59normalize_string(const svn_string_t **str, 60 svn_boolean_t *was_normalized, 61 const char *source_prop_encoding, 62 apr_pool_t *result_pool, 63 apr_pool_t *scratch_pool) 64{ 65 svn_string_t *new_str; 66 67 *was_normalized = FALSE; 68 69 if (*str == NULL) 70 return SVN_NO_ERROR; 71 72 SVN_ERR_ASSERT((*str)->data != NULL); 73 74 if (source_prop_encoding == NULL) 75 source_prop_encoding = "UTF-8"; 76 77 new_str = NULL; 78 SVN_ERR(svn_subst_translate_string2(&new_str, NULL, was_normalized, 79 *str, source_prop_encoding, TRUE, 80 result_pool, scratch_pool)); 81 *str = new_str; 82 83 return SVN_NO_ERROR; 84} 85 86 87/* Normalize the encoding and line ending style of the values of properties 88 * in REV_PROPS that "need translation" (according to 89 * svn_prop_needs_translation(), which is currently all svn:* props) so that 90 * they are encoded in UTF-8 and contain only LF (\n) line endings. 91 * 92 * The number of properties that needed line ending normalization is returned in 93 * *NORMALIZED_COUNT. 94 * 95 * No re-encoding is performed if SOURCE_PROP_ENCODING is NULL. 96 */ 97svn_error_t * 98svnsync_normalize_revprops(apr_hash_t *rev_props, 99 int *normalized_count, 100 const char *source_prop_encoding, 101 apr_pool_t *pool) 102{ 103 apr_hash_index_t *hi; 104 *normalized_count = 0; 105 106 for (hi = apr_hash_first(pool, rev_props); 107 hi; 108 hi = apr_hash_next(hi)) 109 { 110 const char *propname = svn__apr_hash_index_key(hi); 111 const svn_string_t *propval = svn__apr_hash_index_val(hi); 112 113 if (svn_prop_needs_translation(propname)) 114 { 115 svn_boolean_t was_normalized; 116 SVN_ERR(normalize_string(&propval, &was_normalized, 117 source_prop_encoding, pool, pool)); 118 119 /* Replace the existing prop value. */ 120 svn_hash_sets(rev_props, propname, propval); 121 122 if (was_normalized) 123 (*normalized_count)++; /* Count it. */ 124 } 125 } 126 return SVN_NO_ERROR; 127} 128 129 130/*** Synchronization Editor ***/ 131 132/* This editor has a couple of jobs. 133 * 134 * First, it needs to filter out the propchanges that can't be passed over 135 * libsvn_ra. 136 * 137 * Second, it needs to adjust for the fact that we might not actually have 138 * permission to see all of the data from the remote repository, which means 139 * we could get revisions that are totally empty from our point of view. 140 * 141 * Third, it needs to adjust copyfrom paths, adding the root url for the 142 * destination repository to the beginning of them. 143 */ 144 145 146/* Edit baton */ 147typedef struct edit_baton_t { 148 const svn_delta_editor_t *wrapped_editor; 149 void *wrapped_edit_baton; 150 const char *to_url; /* URL we're copying into, for correct copyfrom URLs */ 151 const char *source_prop_encoding; 152 svn_boolean_t called_open_root; 153 svn_boolean_t got_textdeltas; 154 svn_revnum_t base_revision; 155 svn_boolean_t quiet; 156 svn_boolean_t strip_mergeinfo; /* Are we stripping svn:mergeinfo? */ 157 svn_boolean_t migrate_svnmerge; /* Are we converting svnmerge.py data? */ 158 svn_boolean_t mergeinfo_stripped; /* Did we strip svn:mergeinfo? */ 159 svn_boolean_t svnmerge_migrated; /* Did we convert svnmerge.py data? */ 160 svn_boolean_t svnmerge_blocked; /* Was there any blocked svnmerge data? */ 161 int *normalized_node_props_counter; /* Where to count normalizations? */ 162} edit_baton_t; 163 164 165/* A dual-purpose baton for files and directories. */ 166typedef struct node_baton_t { 167 void *edit_baton; 168 void *wrapped_node_baton; 169} node_baton_t; 170 171 172/*** Editor vtable functions ***/ 173 174static svn_error_t * 175set_target_revision(void *edit_baton, 176 svn_revnum_t target_revision, 177 apr_pool_t *pool) 178{ 179 edit_baton_t *eb = edit_baton; 180 return eb->wrapped_editor->set_target_revision(eb->wrapped_edit_baton, 181 target_revision, pool); 182} 183 184static svn_error_t * 185open_root(void *edit_baton, 186 svn_revnum_t base_revision, 187 apr_pool_t *pool, 188 void **root_baton) 189{ 190 edit_baton_t *eb = edit_baton; 191 node_baton_t *dir_baton = apr_palloc(pool, sizeof(*dir_baton)); 192 193 SVN_ERR(eb->wrapped_editor->open_root(eb->wrapped_edit_baton, 194 base_revision, pool, 195 &dir_baton->wrapped_node_baton)); 196 197 eb->called_open_root = TRUE; 198 dir_baton->edit_baton = edit_baton; 199 *root_baton = dir_baton; 200 201 return SVN_NO_ERROR; 202} 203 204static svn_error_t * 205delete_entry(const char *path, 206 svn_revnum_t base_revision, 207 void *parent_baton, 208 apr_pool_t *pool) 209{ 210 node_baton_t *pb = parent_baton; 211 edit_baton_t *eb = pb->edit_baton; 212 213 return eb->wrapped_editor->delete_entry(path, base_revision, 214 pb->wrapped_node_baton, pool); 215} 216 217static svn_error_t * 218add_directory(const char *path, 219 void *parent_baton, 220 const char *copyfrom_path, 221 svn_revnum_t copyfrom_rev, 222 apr_pool_t *pool, 223 void **child_baton) 224{ 225 node_baton_t *pb = parent_baton; 226 edit_baton_t *eb = pb->edit_baton; 227 node_baton_t *b = apr_palloc(pool, sizeof(*b)); 228 229 /* if copyfrom_path is an fspath create a proper uri */ 230 if (copyfrom_path && copyfrom_path[0] == '/') 231 copyfrom_path = svn_path_url_add_component2(eb->to_url, 232 copyfrom_path + 1, pool); 233 234 SVN_ERR(eb->wrapped_editor->add_directory(path, pb->wrapped_node_baton, 235 copyfrom_path, 236 copyfrom_rev, pool, 237 &b->wrapped_node_baton)); 238 239 b->edit_baton = eb; 240 *child_baton = b; 241 242 return SVN_NO_ERROR; 243} 244 245static svn_error_t * 246open_directory(const char *path, 247 void *parent_baton, 248 svn_revnum_t base_revision, 249 apr_pool_t *pool, 250 void **child_baton) 251{ 252 node_baton_t *pb = parent_baton; 253 edit_baton_t *eb = pb->edit_baton; 254 node_baton_t *db = apr_palloc(pool, sizeof(*db)); 255 256 SVN_ERR(eb->wrapped_editor->open_directory(path, pb->wrapped_node_baton, 257 base_revision, pool, 258 &db->wrapped_node_baton)); 259 260 db->edit_baton = eb; 261 *child_baton = db; 262 263 return SVN_NO_ERROR; 264} 265 266static svn_error_t * 267add_file(const char *path, 268 void *parent_baton, 269 const char *copyfrom_path, 270 svn_revnum_t copyfrom_rev, 271 apr_pool_t *pool, 272 void **file_baton) 273{ 274 node_baton_t *pb = parent_baton; 275 edit_baton_t *eb = pb->edit_baton; 276 node_baton_t *fb = apr_palloc(pool, sizeof(*fb)); 277 278 /* if copyfrom_path is an fspath create a proper uri */ 279 if (copyfrom_path && copyfrom_path[0] == '/') 280 copyfrom_path = svn_path_url_add_component2(eb->to_url, 281 copyfrom_path + 1, pool); 282 283 SVN_ERR(eb->wrapped_editor->add_file(path, pb->wrapped_node_baton, 284 copyfrom_path, copyfrom_rev, 285 pool, &fb->wrapped_node_baton)); 286 287 fb->edit_baton = eb; 288 *file_baton = fb; 289 290 return SVN_NO_ERROR; 291} 292 293static svn_error_t * 294open_file(const char *path, 295 void *parent_baton, 296 svn_revnum_t base_revision, 297 apr_pool_t *pool, 298 void **file_baton) 299{ 300 node_baton_t *pb = parent_baton; 301 edit_baton_t *eb = pb->edit_baton; 302 node_baton_t *fb = apr_palloc(pool, sizeof(*fb)); 303 304 SVN_ERR(eb->wrapped_editor->open_file(path, pb->wrapped_node_baton, 305 base_revision, pool, 306 &fb->wrapped_node_baton)); 307 308 fb->edit_baton = eb; 309 *file_baton = fb; 310 311 return SVN_NO_ERROR; 312} 313 314static svn_error_t * 315apply_textdelta(void *file_baton, 316 const char *base_checksum, 317 apr_pool_t *pool, 318 svn_txdelta_window_handler_t *handler, 319 void **handler_baton) 320{ 321 node_baton_t *fb = file_baton; 322 edit_baton_t *eb = fb->edit_baton; 323 324 if (! eb->quiet) 325 { 326 if (! eb->got_textdeltas) 327 SVN_ERR(svn_cmdline_printf(pool, _("Transmitting file data "))); 328 SVN_ERR(svn_cmdline_printf(pool, ".")); 329 SVN_ERR(svn_cmdline_fflush(stdout)); 330 } 331 332 eb->got_textdeltas = TRUE; 333 return eb->wrapped_editor->apply_textdelta(fb->wrapped_node_baton, 334 base_checksum, pool, 335 handler, handler_baton); 336} 337 338static svn_error_t * 339close_file(void *file_baton, 340 const char *text_checksum, 341 apr_pool_t *pool) 342{ 343 node_baton_t *fb = file_baton; 344 edit_baton_t *eb = fb->edit_baton; 345 return eb->wrapped_editor->close_file(fb->wrapped_node_baton, 346 text_checksum, pool); 347} 348 349static svn_error_t * 350absent_file(const char *path, 351 void *file_baton, 352 apr_pool_t *pool) 353{ 354 node_baton_t *fb = file_baton; 355 edit_baton_t *eb = fb->edit_baton; 356 return eb->wrapped_editor->absent_file(path, fb->wrapped_node_baton, pool); 357} 358 359static svn_error_t * 360close_directory(void *dir_baton, 361 apr_pool_t *pool) 362{ 363 node_baton_t *db = dir_baton; 364 edit_baton_t *eb = db->edit_baton; 365 return eb->wrapped_editor->close_directory(db->wrapped_node_baton, pool); 366} 367 368static svn_error_t * 369absent_directory(const char *path, 370 void *dir_baton, 371 apr_pool_t *pool) 372{ 373 node_baton_t *db = dir_baton; 374 edit_baton_t *eb = db->edit_baton; 375 return eb->wrapped_editor->absent_directory(path, db->wrapped_node_baton, 376 pool); 377} 378 379static svn_error_t * 380change_file_prop(void *file_baton, 381 const char *name, 382 const svn_string_t *value, 383 apr_pool_t *pool) 384{ 385 node_baton_t *fb = file_baton; 386 edit_baton_t *eb = fb->edit_baton; 387 388 /* only regular properties can pass over libsvn_ra */ 389 if (svn_property_kind2(name) != svn_prop_regular_kind) 390 return SVN_NO_ERROR; 391 392 /* Maybe drop svn:mergeinfo. */ 393 if (eb->strip_mergeinfo && (strcmp(name, SVN_PROP_MERGEINFO) == 0)) 394 { 395 eb->mergeinfo_stripped = TRUE; 396 return SVN_NO_ERROR; 397 } 398 399 /* Maybe drop (errantly set, as this is a file) svnmerge.py properties. */ 400 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-integrated") == 0)) 401 { 402 eb->svnmerge_migrated = TRUE; 403 return SVN_NO_ERROR; 404 } 405 406 /* Remember if we see any svnmerge-blocked properties. (They really 407 shouldn't be here, as this is a file, but whatever...) */ 408 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-blocked") == 0)) 409 { 410 eb->svnmerge_blocked = TRUE; 411 } 412 413 /* Normalize svn:* properties as necessary. */ 414 if (svn_prop_needs_translation(name)) 415 { 416 svn_boolean_t was_normalized; 417 SVN_ERR(normalize_string(&value, &was_normalized, 418 eb->source_prop_encoding, pool, pool)); 419 if (was_normalized) 420 (*(eb->normalized_node_props_counter))++; 421 } 422 423 return eb->wrapped_editor->change_file_prop(fb->wrapped_node_baton, 424 name, value, pool); 425} 426 427static svn_error_t * 428change_dir_prop(void *dir_baton, 429 const char *name, 430 const svn_string_t *value, 431 apr_pool_t *pool) 432{ 433 node_baton_t *db = dir_baton; 434 edit_baton_t *eb = db->edit_baton; 435 436 /* Only regular properties can pass over libsvn_ra */ 437 if (svn_property_kind2(name) != svn_prop_regular_kind) 438 return SVN_NO_ERROR; 439 440 /* Maybe drop svn:mergeinfo. */ 441 if (eb->strip_mergeinfo && (strcmp(name, SVN_PROP_MERGEINFO) == 0)) 442 { 443 eb->mergeinfo_stripped = TRUE; 444 return SVN_NO_ERROR; 445 } 446 447 /* Maybe convert svnmerge-integrated data into svn:mergeinfo. (We 448 ignore svnmerge-blocked for now.) */ 449 /* ### FIXME: Consult the mirror repository's HEAD prop values and 450 ### merge svn:mergeinfo, svnmerge-integrated, and svnmerge-blocked. */ 451 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-integrated") == 0)) 452 { 453 if (value) 454 { 455 /* svnmerge-integrated differs from svn:mergeinfo in a pair 456 of ways. First, it can use tabs, newlines, or spaces to 457 delimit source information. Secondly, the source paths 458 are relative URLs, whereas svn:mergeinfo uses relative 459 paths (not URI-encoded). */ 460 svn_error_t *err; 461 svn_stringbuf_t *mergeinfo_buf = svn_stringbuf_create_empty(pool); 462 svn_mergeinfo_t mergeinfo; 463 int i; 464 apr_array_header_t *sources = 465 svn_cstring_split(value->data, " \t\n", TRUE, pool); 466 svn_string_t *new_value; 467 468 for (i = 0; i < sources->nelts; i++) 469 { 470 const char *rel_path; 471 apr_array_header_t *path_revs = 472 svn_cstring_split(APR_ARRAY_IDX(sources, i, const char *), 473 ":", TRUE, pool); 474 475 /* ### TODO: Warn? */ 476 if (path_revs->nelts != 2) 477 continue; 478 479 /* Append this source's mergeinfo data. */ 480 rel_path = APR_ARRAY_IDX(path_revs, 0, const char *); 481 rel_path = svn_path_uri_decode(rel_path, pool); 482 svn_stringbuf_appendcstr(mergeinfo_buf, rel_path); 483 svn_stringbuf_appendcstr(mergeinfo_buf, ":"); 484 svn_stringbuf_appendcstr(mergeinfo_buf, 485 APR_ARRAY_IDX(path_revs, 1, 486 const char *)); 487 svn_stringbuf_appendcstr(mergeinfo_buf, "\n"); 488 } 489 490 /* Try to parse the mergeinfo string we've created, just to 491 check for bogosity. If all goes well, we'll unparse it 492 again and use that as our property value. */ 493 err = svn_mergeinfo_parse(&mergeinfo, mergeinfo_buf->data, pool); 494 if (err) 495 { 496 svn_error_clear(err); 497 return SVN_NO_ERROR; 498 } 499 SVN_ERR(svn_mergeinfo_to_string(&new_value, mergeinfo, pool)); 500 value = new_value; 501 } 502 name = SVN_PROP_MERGEINFO; 503 eb->svnmerge_migrated = TRUE; 504 } 505 506 /* Remember if we see any svnmerge-blocked properties. */ 507 if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-blocked") == 0)) 508 { 509 eb->svnmerge_blocked = TRUE; 510 } 511 512 /* Normalize svn:* properties as necessary. */ 513 if (svn_prop_needs_translation(name)) 514 { 515 svn_boolean_t was_normalized; 516 SVN_ERR(normalize_string(&value, &was_normalized, eb->source_prop_encoding, 517 pool, pool)); 518 if (was_normalized) 519 (*(eb->normalized_node_props_counter))++; 520 } 521 522 return eb->wrapped_editor->change_dir_prop(db->wrapped_node_baton, 523 name, value, pool); 524} 525 526static svn_error_t * 527close_edit(void *edit_baton, 528 apr_pool_t *pool) 529{ 530 edit_baton_t *eb = edit_baton; 531 532 /* If we haven't opened the root yet, that means we're transfering 533 an empty revision, probably because we aren't allowed to see the 534 contents for some reason. In any event, we need to open the root 535 and close it again, before we can close out the edit, or the 536 commit will fail. */ 537 538 if (! eb->called_open_root) 539 { 540 void *baton; 541 SVN_ERR(eb->wrapped_editor->open_root(eb->wrapped_edit_baton, 542 eb->base_revision, pool, 543 &baton)); 544 SVN_ERR(eb->wrapped_editor->close_directory(baton, pool)); 545 } 546 547 if (! eb->quiet) 548 { 549 if (eb->got_textdeltas) 550 SVN_ERR(svn_cmdline_printf(pool, "\n")); 551 if (eb->mergeinfo_stripped) 552 SVN_ERR(svn_cmdline_printf(pool, 553 "NOTE: Dropped Subversion mergeinfo " 554 "from this revision.\n")); 555 if (eb->svnmerge_migrated) 556 SVN_ERR(svn_cmdline_printf(pool, 557 "NOTE: Migrated 'svnmerge-integrated' in " 558 "this revision.\n")); 559 if (eb->svnmerge_blocked) 560 SVN_ERR(svn_cmdline_printf(pool, 561 "NOTE: Saw 'svnmerge-blocked' in this " 562 "revision (but didn't migrate it).\n")); 563 } 564 565 return eb->wrapped_editor->close_edit(eb->wrapped_edit_baton, pool); 566} 567 568static svn_error_t * 569abort_edit(void *edit_baton, 570 apr_pool_t *pool) 571{ 572 edit_baton_t *eb = edit_baton; 573 return eb->wrapped_editor->abort_edit(eb->wrapped_edit_baton, pool); 574} 575 576 577/*** Editor factory function ***/ 578 579svn_error_t * 580svnsync_get_sync_editor(const svn_delta_editor_t *wrapped_editor, 581 void *wrapped_edit_baton, 582 svn_revnum_t base_revision, 583 const char *to_url, 584 const char *source_prop_encoding, 585 svn_boolean_t quiet, 586 const svn_delta_editor_t **editor, 587 void **edit_baton, 588 int *normalized_node_props_counter, 589 apr_pool_t *pool) 590{ 591 svn_delta_editor_t *tree_editor = svn_delta_default_editor(pool); 592 edit_baton_t *eb = apr_pcalloc(pool, sizeof(*eb)); 593 594 tree_editor->set_target_revision = set_target_revision; 595 tree_editor->open_root = open_root; 596 tree_editor->delete_entry = delete_entry; 597 tree_editor->add_directory = add_directory; 598 tree_editor->open_directory = open_directory; 599 tree_editor->change_dir_prop = change_dir_prop; 600 tree_editor->close_directory = close_directory; 601 tree_editor->absent_directory = absent_directory; 602 tree_editor->add_file = add_file; 603 tree_editor->open_file = open_file; 604 tree_editor->apply_textdelta = apply_textdelta; 605 tree_editor->change_file_prop = change_file_prop; 606 tree_editor->close_file = close_file; 607 tree_editor->absent_file = absent_file; 608 tree_editor->close_edit = close_edit; 609 tree_editor->abort_edit = abort_edit; 610 611 eb->wrapped_editor = wrapped_editor; 612 eb->wrapped_edit_baton = wrapped_edit_baton; 613 eb->base_revision = base_revision; 614 eb->to_url = to_url; 615 eb->source_prop_encoding = source_prop_encoding; 616 eb->quiet = quiet; 617 eb->normalized_node_props_counter = normalized_node_props_counter; 618 619 if (getenv("SVNSYNC_UNSUPPORTED_STRIP_MERGEINFO")) 620 { 621 eb->strip_mergeinfo = TRUE; 622 } 623 if (getenv("SVNSYNC_UNSUPPORTED_MIGRATE_SVNMERGE")) 624 { 625 /* Current we can't merge property values. That's only possible 626 if all the properties to be merged were always modified in 627 exactly the same revisions, or if we allow ourselves to 628 lookup the current state of properties in the sync 629 destination. So for now, migrating svnmerge.py data implies 630 stripping pre-existing svn:mergeinfo. */ 631 /* ### FIXME: Do a real migration by consulting the mirror 632 ### repository's HEAD propvalues and merging svn:mergeinfo, 633 ### svnmerge-integrated, and svnmerge-blocked together. */ 634 eb->migrate_svnmerge = TRUE; 635 eb->strip_mergeinfo = TRUE; 636 } 637 638 *editor = tree_editor; 639 *edit_baton = eb; 640 641 return SVN_NO_ERROR; 642} 643 644