1/*
2 * ====================================================================
3 *    Licensed to the Apache Software Foundation (ASF) under one
4 *    or more contributor license agreements.  See the NOTICE file
5 *    distributed with this work for additional information
6 *    regarding copyright ownership.  The ASF licenses this file
7 *    to you under the Apache License, Version 2.0 (the
8 *    "License"); you may not use this file except in compliance
9 *    with the License.  You may obtain a copy of the License at
10 *
11 *      http://www.apache.org/licenses/LICENSE-2.0
12 *
13 *    Unless required by applicable law or agreed to in writing,
14 *    software distributed under the License is distributed on an
15 *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16 *    KIND, either express or implied.  See the License for the
17 *    specific language governing permissions and limitations
18 *    under the License.
19 * ====================================================================
20 */
21
22#include "svn_hash.h"
23#include "svn_cmdline.h"
24#include "svn_config.h"
25#include "svn_pools.h"
26#include "svn_delta.h"
27#include "svn_dirent_uri.h"
28#include "svn_path.h"
29#include "svn_props.h"
30#include "svn_auth.h"
31#include "svn_opt.h"
32#include "svn_ra.h"
33#include "svn_utf.h"
34#include "svn_subst.h"
35#include "svn_string.h"
36
37#include "sync.h"
38
39#include "svn_private_config.h"
40
41#include <apr_network_io.h>
42#include <apr_signal.h>
43#include <apr_uuid.h>
44
45
46/* Normalize the encoding and line ending style of *STR, so that it contains
47 * only LF (\n) line endings and is encoded in UTF-8. After return, *STR may
48 * point at a new svn_string_t* allocated in RESULT_POOL.
49 *
50 * If SOURCE_PROP_ENCODING is NULL, then *STR is presumed to be encoded in
51 * UTF-8.
52 *
53 * *WAS_NORMALIZED is set to TRUE when *STR needed line ending normalization.
54 * Otherwise it is set to FALSE.
55 *
56 * SCRATCH_POOL is used for temporary allocations.
57 */
58static svn_error_t *
59normalize_string(const svn_string_t **str,
60                 svn_boolean_t *was_normalized,
61                 const char *source_prop_encoding,
62                 apr_pool_t *result_pool,
63                 apr_pool_t *scratch_pool)
64{
65  svn_string_t *new_str;
66
67  *was_normalized = FALSE;
68
69  if (*str == NULL)
70    return SVN_NO_ERROR;
71
72  SVN_ERR_ASSERT((*str)->data != NULL);
73
74  if (source_prop_encoding == NULL)
75    source_prop_encoding = "UTF-8";
76
77  new_str = NULL;
78  SVN_ERR(svn_subst_translate_string2(&new_str, NULL, was_normalized,
79                                      *str, source_prop_encoding, TRUE,
80                                      result_pool, scratch_pool));
81  *str = new_str;
82
83  return SVN_NO_ERROR;
84}
85
86
87/* Normalize the encoding and line ending style of the values of properties
88 * in REV_PROPS that "need translation" (according to
89 * svn_prop_needs_translation(), which is currently all svn:* props) so that
90 * they are encoded in UTF-8 and contain only LF (\n) line endings.
91 *
92 * The number of properties that needed line ending normalization is returned in
93 * *NORMALIZED_COUNT.
94 *
95 * No re-encoding is performed if SOURCE_PROP_ENCODING is NULL.
96 */
97svn_error_t *
98svnsync_normalize_revprops(apr_hash_t *rev_props,
99                           int *normalized_count,
100                           const char *source_prop_encoding,
101                           apr_pool_t *pool)
102{
103  apr_hash_index_t *hi;
104  *normalized_count = 0;
105
106  for (hi = apr_hash_first(pool, rev_props);
107       hi;
108       hi = apr_hash_next(hi))
109    {
110      const char *propname = svn__apr_hash_index_key(hi);
111      const svn_string_t *propval = svn__apr_hash_index_val(hi);
112
113      if (svn_prop_needs_translation(propname))
114        {
115          svn_boolean_t was_normalized;
116          SVN_ERR(normalize_string(&propval, &was_normalized,
117                  source_prop_encoding, pool, pool));
118
119          /* Replace the existing prop value. */
120          svn_hash_sets(rev_props, propname, propval);
121
122          if (was_normalized)
123            (*normalized_count)++; /* Count it. */
124        }
125    }
126  return SVN_NO_ERROR;
127}
128
129
130/*** Synchronization Editor ***/
131
132/* This editor has a couple of jobs.
133 *
134 * First, it needs to filter out the propchanges that can't be passed over
135 * libsvn_ra.
136 *
137 * Second, it needs to adjust for the fact that we might not actually have
138 * permission to see all of the data from the remote repository, which means
139 * we could get revisions that are totally empty from our point of view.
140 *
141 * Third, it needs to adjust copyfrom paths, adding the root url for the
142 * destination repository to the beginning of them.
143 */
144
145
146/* Edit baton */
147typedef struct edit_baton_t {
148  const svn_delta_editor_t *wrapped_editor;
149  void *wrapped_edit_baton;
150  const char *to_url;  /* URL we're copying into, for correct copyfrom URLs */
151  const char *source_prop_encoding;
152  svn_boolean_t called_open_root;
153  svn_boolean_t got_textdeltas;
154  svn_revnum_t base_revision;
155  svn_boolean_t quiet;
156  svn_boolean_t strip_mergeinfo;    /* Are we stripping svn:mergeinfo? */
157  svn_boolean_t migrate_svnmerge;   /* Are we converting svnmerge.py data? */
158  svn_boolean_t mergeinfo_stripped; /* Did we strip svn:mergeinfo? */
159  svn_boolean_t svnmerge_migrated;  /* Did we convert svnmerge.py data? */
160  svn_boolean_t svnmerge_blocked;   /* Was there any blocked svnmerge data? */
161  int *normalized_node_props_counter;  /* Where to count normalizations? */
162} edit_baton_t;
163
164
165/* A dual-purpose baton for files and directories. */
166typedef struct node_baton_t {
167  void *edit_baton;
168  void *wrapped_node_baton;
169} node_baton_t;
170
171
172/*** Editor vtable functions ***/
173
174static svn_error_t *
175set_target_revision(void *edit_baton,
176                    svn_revnum_t target_revision,
177                    apr_pool_t *pool)
178{
179  edit_baton_t *eb = edit_baton;
180  return eb->wrapped_editor->set_target_revision(eb->wrapped_edit_baton,
181                                                 target_revision, pool);
182}
183
184static svn_error_t *
185open_root(void *edit_baton,
186          svn_revnum_t base_revision,
187          apr_pool_t *pool,
188          void **root_baton)
189{
190  edit_baton_t *eb = edit_baton;
191  node_baton_t *dir_baton = apr_palloc(pool, sizeof(*dir_baton));
192
193  SVN_ERR(eb->wrapped_editor->open_root(eb->wrapped_edit_baton,
194                                        base_revision, pool,
195                                        &dir_baton->wrapped_node_baton));
196
197  eb->called_open_root = TRUE;
198  dir_baton->edit_baton = edit_baton;
199  *root_baton = dir_baton;
200
201  return SVN_NO_ERROR;
202}
203
204static svn_error_t *
205delete_entry(const char *path,
206             svn_revnum_t base_revision,
207             void *parent_baton,
208             apr_pool_t *pool)
209{
210  node_baton_t *pb = parent_baton;
211  edit_baton_t *eb = pb->edit_baton;
212
213  return eb->wrapped_editor->delete_entry(path, base_revision,
214                                          pb->wrapped_node_baton, pool);
215}
216
217static svn_error_t *
218add_directory(const char *path,
219              void *parent_baton,
220              const char *copyfrom_path,
221              svn_revnum_t copyfrom_rev,
222              apr_pool_t *pool,
223              void **child_baton)
224{
225  node_baton_t *pb = parent_baton;
226  edit_baton_t *eb = pb->edit_baton;
227  node_baton_t *b = apr_palloc(pool, sizeof(*b));
228
229  /* if copyfrom_path is an fspath create a proper uri */
230  if (copyfrom_path && copyfrom_path[0] == '/')
231    copyfrom_path = svn_path_url_add_component2(eb->to_url,
232                                                copyfrom_path + 1, pool);
233
234  SVN_ERR(eb->wrapped_editor->add_directory(path, pb->wrapped_node_baton,
235                                            copyfrom_path,
236                                            copyfrom_rev, pool,
237                                            &b->wrapped_node_baton));
238
239  b->edit_baton = eb;
240  *child_baton = b;
241
242  return SVN_NO_ERROR;
243}
244
245static svn_error_t *
246open_directory(const char *path,
247               void *parent_baton,
248               svn_revnum_t base_revision,
249               apr_pool_t *pool,
250               void **child_baton)
251{
252  node_baton_t *pb = parent_baton;
253  edit_baton_t *eb = pb->edit_baton;
254  node_baton_t *db = apr_palloc(pool, sizeof(*db));
255
256  SVN_ERR(eb->wrapped_editor->open_directory(path, pb->wrapped_node_baton,
257                                             base_revision, pool,
258                                             &db->wrapped_node_baton));
259
260  db->edit_baton = eb;
261  *child_baton = db;
262
263  return SVN_NO_ERROR;
264}
265
266static svn_error_t *
267add_file(const char *path,
268         void *parent_baton,
269         const char *copyfrom_path,
270         svn_revnum_t copyfrom_rev,
271         apr_pool_t *pool,
272         void **file_baton)
273{
274  node_baton_t *pb = parent_baton;
275  edit_baton_t *eb = pb->edit_baton;
276  node_baton_t *fb = apr_palloc(pool, sizeof(*fb));
277
278  /* if copyfrom_path is an fspath create a proper uri */
279  if (copyfrom_path && copyfrom_path[0] == '/')
280    copyfrom_path = svn_path_url_add_component2(eb->to_url,
281                                                copyfrom_path + 1, pool);
282
283  SVN_ERR(eb->wrapped_editor->add_file(path, pb->wrapped_node_baton,
284                                       copyfrom_path, copyfrom_rev,
285                                       pool, &fb->wrapped_node_baton));
286
287  fb->edit_baton = eb;
288  *file_baton = fb;
289
290  return SVN_NO_ERROR;
291}
292
293static svn_error_t *
294open_file(const char *path,
295          void *parent_baton,
296          svn_revnum_t base_revision,
297          apr_pool_t *pool,
298          void **file_baton)
299{
300  node_baton_t *pb = parent_baton;
301  edit_baton_t *eb = pb->edit_baton;
302  node_baton_t *fb = apr_palloc(pool, sizeof(*fb));
303
304  SVN_ERR(eb->wrapped_editor->open_file(path, pb->wrapped_node_baton,
305                                        base_revision, pool,
306                                        &fb->wrapped_node_baton));
307
308  fb->edit_baton = eb;
309  *file_baton = fb;
310
311  return SVN_NO_ERROR;
312}
313
314static svn_error_t *
315apply_textdelta(void *file_baton,
316                const char *base_checksum,
317                apr_pool_t *pool,
318                svn_txdelta_window_handler_t *handler,
319                void **handler_baton)
320{
321  node_baton_t *fb = file_baton;
322  edit_baton_t *eb = fb->edit_baton;
323
324  if (! eb->quiet)
325    {
326      if (! eb->got_textdeltas)
327        SVN_ERR(svn_cmdline_printf(pool, _("Transmitting file data ")));
328      SVN_ERR(svn_cmdline_printf(pool, "."));
329      SVN_ERR(svn_cmdline_fflush(stdout));
330    }
331
332  eb->got_textdeltas = TRUE;
333  return eb->wrapped_editor->apply_textdelta(fb->wrapped_node_baton,
334                                             base_checksum, pool,
335                                             handler, handler_baton);
336}
337
338static svn_error_t *
339close_file(void *file_baton,
340           const char *text_checksum,
341           apr_pool_t *pool)
342{
343  node_baton_t *fb = file_baton;
344  edit_baton_t *eb = fb->edit_baton;
345  return eb->wrapped_editor->close_file(fb->wrapped_node_baton,
346                                        text_checksum, pool);
347}
348
349static svn_error_t *
350absent_file(const char *path,
351            void *file_baton,
352            apr_pool_t *pool)
353{
354  node_baton_t *fb = file_baton;
355  edit_baton_t *eb = fb->edit_baton;
356  return eb->wrapped_editor->absent_file(path, fb->wrapped_node_baton, pool);
357}
358
359static svn_error_t *
360close_directory(void *dir_baton,
361                apr_pool_t *pool)
362{
363  node_baton_t *db = dir_baton;
364  edit_baton_t *eb = db->edit_baton;
365  return eb->wrapped_editor->close_directory(db->wrapped_node_baton, pool);
366}
367
368static svn_error_t *
369absent_directory(const char *path,
370                 void *dir_baton,
371                 apr_pool_t *pool)
372{
373  node_baton_t *db = dir_baton;
374  edit_baton_t *eb = db->edit_baton;
375  return eb->wrapped_editor->absent_directory(path, db->wrapped_node_baton,
376                                              pool);
377}
378
379static svn_error_t *
380change_file_prop(void *file_baton,
381                 const char *name,
382                 const svn_string_t *value,
383                 apr_pool_t *pool)
384{
385  node_baton_t *fb = file_baton;
386  edit_baton_t *eb = fb->edit_baton;
387
388  /* only regular properties can pass over libsvn_ra */
389  if (svn_property_kind2(name) != svn_prop_regular_kind)
390    return SVN_NO_ERROR;
391
392  /* Maybe drop svn:mergeinfo.  */
393  if (eb->strip_mergeinfo && (strcmp(name, SVN_PROP_MERGEINFO) == 0))
394    {
395      eb->mergeinfo_stripped = TRUE;
396      return SVN_NO_ERROR;
397    }
398
399  /* Maybe drop (errantly set, as this is a file) svnmerge.py properties. */
400  if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-integrated") == 0))
401    {
402      eb->svnmerge_migrated = TRUE;
403      return SVN_NO_ERROR;
404    }
405
406  /* Remember if we see any svnmerge-blocked properties.  (They really
407     shouldn't be here, as this is a file, but whatever...)  */
408  if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-blocked") == 0))
409    {
410      eb->svnmerge_blocked = TRUE;
411    }
412
413  /* Normalize svn:* properties as necessary. */
414  if (svn_prop_needs_translation(name))
415    {
416      svn_boolean_t was_normalized;
417      SVN_ERR(normalize_string(&value, &was_normalized,
418                               eb->source_prop_encoding, pool, pool));
419      if (was_normalized)
420        (*(eb->normalized_node_props_counter))++;
421    }
422
423  return eb->wrapped_editor->change_file_prop(fb->wrapped_node_baton,
424                                              name, value, pool);
425}
426
427static svn_error_t *
428change_dir_prop(void *dir_baton,
429                const char *name,
430                const svn_string_t *value,
431                apr_pool_t *pool)
432{
433  node_baton_t *db = dir_baton;
434  edit_baton_t *eb = db->edit_baton;
435
436  /* Only regular properties can pass over libsvn_ra */
437  if (svn_property_kind2(name) != svn_prop_regular_kind)
438    return SVN_NO_ERROR;
439
440  /* Maybe drop svn:mergeinfo.  */
441  if (eb->strip_mergeinfo && (strcmp(name, SVN_PROP_MERGEINFO) == 0))
442    {
443      eb->mergeinfo_stripped = TRUE;
444      return SVN_NO_ERROR;
445    }
446
447  /* Maybe convert svnmerge-integrated data into svn:mergeinfo.  (We
448     ignore svnmerge-blocked for now.) */
449  /* ### FIXME: Consult the mirror repository's HEAD prop values and
450     ### merge svn:mergeinfo, svnmerge-integrated, and svnmerge-blocked. */
451  if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-integrated") == 0))
452    {
453      if (value)
454        {
455          /* svnmerge-integrated differs from svn:mergeinfo in a pair
456             of ways.  First, it can use tabs, newlines, or spaces to
457             delimit source information.  Secondly, the source paths
458             are relative URLs, whereas svn:mergeinfo uses relative
459             paths (not URI-encoded). */
460          svn_error_t *err;
461          svn_stringbuf_t *mergeinfo_buf = svn_stringbuf_create_empty(pool);
462          svn_mergeinfo_t mergeinfo;
463          int i;
464          apr_array_header_t *sources =
465            svn_cstring_split(value->data, " \t\n", TRUE, pool);
466          svn_string_t *new_value;
467
468          for (i = 0; i < sources->nelts; i++)
469            {
470              const char *rel_path;
471              apr_array_header_t *path_revs =
472                svn_cstring_split(APR_ARRAY_IDX(sources, i, const char *),
473                                  ":", TRUE, pool);
474
475              /* ### TODO: Warn? */
476              if (path_revs->nelts != 2)
477                continue;
478
479              /* Append this source's mergeinfo data. */
480              rel_path = APR_ARRAY_IDX(path_revs, 0, const char *);
481              rel_path = svn_path_uri_decode(rel_path, pool);
482              svn_stringbuf_appendcstr(mergeinfo_buf, rel_path);
483              svn_stringbuf_appendcstr(mergeinfo_buf, ":");
484              svn_stringbuf_appendcstr(mergeinfo_buf,
485                                       APR_ARRAY_IDX(path_revs, 1,
486                                                     const char *));
487              svn_stringbuf_appendcstr(mergeinfo_buf, "\n");
488            }
489
490          /* Try to parse the mergeinfo string we've created, just to
491             check for bogosity.  If all goes well, we'll unparse it
492             again and use that as our property value.  */
493          err = svn_mergeinfo_parse(&mergeinfo, mergeinfo_buf->data, pool);
494          if (err)
495            {
496              svn_error_clear(err);
497              return SVN_NO_ERROR;
498            }
499          SVN_ERR(svn_mergeinfo_to_string(&new_value, mergeinfo, pool));
500          value = new_value;
501        }
502      name = SVN_PROP_MERGEINFO;
503      eb->svnmerge_migrated = TRUE;
504    }
505
506  /* Remember if we see any svnmerge-blocked properties. */
507  if (eb->migrate_svnmerge && (strcmp(name, "svnmerge-blocked") == 0))
508    {
509      eb->svnmerge_blocked = TRUE;
510    }
511
512  /* Normalize svn:* properties as necessary. */
513  if (svn_prop_needs_translation(name))
514    {
515      svn_boolean_t was_normalized;
516      SVN_ERR(normalize_string(&value, &was_normalized, eb->source_prop_encoding,
517                               pool, pool));
518      if (was_normalized)
519        (*(eb->normalized_node_props_counter))++;
520    }
521
522  return eb->wrapped_editor->change_dir_prop(db->wrapped_node_baton,
523                                             name, value, pool);
524}
525
526static svn_error_t *
527close_edit(void *edit_baton,
528           apr_pool_t *pool)
529{
530  edit_baton_t *eb = edit_baton;
531
532  /* If we haven't opened the root yet, that means we're transfering
533     an empty revision, probably because we aren't allowed to see the
534     contents for some reason.  In any event, we need to open the root
535     and close it again, before we can close out the edit, or the
536     commit will fail. */
537
538  if (! eb->called_open_root)
539    {
540      void *baton;
541      SVN_ERR(eb->wrapped_editor->open_root(eb->wrapped_edit_baton,
542                                            eb->base_revision, pool,
543                                            &baton));
544      SVN_ERR(eb->wrapped_editor->close_directory(baton, pool));
545    }
546
547  if (! eb->quiet)
548    {
549      if (eb->got_textdeltas)
550        SVN_ERR(svn_cmdline_printf(pool, "\n"));
551      if (eb->mergeinfo_stripped)
552        SVN_ERR(svn_cmdline_printf(pool,
553                                   "NOTE: Dropped Subversion mergeinfo "
554                                   "from this revision.\n"));
555      if (eb->svnmerge_migrated)
556        SVN_ERR(svn_cmdline_printf(pool,
557                                   "NOTE: Migrated 'svnmerge-integrated' in "
558                                   "this revision.\n"));
559      if (eb->svnmerge_blocked)
560        SVN_ERR(svn_cmdline_printf(pool,
561                                   "NOTE: Saw 'svnmerge-blocked' in this "
562                                   "revision (but didn't migrate it).\n"));
563    }
564
565  return eb->wrapped_editor->close_edit(eb->wrapped_edit_baton, pool);
566}
567
568static svn_error_t *
569abort_edit(void *edit_baton,
570           apr_pool_t *pool)
571{
572  edit_baton_t *eb = edit_baton;
573  return eb->wrapped_editor->abort_edit(eb->wrapped_edit_baton, pool);
574}
575
576
577/*** Editor factory function ***/
578
579svn_error_t *
580svnsync_get_sync_editor(const svn_delta_editor_t *wrapped_editor,
581                        void *wrapped_edit_baton,
582                        svn_revnum_t base_revision,
583                        const char *to_url,
584                        const char *source_prop_encoding,
585                        svn_boolean_t quiet,
586                        const svn_delta_editor_t **editor,
587                        void **edit_baton,
588                        int *normalized_node_props_counter,
589                        apr_pool_t *pool)
590{
591  svn_delta_editor_t *tree_editor = svn_delta_default_editor(pool);
592  edit_baton_t *eb = apr_pcalloc(pool, sizeof(*eb));
593
594  tree_editor->set_target_revision = set_target_revision;
595  tree_editor->open_root = open_root;
596  tree_editor->delete_entry = delete_entry;
597  tree_editor->add_directory = add_directory;
598  tree_editor->open_directory = open_directory;
599  tree_editor->change_dir_prop = change_dir_prop;
600  tree_editor->close_directory = close_directory;
601  tree_editor->absent_directory = absent_directory;
602  tree_editor->add_file = add_file;
603  tree_editor->open_file = open_file;
604  tree_editor->apply_textdelta = apply_textdelta;
605  tree_editor->change_file_prop = change_file_prop;
606  tree_editor->close_file = close_file;
607  tree_editor->absent_file = absent_file;
608  tree_editor->close_edit = close_edit;
609  tree_editor->abort_edit = abort_edit;
610
611  eb->wrapped_editor = wrapped_editor;
612  eb->wrapped_edit_baton = wrapped_edit_baton;
613  eb->base_revision = base_revision;
614  eb->to_url = to_url;
615  eb->source_prop_encoding = source_prop_encoding;
616  eb->quiet = quiet;
617  eb->normalized_node_props_counter = normalized_node_props_counter;
618
619  if (getenv("SVNSYNC_UNSUPPORTED_STRIP_MERGEINFO"))
620    {
621      eb->strip_mergeinfo = TRUE;
622    }
623  if (getenv("SVNSYNC_UNSUPPORTED_MIGRATE_SVNMERGE"))
624    {
625      /* Current we can't merge property values.  That's only possible
626         if all the properties to be merged were always modified in
627         exactly the same revisions, or if we allow ourselves to
628         lookup the current state of properties in the sync
629         destination.  So for now, migrating svnmerge.py data implies
630         stripping pre-existing svn:mergeinfo. */
631      /* ### FIXME: Do a real migration by consulting the mirror
632         ### repository's HEAD propvalues and merging svn:mergeinfo,
633         ### svnmerge-integrated, and svnmerge-blocked together. */
634      eb->migrate_svnmerge = TRUE;
635      eb->strip_mergeinfo = TRUE;
636    }
637
638  *editor = tree_editor;
639  *edit_baton = eb;
640
641  return SVN_NO_ERROR;
642}
643
644