cache-memcache.c revision 299742 
1/*
2 * cache-memcache.c: memcached caching for Subversion
3 *
4 * ====================================================================
5 *    Licensed to the Apache Software Foundation (ASF) under one
6 *    or more contributor license agreements.  See the NOTICE file
7 *    distributed with this work for additional information
8 *    regarding copyright ownership.  The ASF licenses this file
9 *    to you under the Apache License, Version 2.0 (the
10 *    "License"); you may not use this file except in compliance
11 *    with the License.  You may obtain a copy of the License at
12 *
13 *      http://www.apache.org/licenses/LICENSE-2.0
14 *
15 *    Unless required by applicable law or agreed to in writing,
16 *    software distributed under the License is distributed on an
17 *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 *    KIND, either express or implied.  See the License for the
19 *    specific language governing permissions and limitations
20 *    under the License.
21 * ====================================================================
22 */
23
24#include <apr_md5.h>
25
26#include "svn_pools.h"
27#include "svn_base64.h"
28#include "svn_path.h"
29
30#include "svn_private_config.h"
31#include "private/svn_cache.h"
32#include "private/svn_dep_compat.h"
33
34#include "cache.h"
35
36#ifdef SVN_HAVE_MEMCACHE
37
38#include <apr_memcache.h>
39
40/* A note on thread safety:
41
42   The apr_memcache_t object does its own mutex handling, and nothing
43   else in memcache_t is ever modified, so this implementation should
44   be fully thread-safe.
45*/
46
47/* The (internal) cache object. */
48typedef struct memcache_t {
49  /* The memcached server set we're using. */
50  apr_memcache_t *memcache;
51
52  /* A prefix used to differentiate our data from any other data in
53   * the memcached (URI-encoded). */
54  const char *prefix;
55
56  /* The size of the key: either a fixed number of bytes or
57   * APR_HASH_KEY_STRING. */
58  apr_ssize_t klen;
59
60
61  /* Used to marshal values in and out of the cache. */
62  svn_cache__serialize_func_t serialize_func;
63  svn_cache__deserialize_func_t deserialize_func;
64} memcache_t;
65
66/* The wrapper around apr_memcache_t. */
67struct svn_memcache_t {
68  apr_memcache_t *c;
69};
70
71
72/* The memcached protocol says the maximum key length is 250.  Let's
73   just say 249, to be safe. */
74#define MAX_MEMCACHED_KEY_LEN 249
75#define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \
76                                    2 * APR_MD5_DIGESTSIZE)
77
78
79/* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated
80   in POOL. */
81static svn_error_t *
82build_key(const char **mc_key,
83          memcache_t *cache,
84          const void *raw_key,
85          apr_pool_t *pool)
86{
87  const char *encoded_suffix;
88  const char *long_key;
89  apr_size_t long_key_len;
90
91  if (cache->klen == APR_HASH_KEY_STRING)
92    encoded_suffix = svn_path_uri_encode(raw_key, pool);
93  else
94    {
95      const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool);
96      const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE,
97                                                              pool);
98      encoded_suffix = encoded->data;
99    }
100
101  long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix,
102                         SVN_VA_NULL);
103  long_key_len = strlen(long_key);
104
105  /* We don't want to have a key that's too big.  If it was going to
106     be too big, we MD5 the entire string, then replace the last bit
107     with the checksum.  Note that APR_MD5_DIGESTSIZE is for the pure
108     binary digest; we have to double that when we convert to hex.
109
110     Every key we use will either be at most
111     MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly
112     MAX_MEMCACHED_KEY_LEN bytes long. */
113  if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN)
114    {
115      svn_checksum_t *checksum;
116      SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len,
117                           pool));
118
119      long_key = apr_pstrcat(pool,
120                             apr_pstrmemdup(pool, long_key,
121                                            MEMCACHED_KEY_UNHASHED_LEN),
122                             svn_checksum_to_cstring_display(checksum, pool),
123                             SVN_VA_NULL);
124    }
125
126  *mc_key = long_key;
127  return SVN_NO_ERROR;
128}
129
130/* Core functionality of our getter functions: fetch DATA from the memcached
131 * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and
132 * use a tempoary sub-pool of POOL for allocations.
133 */
134static svn_error_t *
135memcache_internal_get(char **data,
136                      apr_size_t *size,
137                      svn_boolean_t *found,
138                      void *cache_void,
139                      const void *key,
140                      apr_pool_t *pool)
141{
142  memcache_t *cache = cache_void;
143  apr_status_t apr_err;
144  const char *mc_key;
145  apr_pool_t *subpool;
146
147  if (key == NULL)
148    {
149      *found = FALSE;
150      return SVN_NO_ERROR;
151    }
152
153  subpool = svn_pool_create(pool);
154  SVN_ERR(build_key(&mc_key, cache, key, subpool));
155
156  apr_err = apr_memcache_getp(cache->memcache,
157                              pool,
158                              mc_key,
159                              data,
160                              size,
161                              NULL /* ignore flags */);
162  if (apr_err == APR_NOTFOUND)
163    {
164      *found = FALSE;
165      svn_pool_destroy(subpool);
166      return SVN_NO_ERROR;
167    }
168  else if (apr_err != APR_SUCCESS || !*data)
169    return svn_error_wrap_apr(apr_err,
170                              _("Unknown memcached error while reading"));
171
172  *found = TRUE;
173
174  svn_pool_destroy(subpool);
175  return SVN_NO_ERROR;
176}
177
178
179static svn_error_t *
180memcache_get(void **value_p,
181             svn_boolean_t *found,
182             void *cache_void,
183             const void *key,
184             apr_pool_t *result_pool)
185{
186  memcache_t *cache = cache_void;
187  char *data;
188  apr_size_t data_len;
189  SVN_ERR(memcache_internal_get(&data,
190                                &data_len,
191                                found,
192                                cache_void,
193                                key,
194                                result_pool));
195
196  /* If we found it, de-serialize it. */
197  if (*found)
198    {
199      if (cache->deserialize_func)
200        {
201          SVN_ERR((cache->deserialize_func)(value_p, data, data_len,
202                                            result_pool));
203        }
204      else
205        {
206          svn_stringbuf_t *value = svn_stringbuf_create_empty(result_pool);
207          value->data = data;
208          value->blocksize = data_len;
209          value->len = data_len - 1; /* account for trailing NUL */
210          *value_p = value;
211        }
212    }
213
214  return SVN_NO_ERROR;
215}
216
217/* Implement vtable.has_key in terms of the getter.
218 */
219static svn_error_t *
220memcache_has_key(svn_boolean_t *found,
221                 void *cache_void,
222                 const void *key,
223                 apr_pool_t *scratch_pool)
224{
225  char *data;
226  apr_size_t data_len;
227  SVN_ERR(memcache_internal_get(&data,
228                                &data_len,
229                                found,
230                                cache_void,
231                                key,
232                                scratch_pool));
233
234  return SVN_NO_ERROR;
235}
236
237/* Core functionality of our setter functions: store LENGH bytes of DATA
238 * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL
239 * for temporary allocations.
240 */
241static svn_error_t *
242memcache_internal_set(void *cache_void,
243                      const void *key,
244                      const char *data,
245                      apr_size_t len,
246                      apr_pool_t *scratch_pool)
247{
248  memcache_t *cache = cache_void;
249  const char *mc_key;
250  apr_status_t apr_err;
251
252  SVN_ERR(build_key(&mc_key, cache, key, scratch_pool));
253  apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0);
254
255  /* ### Maybe write failures should be ignored (but logged)? */
256  if (apr_err != APR_SUCCESS)
257    return svn_error_wrap_apr(apr_err,
258                              _("Unknown memcached error while writing"));
259
260  return SVN_NO_ERROR;
261}
262
263
264static svn_error_t *
265memcache_set(void *cache_void,
266             const void *key,
267             void *value,
268             apr_pool_t *scratch_pool)
269{
270  memcache_t *cache = cache_void;
271  apr_pool_t *subpool = svn_pool_create(scratch_pool);
272  void *data;
273  apr_size_t data_len;
274  svn_error_t *err;
275
276  if (key == NULL)
277    return SVN_NO_ERROR;
278
279  if (cache->serialize_func)
280    {
281      SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool));
282    }
283  else
284    {
285      svn_stringbuf_t *value_str = value;
286      data = value_str->data;
287      data_len = value_str->len + 1; /* copy trailing NUL */
288    }
289
290  err = memcache_internal_set(cache_void, key, data, data_len, subpool);
291
292  svn_pool_destroy(subpool);
293  return err;
294}
295
296static svn_error_t *
297memcache_get_partial(void **value_p,
298                     svn_boolean_t *found,
299                     void *cache_void,
300                     const void *key,
301                     svn_cache__partial_getter_func_t func,
302                     void *baton,
303                     apr_pool_t *result_pool)
304{
305  svn_error_t *err = SVN_NO_ERROR;
306
307  char *data;
308  apr_size_t size;
309  SVN_ERR(memcache_internal_get(&data,
310                                &size,
311                                found,
312                                cache_void,
313                                key,
314                                result_pool));
315
316  /* If we found it, de-serialize it. */
317  return *found
318    ? func(value_p, data, size, baton, result_pool)
319    : err;
320}
321
322
323static svn_error_t *
324memcache_set_partial(void *cache_void,
325                     const void *key,
326                     svn_cache__partial_setter_func_t func,
327                     void *baton,
328                     apr_pool_t *scratch_pool)
329{
330  svn_error_t *err = SVN_NO_ERROR;
331
332  void *data;
333  apr_size_t size;
334  svn_boolean_t found = FALSE;
335
336  apr_pool_t *subpool = svn_pool_create(scratch_pool);
337  SVN_ERR(memcache_internal_get((char **)&data,
338                                &size,
339                                &found,
340                                cache_void,
341                                key,
342                                subpool));
343
344  /* If we found it, modify it and write it back to cache */
345  if (found)
346    {
347      SVN_ERR(func(&data, &size, baton, subpool));
348      err = memcache_internal_set(cache_void, key, data, size, subpool);
349    }
350
351  svn_pool_destroy(subpool);
352  return err;
353}
354
355
356static svn_error_t *
357memcache_iter(svn_boolean_t *completed,
358              void *cache_void,
359              svn_iter_apr_hash_cb_t user_cb,
360              void *user_baton,
361              apr_pool_t *scratch_pool)
362{
363  return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
364                          _("Can't iterate a memcached cache"));
365}
366
367static svn_boolean_t
368memcache_is_cachable(void *unused, apr_size_t size)
369{
370  SVN_UNUSED(unused);
371
372  /* The memcached cutoff seems to be a bit (header length?) under a megabyte.
373   * We round down a little to be safe.
374   */
375  return size < 1000000;
376}
377
378static svn_error_t *
379memcache_get_info(void *cache_void,
380                  svn_cache__info_t *info,
381                  svn_boolean_t reset,
382                  apr_pool_t *result_pool)
383{
384  memcache_t *cache = cache_void;
385
386  info->id = apr_pstrdup(result_pool, cache->prefix);
387
388  /* we don't have any memory allocation info */
389
390  return SVN_NO_ERROR;
391}
392
393static svn_cache__vtable_t memcache_vtable = {
394  memcache_get,
395  memcache_has_key,
396  memcache_set,
397  memcache_iter,
398  memcache_is_cachable,
399  memcache_get_partial,
400  memcache_set_partial,
401  memcache_get_info
402};
403
404svn_error_t *
405svn_cache__create_memcache(svn_cache__t **cache_p,
406                          svn_memcache_t *memcache,
407                          svn_cache__serialize_func_t serialize_func,
408                          svn_cache__deserialize_func_t deserialize_func,
409                          apr_ssize_t klen,
410                          const char *prefix,
411                          apr_pool_t *pool)
412{
413  svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper));
414  memcache_t *cache = apr_pcalloc(pool, sizeof(*cache));
415
416  cache->serialize_func = serialize_func;
417  cache->deserialize_func = deserialize_func;
418  cache->klen = klen;
419  cache->prefix = svn_path_uri_encode(prefix, pool);
420  cache->memcache = memcache->c;
421
422  wrapper->vtable = &memcache_vtable;
423  wrapper->cache_internal = cache;
424  wrapper->error_handler = 0;
425  wrapper->error_baton = 0;
426  wrapper->pretend_empty = !!getenv("SVN_X_DOES_NOT_MARK_THE_SPOT");
427
428  *cache_p = wrapper;
429  return SVN_NO_ERROR;
430}
431
432
433/*** Creating apr_memcache_t from svn_config_t. ***/
434
435/* Baton for add_memcache_server. */
436struct ams_baton {
437  apr_memcache_t *memcache;
438  apr_pool_t *memcache_pool;
439  svn_error_t *err;
440};
441
442/* Implements svn_config_enumerator2_t. */
443static svn_boolean_t
444add_memcache_server(const char *name,
445                    const char *value,
446                    void *baton,
447                    apr_pool_t *pool)
448{
449  struct ams_baton *b = baton;
450  char *host, *scope;
451  apr_port_t port;
452  apr_status_t apr_err;
453  apr_memcache_server_t *server;
454
455  apr_err = apr_parse_addr_port(&host, &scope, &port,
456                                value, pool);
457  if (apr_err != APR_SUCCESS)
458    {
459      b->err = svn_error_wrap_apr(apr_err,
460                                  _("Error parsing memcache server '%s'"),
461                                  name);
462      return FALSE;
463    }
464
465  if (scope)
466    {
467      b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
468                                  _("Scope not allowed in memcache server "
469                                    "'%s'"),
470                                  name);
471      return FALSE;
472    }
473  if (!host || !port)
474    {
475      b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
476                                  _("Must specify host and port for memcache "
477                                    "server '%s'"),
478                                  name);
479      return FALSE;
480    }
481
482  /* Note: the four numbers here are only relevant when an
483     apr_memcache_t is being shared by multiple threads. */
484  apr_err = apr_memcache_server_create(b->memcache_pool,
485                                       host,
486                                       port,
487                                       0,  /* min connections */
488                                       5,  /* soft max connections */
489                                       10, /* hard max connections */
490                                       /*  time to live (in microseconds) */
491                                       apr_time_from_sec(50),
492                                       &server);
493  if (apr_err != APR_SUCCESS)
494    {
495      b->err = svn_error_wrap_apr(apr_err,
496                                  _("Unknown error creating memcache server"));
497      return FALSE;
498    }
499
500  apr_err = apr_memcache_add_server(b->memcache, server);
501  if (apr_err != APR_SUCCESS)
502    {
503      b->err = svn_error_wrap_apr(apr_err,
504                                  _("Unknown error adding server to memcache"));
505      return FALSE;
506    }
507
508  return TRUE;
509}
510
511#else /* ! SVN_HAVE_MEMCACHE */
512
513/* Stubs for no apr memcache library. */
514
515struct svn_memcache_t {
516  void *unused; /* Let's not have a size-zero struct. */
517};
518
519svn_error_t *
520svn_cache__create_memcache(svn_cache__t **cache_p,
521                          svn_memcache_t *memcache,
522                          svn_cache__serialize_func_t serialize_func,
523                          svn_cache__deserialize_func_t deserialize_func,
524                          apr_ssize_t klen,
525                          const char *prefix,
526                          apr_pool_t *pool)
527{
528  return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
529}
530
531#endif /* SVN_HAVE_MEMCACHE */
532
533/* Implements svn_config_enumerator2_t.  Just used for the
534   entry-counting return value of svn_config_enumerate2. */
535static svn_boolean_t
536nop_enumerator(const char *name,
537               const char *value,
538               void *baton,
539               apr_pool_t *pool)
540{
541  return TRUE;
542}
543
544svn_error_t *
545svn_cache__make_memcache_from_config(svn_memcache_t **memcache_p,
546                                    svn_config_t *config,
547                                    apr_pool_t *result_pool,
548                                    apr_pool_t *scratch_pool)
549{
550  int server_count =
551    svn_config_enumerate2(config,
552                          SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
553                          nop_enumerator, NULL, scratch_pool);
554
555  if (server_count == 0)
556    {
557      *memcache_p = NULL;
558      return SVN_NO_ERROR;
559    }
560
561  if (server_count > APR_INT16_MAX)
562    return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL);
563
564#ifdef SVN_HAVE_MEMCACHE
565  {
566    struct ams_baton b;
567    svn_memcache_t *memcache = apr_pcalloc(result_pool, sizeof(*memcache));
568    apr_status_t apr_err = apr_memcache_create(result_pool,
569                                               (apr_uint16_t)server_count,
570                                               0, /* flags */
571                                               &(memcache->c));
572    if (apr_err != APR_SUCCESS)
573      return svn_error_wrap_apr(apr_err,
574                                _("Unknown error creating apr_memcache_t"));
575
576    b.memcache = memcache->c;
577    b.memcache_pool = result_pool;
578    b.err = SVN_NO_ERROR;
579    svn_config_enumerate2(config,
580                          SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
581                          add_memcache_server, &b,
582                          scratch_pool);
583
584    if (b.err)
585      return b.err;
586
587    *memcache_p = memcache;
588
589    return SVN_NO_ERROR;
590  }
591#else /* ! SVN_HAVE_MEMCACHE */
592  {
593    return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
594  }
595#endif /* SVN_HAVE_MEMCACHE */
596}
597