cache-memcache.c revision 269847 
1/*
2 * cache-memcache.c: memcached caching for Subversion
3 *
4 * ====================================================================
5 *    Licensed to the Apache Software Foundation (ASF) under one
6 *    or more contributor license agreements.  See the NOTICE file
7 *    distributed with this work for additional information
8 *    regarding copyright ownership.  The ASF licenses this file
9 *    to you under the Apache License, Version 2.0 (the
10 *    "License"); you may not use this file except in compliance
11 *    with the License.  You may obtain a copy of the License at
12 *
13 *      http://www.apache.org/licenses/LICENSE-2.0
14 *
15 *    Unless required by applicable law or agreed to in writing,
16 *    software distributed under the License is distributed on an
17 *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 *    KIND, either express or implied.  See the License for the
19 *    specific language governing permissions and limitations
20 *    under the License.
21 * ====================================================================
22 */
23
24#include <apr_md5.h>
25
26#include "svn_pools.h"
27#include "svn_base64.h"
28#include "svn_path.h"
29
30#include "svn_private_config.h"
31#include "private/svn_cache.h"
32#include "private/svn_dep_compat.h"
33
34#include "cache.h"
35
36#ifdef SVN_HAVE_MEMCACHE
37
38#include <apr_memcache.h>
39
40/* A note on thread safety:
41
42   The apr_memcache_t object does its own mutex handling, and nothing
43   else in memcache_t is ever modified, so this implementation should
44   be fully thread-safe.
45*/
46
47/* The (internal) cache object. */
48typedef struct memcache_t {
49  /* The memcached server set we're using. */
50  apr_memcache_t *memcache;
51
52  /* A prefix used to differentiate our data from any other data in
53   * the memcached (URI-encoded). */
54  const char *prefix;
55
56  /* The size of the key: either a fixed number of bytes or
57   * APR_HASH_KEY_STRING. */
58  apr_ssize_t klen;
59
60
61  /* Used to marshal values in and out of the cache. */
62  svn_cache__serialize_func_t serialize_func;
63  svn_cache__deserialize_func_t deserialize_func;
64} memcache_t;
65
66/* The wrapper around apr_memcache_t. */
67struct svn_memcache_t {
68  apr_memcache_t *c;
69};
70
71
72/* The memcached protocol says the maximum key length is 250.  Let's
73   just say 249, to be safe. */
74#define MAX_MEMCACHED_KEY_LEN 249
75#define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \
76                                    2 * APR_MD5_DIGESTSIZE)
77
78
79/* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated
80   in POOL. */
81static svn_error_t *
82build_key(const char **mc_key,
83          memcache_t *cache,
84          const void *raw_key,
85          apr_pool_t *pool)
86{
87  const char *encoded_suffix;
88  const char *long_key;
89  apr_size_t long_key_len;
90
91  if (cache->klen == APR_HASH_KEY_STRING)
92    encoded_suffix = svn_path_uri_encode(raw_key, pool);
93  else
94    {
95      const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool);
96      const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE,
97                                                              pool);
98      encoded_suffix = encoded->data;
99    }
100
101  long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix,
102                         (char *)NULL);
103  long_key_len = strlen(long_key);
104
105  /* We don't want to have a key that's too big.  If it was going to
106     be too big, we MD5 the entire string, then replace the last bit
107     with the checksum.  Note that APR_MD5_DIGESTSIZE is for the pure
108     binary digest; we have to double that when we convert to hex.
109
110     Every key we use will either be at most
111     MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly
112     MAX_MEMCACHED_KEY_LEN bytes long. */
113  if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN)
114    {
115      svn_checksum_t *checksum;
116      SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len,
117                           pool));
118
119      long_key = apr_pstrcat(pool,
120                             apr_pstrmemdup(pool, long_key,
121                                            MEMCACHED_KEY_UNHASHED_LEN),
122                             svn_checksum_to_cstring_display(checksum, pool),
123                             (char *)NULL);
124    }
125
126  *mc_key = long_key;
127  return SVN_NO_ERROR;
128}
129
130/* Core functionality of our getter functions: fetch DATA from the memcached
131 * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and
132 * use a tempoary sub-pool of POOL for allocations.
133 */
134static svn_error_t *
135memcache_internal_get(char **data,
136                      apr_size_t *size,
137                      svn_boolean_t *found,
138                      void *cache_void,
139                      const void *key,
140                      apr_pool_t *pool)
141{
142  memcache_t *cache = cache_void;
143  apr_status_t apr_err;
144  const char *mc_key;
145  apr_pool_t *subpool;
146
147  if (key == NULL)
148    {
149      *found = FALSE;
150      return SVN_NO_ERROR;
151    }
152
153  subpool = svn_pool_create(pool);
154  SVN_ERR(build_key(&mc_key, cache, key, subpool));
155
156  apr_err = apr_memcache_getp(cache->memcache,
157                              pool,
158                              mc_key,
159                              data,
160                              size,
161                              NULL /* ignore flags */);
162  if (apr_err == APR_NOTFOUND)
163    {
164      *found = FALSE;
165      svn_pool_destroy(subpool);
166      return SVN_NO_ERROR;
167    }
168  else if (apr_err != APR_SUCCESS || !*data)
169    return svn_error_wrap_apr(apr_err,
170                              _("Unknown memcached error while reading"));
171
172  *found = TRUE;
173
174  svn_pool_destroy(subpool);
175  return SVN_NO_ERROR;
176}
177
178
179static svn_error_t *
180memcache_get(void **value_p,
181             svn_boolean_t *found,
182             void *cache_void,
183             const void *key,
184             apr_pool_t *result_pool)
185{
186  memcache_t *cache = cache_void;
187  char *data;
188  apr_size_t data_len;
189  SVN_ERR(memcache_internal_get(&data,
190                                &data_len,
191                                found,
192                                cache_void,
193                                key,
194                                result_pool));
195
196  /* If we found it, de-serialize it. */
197  if (*found)
198    {
199      if (cache->deserialize_func)
200        {
201          SVN_ERR((cache->deserialize_func)(value_p, data, data_len,
202                                            result_pool));
203        }
204      else
205        {
206          svn_stringbuf_t *value = svn_stringbuf_create_empty(result_pool);
207          value->data = data;
208          value->blocksize = data_len;
209          value->len = data_len - 1; /* account for trailing NUL */
210          *value_p = value;
211        }
212    }
213
214  return SVN_NO_ERROR;
215}
216
217/* Core functionality of our setter functions: store LENGH bytes of DATA
218 * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL
219 * for temporary allocations.
220 */
221static svn_error_t *
222memcache_internal_set(void *cache_void,
223                      const void *key,
224                      const char *data,
225                      apr_size_t len,
226                      apr_pool_t *scratch_pool)
227{
228  memcache_t *cache = cache_void;
229  const char *mc_key;
230  apr_status_t apr_err;
231
232  SVN_ERR(build_key(&mc_key, cache, key, scratch_pool));
233  apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0);
234
235  /* ### Maybe write failures should be ignored (but logged)? */
236  if (apr_err != APR_SUCCESS)
237    return svn_error_wrap_apr(apr_err,
238                              _("Unknown memcached error while writing"));
239
240  return SVN_NO_ERROR;
241}
242
243
244static svn_error_t *
245memcache_set(void *cache_void,
246             const void *key,
247             void *value,
248             apr_pool_t *scratch_pool)
249{
250  memcache_t *cache = cache_void;
251  apr_pool_t *subpool = svn_pool_create(scratch_pool);
252  void *data;
253  apr_size_t data_len;
254  svn_error_t *err;
255
256  if (key == NULL)
257    return SVN_NO_ERROR;
258
259  if (cache->serialize_func)
260    {
261      SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool));
262    }
263  else
264    {
265      svn_stringbuf_t *value_str = value;
266      data = value_str->data;
267      data_len = value_str->len + 1; /* copy trailing NUL */
268    }
269
270  err = memcache_internal_set(cache_void, key, data, data_len, subpool);
271
272  svn_pool_destroy(subpool);
273  return err;
274}
275
276static svn_error_t *
277memcache_get_partial(void **value_p,
278                     svn_boolean_t *found,
279                     void *cache_void,
280                     const void *key,
281                     svn_cache__partial_getter_func_t func,
282                     void *baton,
283                     apr_pool_t *result_pool)
284{
285  svn_error_t *err = SVN_NO_ERROR;
286
287  char *data;
288  apr_size_t size;
289  SVN_ERR(memcache_internal_get(&data,
290                                &size,
291                                found,
292                                cache_void,
293                                key,
294                                result_pool));
295
296  /* If we found it, de-serialize it. */
297  return *found
298    ? func(value_p, data, size, baton, result_pool)
299    : err;
300}
301
302
303static svn_error_t *
304memcache_set_partial(void *cache_void,
305                     const void *key,
306                     svn_cache__partial_setter_func_t func,
307                     void *baton,
308                     apr_pool_t *scratch_pool)
309{
310  svn_error_t *err = SVN_NO_ERROR;
311
312  void *data;
313  apr_size_t size;
314  svn_boolean_t found = FALSE;
315
316  apr_pool_t *subpool = svn_pool_create(scratch_pool);
317  SVN_ERR(memcache_internal_get((char **)&data,
318                                &size,
319                                &found,
320                                cache_void,
321                                key,
322                                subpool));
323
324  /* If we found it, modify it and write it back to cache */
325  if (found)
326    {
327      SVN_ERR(func(&data, &size, baton, subpool));
328      err = memcache_internal_set(cache_void, key, data, size, subpool);
329    }
330
331  svn_pool_destroy(subpool);
332  return err;
333}
334
335
336static svn_error_t *
337memcache_iter(svn_boolean_t *completed,
338              void *cache_void,
339              svn_iter_apr_hash_cb_t user_cb,
340              void *user_baton,
341              apr_pool_t *scratch_pool)
342{
343  return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
344                          _("Can't iterate a memcached cache"));
345}
346
347static svn_boolean_t
348memcache_is_cachable(void *unused, apr_size_t size)
349{
350  (void)unused;  /* silence gcc warning. */
351
352  /* The memcached cutoff seems to be a bit (header length?) under a megabyte.
353   * We round down a little to be safe.
354   */
355  return size < 1000000;
356}
357
358static svn_error_t *
359memcache_get_info(void *cache_void,
360                  svn_cache__info_t *info,
361                  svn_boolean_t reset,
362                  apr_pool_t *result_pool)
363{
364  memcache_t *cache = cache_void;
365
366  info->id = apr_pstrdup(result_pool, cache->prefix);
367
368  /* we don't have any memory allocation info */
369
370  info->used_size = 0;
371  info->total_size = 0;
372  info->data_size = 0;
373  info->used_entries = 0;
374  info->total_entries = 0;
375
376  return SVN_NO_ERROR;
377}
378
379static svn_cache__vtable_t memcache_vtable = {
380  memcache_get,
381  memcache_set,
382  memcache_iter,
383  memcache_is_cachable,
384  memcache_get_partial,
385  memcache_set_partial,
386  memcache_get_info
387};
388
389svn_error_t *
390svn_cache__create_memcache(svn_cache__t **cache_p,
391                          svn_memcache_t *memcache,
392                          svn_cache__serialize_func_t serialize_func,
393                          svn_cache__deserialize_func_t deserialize_func,
394                          apr_ssize_t klen,
395                          const char *prefix,
396                          apr_pool_t *pool)
397{
398  svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper));
399  memcache_t *cache = apr_pcalloc(pool, sizeof(*cache));
400
401  cache->serialize_func = serialize_func;
402  cache->deserialize_func = deserialize_func;
403  cache->klen = klen;
404  cache->prefix = svn_path_uri_encode(prefix, pool);
405  cache->memcache = memcache->c;
406
407  wrapper->vtable = &memcache_vtable;
408  wrapper->cache_internal = cache;
409  wrapper->error_handler = 0;
410  wrapper->error_baton = 0;
411
412  *cache_p = wrapper;
413  return SVN_NO_ERROR;
414}
415
416
417/*** Creating apr_memcache_t from svn_config_t. ***/
418
419/* Baton for add_memcache_server. */
420struct ams_baton {
421  apr_memcache_t *memcache;
422  apr_pool_t *memcache_pool;
423  svn_error_t *err;
424};
425
426/* Implements svn_config_enumerator2_t. */
427static svn_boolean_t
428add_memcache_server(const char *name,
429                    const char *value,
430                    void *baton,
431                    apr_pool_t *pool)
432{
433  struct ams_baton *b = baton;
434  char *host, *scope;
435  apr_port_t port;
436  apr_status_t apr_err;
437  apr_memcache_server_t *server;
438
439  apr_err = apr_parse_addr_port(&host, &scope, &port,
440                                value, pool);
441  if (apr_err != APR_SUCCESS)
442    {
443      b->err = svn_error_wrap_apr(apr_err,
444                                  _("Error parsing memcache server '%s'"),
445                                  name);
446      return FALSE;
447    }
448
449  if (scope)
450    {
451      b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
452                                  _("Scope not allowed in memcache server "
453                                    "'%s'"),
454                                  name);
455      return FALSE;
456    }
457  if (!host || !port)
458    {
459      b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
460                                  _("Must specify host and port for memcache "
461                                    "server '%s'"),
462                                  name);
463      return FALSE;
464    }
465
466  /* Note: the four numbers here are only relevant when an
467     apr_memcache_t is being shared by multiple threads. */
468  apr_err = apr_memcache_server_create(b->memcache_pool,
469                                       host,
470                                       port,
471                                       0,  /* min connections */
472                                       5,  /* soft max connections */
473                                       10, /* hard max connections */
474                                       /*  time to live (in microseconds) */
475                                       apr_time_from_sec(50),
476                                       &server);
477  if (apr_err != APR_SUCCESS)
478    {
479      b->err = svn_error_wrap_apr(apr_err,
480                                  _("Unknown error creating memcache server"));
481      return FALSE;
482    }
483
484  apr_err = apr_memcache_add_server(b->memcache, server);
485  if (apr_err != APR_SUCCESS)
486    {
487      b->err = svn_error_wrap_apr(apr_err,
488                                  _("Unknown error adding server to memcache"));
489      return FALSE;
490    }
491
492  return TRUE;
493}
494
495#else /* ! SVN_HAVE_MEMCACHE */
496
497/* Stubs for no apr memcache library. */
498
499struct svn_memcache_t {
500  void *unused; /* Let's not have a size-zero struct. */
501};
502
503svn_error_t *
504svn_cache__create_memcache(svn_cache__t **cache_p,
505                          svn_memcache_t *memcache,
506                          svn_cache__serialize_func_t serialize_func,
507                          svn_cache__deserialize_func_t deserialize_func,
508                          apr_ssize_t klen,
509                          const char *prefix,
510                          apr_pool_t *pool)
511{
512  return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
513}
514
515#endif /* SVN_HAVE_MEMCACHE */
516
517/* Implements svn_config_enumerator2_t.  Just used for the
518   entry-counting return value of svn_config_enumerate2. */
519static svn_boolean_t
520nop_enumerator(const char *name,
521               const char *value,
522               void *baton,
523               apr_pool_t *pool)
524{
525  return TRUE;
526}
527
528svn_error_t *
529svn_cache__make_memcache_from_config(svn_memcache_t **memcache_p,
530                                    svn_config_t *config,
531                                    apr_pool_t *pool)
532{
533  int server_count;
534  apr_pool_t *subpool = svn_pool_create(pool);
535
536  server_count =
537    svn_config_enumerate2(config,
538                          SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
539                          nop_enumerator, NULL, subpool);
540
541  if (server_count == 0)
542    {
543      *memcache_p = NULL;
544      svn_pool_destroy(subpool);
545      return SVN_NO_ERROR;
546    }
547
548  if (server_count > APR_INT16_MAX)
549    return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL);
550
551#ifdef SVN_HAVE_MEMCACHE
552  {
553    struct ams_baton b;
554    svn_memcache_t *memcache = apr_pcalloc(pool, sizeof(*memcache));
555    apr_status_t apr_err = apr_memcache_create(pool,
556                                               (apr_uint16_t)server_count,
557                                               0, /* flags */
558                                               &(memcache->c));
559    if (apr_err != APR_SUCCESS)
560      return svn_error_wrap_apr(apr_err,
561                                _("Unknown error creating apr_memcache_t"));
562
563    b.memcache = memcache->c;
564    b.memcache_pool = pool;
565    b.err = SVN_NO_ERROR;
566    svn_config_enumerate2(config,
567                          SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
568                          add_memcache_server, &b,
569                          subpool);
570
571    if (b.err)
572      return b.err;
573
574    *memcache_p = memcache;
575
576    svn_pool_destroy(subpool);
577    return SVN_NO_ERROR;
578  }
579#else /* ! SVN_HAVE_MEMCACHE */
580  {
581    return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
582  }
583#endif /* SVN_HAVE_MEMCACHE */
584}
585