1/*
2 * cache-memcache.c: memcached caching for Subversion
3 *
4 * ====================================================================
5 *    Licensed to the Apache Software Foundation (ASF) under one
6 *    or more contributor license agreements.  See the NOTICE file
7 *    distributed with this work for additional information
8 *    regarding copyright ownership.  The ASF licenses this file
9 *    to you under the Apache License, Version 2.0 (the
10 *    "License"); you may not use this file except in compliance
11 *    with the License.  You may obtain a copy of the License at
12 *
13 *      http://www.apache.org/licenses/LICENSE-2.0
14 *
15 *    Unless required by applicable law or agreed to in writing,
16 *    software distributed under the License is distributed on an
17 *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
18 *    KIND, either express or implied.  See the License for the
19 *    specific language governing permissions and limitations
20 *    under the License.
21 * ====================================================================
22 */
23
24#include <apr_md5.h>
25
26#include "svn_pools.h"
27#include "svn_base64.h"
28#include "svn_path.h"
29
30#include "svn_private_config.h"
31#include "private/svn_cache.h"
32#include "private/svn_dep_compat.h"
33
34#include "cache.h"
35
36#ifdef SVN_HAVE_MEMCACHE
37
38#include <apr_memcache.h>
39
40/* A note on thread safety:
41
42   The apr_memcache_t object does its own mutex handling, and nothing
43   else in memcache_t is ever modified, so this implementation should
44   be fully thread-safe.
45*/
46
47/* The (internal) cache object. */
48typedef struct memcache_t {
49  /* The memcached server set we're using. */
50  apr_memcache_t *memcache;
51
52  /* A prefix used to differentiate our data from any other data in
53   * the memcached (URI-encoded). */
54  const char *prefix;
55
56  /* The size of the key: either a fixed number of bytes or
57   * APR_HASH_KEY_STRING. */
58  apr_ssize_t klen;
59
60
61  /* Used to marshal values in and out of the cache. */
62  svn_cache__serialize_func_t serialize_func;
63  svn_cache__deserialize_func_t deserialize_func;
64} memcache_t;
65
66/* The wrapper around apr_memcache_t. */
67struct svn_memcache_t {
68  apr_memcache_t *c;
69};
70
71
72/* The memcached protocol says the maximum key length is 250.  Let's
73   just say 249, to be safe. */
74#define MAX_MEMCACHED_KEY_LEN 249
75#define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \
76                                    2 * APR_MD5_DIGESTSIZE)
77
78
79/* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated
80   in POOL. */
81static svn_error_t *
82build_key(const char **mc_key,
83          memcache_t *cache,
84          const void *raw_key,
85          apr_pool_t *pool)
86{
87  const char *encoded_suffix;
88  const char *long_key;
89  apr_size_t long_key_len;
90
91  if (cache->klen == APR_HASH_KEY_STRING)
92    encoded_suffix = svn_path_uri_encode(raw_key, pool);
93  else
94    {
95      const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool);
96      const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE,
97                                                              pool);
98      encoded_suffix = encoded->data;
99    }
100
101  long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix,
102                         (char *)NULL);
103  long_key_len = strlen(long_key);
104
105  /* We don't want to have a key that's too big.  If it was going to
106     be too big, we MD5 the entire string, then replace the last bit
107     with the checksum.  Note that APR_MD5_DIGESTSIZE is for the pure
108     binary digest; we have to double that when we convert to hex.
109
110     Every key we use will either be at most
111     MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly
112     MAX_MEMCACHED_KEY_LEN bytes long. */
113  if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN)
114    {
115      svn_checksum_t *checksum;
116      SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len,
117                           pool));
118
119      long_key = apr_pstrcat(pool,
120                             apr_pstrmemdup(pool, long_key,
121                                            MEMCACHED_KEY_UNHASHED_LEN),
122                             svn_checksum_to_cstring_display(checksum, pool),
123                             (char *)NULL);
124    }
125
126  *mc_key = long_key;
127  return SVN_NO_ERROR;
128}
129
130/* Core functionality of our getter functions: fetch DATA from the memcached
131 * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and
132 * use a tempoary sub-pool of POOL for allocations.
133 */
134static svn_error_t *
135memcache_internal_get(char **data,
136                      apr_size_t *size,
137                      svn_boolean_t *found,
138                      void *cache_void,
139                      const void *key,
140                      apr_pool_t *pool)
141{
142  memcache_t *cache = cache_void;
143  apr_status_t apr_err;
144  const char *mc_key;
145  apr_pool_t *subpool;
146
147  if (key == NULL)
148    {
149      *found = FALSE;
150      return SVN_NO_ERROR;
151    }
152
153  subpool = svn_pool_create(pool);
154  SVN_ERR(build_key(&mc_key, cache, key, subpool));
155
156  apr_err = apr_memcache_getp(cache->memcache,
157                              pool,
158                              mc_key,
159                              data,
160                              size,
161                              NULL /* ignore flags */);
162  if (apr_err == APR_NOTFOUND)
163    {
164      *found = FALSE;
165      svn_pool_destroy(subpool);
166      return SVN_NO_ERROR;
167    }
168  else if (apr_err != APR_SUCCESS || !*data)
169    return svn_error_wrap_apr(apr_err,
170                              _("Unknown memcached error while reading"));
171
172  *found = TRUE;
173
174  svn_pool_destroy(subpool);
175  return SVN_NO_ERROR;
176}
177
178
179static svn_error_t *
180memcache_get(void **value_p,
181             svn_boolean_t *found,
182             void *cache_void,
183             const void *key,
184             apr_pool_t *result_pool)
185{
186  memcache_t *cache = cache_void;
187  char *data;
188  apr_size_t data_len;
189  SVN_ERR(memcache_internal_get(&data,
190                                &data_len,
191                                found,
192                                cache_void,
193                                key,
194                                result_pool));
195
196  /* If we found it, de-serialize it. */
197  if (*found)
198    {
199      if (cache->deserialize_func)
200        {
201          SVN_ERR((cache->deserialize_func)(value_p, data, data_len,
202                                            result_pool));
203        }
204      else
205        {
206          svn_string_t *value = apr_pcalloc(result_pool, sizeof(*value));
207          value->data = data;
208          value->len = data_len;
209          *value_p = value;
210        }
211    }
212
213  return SVN_NO_ERROR;
214}
215
216/* Core functionality of our setter functions: store LENGH bytes of DATA
217 * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL
218 * for temporary allocations.
219 */
220static svn_error_t *
221memcache_internal_set(void *cache_void,
222                      const void *key,
223                      const char *data,
224                      apr_size_t len,
225                      apr_pool_t *scratch_pool)
226{
227  memcache_t *cache = cache_void;
228  const char *mc_key;
229  apr_status_t apr_err;
230
231  SVN_ERR(build_key(&mc_key, cache, key, scratch_pool));
232  apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0);
233
234  /* ### Maybe write failures should be ignored (but logged)? */
235  if (apr_err != APR_SUCCESS)
236    return svn_error_wrap_apr(apr_err,
237                              _("Unknown memcached error while writing"));
238
239  return SVN_NO_ERROR;
240}
241
242
243static svn_error_t *
244memcache_set(void *cache_void,
245             const void *key,
246             void *value,
247             apr_pool_t *scratch_pool)
248{
249  memcache_t *cache = cache_void;
250  apr_pool_t *subpool = svn_pool_create(scratch_pool);
251  void *data;
252  apr_size_t data_len;
253  svn_error_t *err;
254
255  if (key == NULL)
256    return SVN_NO_ERROR;
257
258  if (cache->serialize_func)
259    {
260      SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool));
261    }
262  else
263    {
264      svn_stringbuf_t *value_str = value;
265      data = value_str->data;
266      data_len = value_str->len;
267    }
268
269  err = memcache_internal_set(cache_void, key, data, data_len, subpool);
270
271  svn_pool_destroy(subpool);
272  return err;
273}
274
275static svn_error_t *
276memcache_get_partial(void **value_p,
277                     svn_boolean_t *found,
278                     void *cache_void,
279                     const void *key,
280                     svn_cache__partial_getter_func_t func,
281                     void *baton,
282                     apr_pool_t *result_pool)
283{
284  svn_error_t *err = SVN_NO_ERROR;
285
286  char *data;
287  apr_size_t size;
288  SVN_ERR(memcache_internal_get(&data,
289                                &size,
290                                found,
291                                cache_void,
292                                key,
293                                result_pool));
294
295  /* If we found it, de-serialize it. */
296  return *found
297    ? func(value_p, data, size, baton, result_pool)
298    : err;
299}
300
301
302static svn_error_t *
303memcache_set_partial(void *cache_void,
304                     const void *key,
305                     svn_cache__partial_setter_func_t func,
306                     void *baton,
307                     apr_pool_t *scratch_pool)
308{
309  svn_error_t *err = SVN_NO_ERROR;
310
311  void *data;
312  apr_size_t size;
313  svn_boolean_t found = FALSE;
314
315  apr_pool_t *subpool = svn_pool_create(scratch_pool);
316  SVN_ERR(memcache_internal_get((char **)&data,
317                                &size,
318                                &found,
319                                cache_void,
320                                key,
321                                subpool));
322
323  /* If we found it, modify it and write it back to cache */
324  if (found)
325    {
326      SVN_ERR(func(&data, &size, baton, subpool));
327      err = memcache_internal_set(cache_void, key, data, size, subpool);
328    }
329
330  svn_pool_destroy(subpool);
331  return err;
332}
333
334
335static svn_error_t *
336memcache_iter(svn_boolean_t *completed,
337              void *cache_void,
338              svn_iter_apr_hash_cb_t user_cb,
339              void *user_baton,
340              apr_pool_t *scratch_pool)
341{
342  return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL,
343                          _("Can't iterate a memcached cache"));
344}
345
346static svn_boolean_t
347memcache_is_cachable(void *unused, apr_size_t size)
348{
349  (void)unused;  /* silence gcc warning. */
350
351  /* The memcached cutoff seems to be a bit (header length?) under a megabyte.
352   * We round down a little to be safe.
353   */
354  return size < 1000000;
355}
356
357static svn_error_t *
358memcache_get_info(void *cache_void,
359                  svn_cache__info_t *info,
360                  svn_boolean_t reset,
361                  apr_pool_t *result_pool)
362{
363  memcache_t *cache = cache_void;
364
365  info->id = apr_pstrdup(result_pool, cache->prefix);
366
367  /* we don't have any memory allocation info */
368
369  info->used_size = 0;
370  info->total_size = 0;
371  info->data_size = 0;
372  info->used_entries = 0;
373  info->total_entries = 0;
374
375  return SVN_NO_ERROR;
376}
377
378static svn_cache__vtable_t memcache_vtable = {
379  memcache_get,
380  memcache_set,
381  memcache_iter,
382  memcache_is_cachable,
383  memcache_get_partial,
384  memcache_set_partial,
385  memcache_get_info
386};
387
388svn_error_t *
389svn_cache__create_memcache(svn_cache__t **cache_p,
390                          svn_memcache_t *memcache,
391                          svn_cache__serialize_func_t serialize_func,
392                          svn_cache__deserialize_func_t deserialize_func,
393                          apr_ssize_t klen,
394                          const char *prefix,
395                          apr_pool_t *pool)
396{
397  svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper));
398  memcache_t *cache = apr_pcalloc(pool, sizeof(*cache));
399
400  cache->serialize_func = serialize_func;
401  cache->deserialize_func = deserialize_func;
402  cache->klen = klen;
403  cache->prefix = svn_path_uri_encode(prefix, pool);
404  cache->memcache = memcache->c;
405
406  wrapper->vtable = &memcache_vtable;
407  wrapper->cache_internal = cache;
408  wrapper->error_handler = 0;
409  wrapper->error_baton = 0;
410
411  *cache_p = wrapper;
412  return SVN_NO_ERROR;
413}
414
415
416/*** Creating apr_memcache_t from svn_config_t. ***/
417
418/* Baton for add_memcache_server. */
419struct ams_baton {
420  apr_memcache_t *memcache;
421  apr_pool_t *memcache_pool;
422  svn_error_t *err;
423};
424
425/* Implements svn_config_enumerator2_t. */
426static svn_boolean_t
427add_memcache_server(const char *name,
428                    const char *value,
429                    void *baton,
430                    apr_pool_t *pool)
431{
432  struct ams_baton *b = baton;
433  char *host, *scope;
434  apr_port_t port;
435  apr_status_t apr_err;
436  apr_memcache_server_t *server;
437
438  apr_err = apr_parse_addr_port(&host, &scope, &port,
439                                value, pool);
440  if (apr_err != APR_SUCCESS)
441    {
442      b->err = svn_error_wrap_apr(apr_err,
443                                  _("Error parsing memcache server '%s'"),
444                                  name);
445      return FALSE;
446    }
447
448  if (scope)
449    {
450      b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
451                                  _("Scope not allowed in memcache server "
452                                    "'%s'"),
453                                  name);
454      return FALSE;
455    }
456  if (!host || !port)
457    {
458      b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL,
459                                  _("Must specify host and port for memcache "
460                                    "server '%s'"),
461                                  name);
462      return FALSE;
463    }
464
465  /* Note: the four numbers here are only relevant when an
466     apr_memcache_t is being shared by multiple threads. */
467  apr_err = apr_memcache_server_create(b->memcache_pool,
468                                       host,
469                                       port,
470                                       0,  /* min connections */
471                                       5,  /* soft max connections */
472                                       10, /* hard max connections */
473                                       /*  time to live (in microseconds) */
474                                       apr_time_from_sec(50),
475                                       &server);
476  if (apr_err != APR_SUCCESS)
477    {
478      b->err = svn_error_wrap_apr(apr_err,
479                                  _("Unknown error creating memcache server"));
480      return FALSE;
481    }
482
483  apr_err = apr_memcache_add_server(b->memcache, server);
484  if (apr_err != APR_SUCCESS)
485    {
486      b->err = svn_error_wrap_apr(apr_err,
487                                  _("Unknown error adding server to memcache"));
488      return FALSE;
489    }
490
491  return TRUE;
492}
493
494#else /* ! SVN_HAVE_MEMCACHE */
495
496/* Stubs for no apr memcache library. */
497
498struct svn_memcache_t {
499  void *unused; /* Let's not have a size-zero struct. */
500};
501
502svn_error_t *
503svn_cache__create_memcache(svn_cache__t **cache_p,
504                          svn_memcache_t *memcache,
505                          svn_cache__serialize_func_t serialize_func,
506                          svn_cache__deserialize_func_t deserialize_func,
507                          apr_ssize_t klen,
508                          const char *prefix,
509                          apr_pool_t *pool)
510{
511  return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
512}
513
514#endif /* SVN_HAVE_MEMCACHE */
515
516/* Implements svn_config_enumerator2_t.  Just used for the
517   entry-counting return value of svn_config_enumerate2. */
518static svn_boolean_t
519nop_enumerator(const char *name,
520               const char *value,
521               void *baton,
522               apr_pool_t *pool)
523{
524  return TRUE;
525}
526
527svn_error_t *
528svn_cache__make_memcache_from_config(svn_memcache_t **memcache_p,
529                                    svn_config_t *config,
530                                    apr_pool_t *pool)
531{
532  int server_count;
533  apr_pool_t *subpool = svn_pool_create(pool);
534
535  server_count =
536    svn_config_enumerate2(config,
537                          SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
538                          nop_enumerator, NULL, subpool);
539
540  if (server_count == 0)
541    {
542      *memcache_p = NULL;
543      svn_pool_destroy(subpool);
544      return SVN_NO_ERROR;
545    }
546
547  if (server_count > APR_INT16_MAX)
548    return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL);
549
550#ifdef SVN_HAVE_MEMCACHE
551  {
552    struct ams_baton b;
553    svn_memcache_t *memcache = apr_pcalloc(pool, sizeof(*memcache));
554    apr_status_t apr_err = apr_memcache_create(pool,
555                                               (apr_uint16_t)server_count,
556                                               0, /* flags */
557                                               &(memcache->c));
558    if (apr_err != APR_SUCCESS)
559      return svn_error_wrap_apr(apr_err,
560                                _("Unknown error creating apr_memcache_t"));
561
562    b.memcache = memcache->c;
563    b.memcache_pool = pool;
564    b.err = SVN_NO_ERROR;
565    svn_config_enumerate2(config,
566                          SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS,
567                          add_memcache_server, &b,
568                          subpool);
569
570    if (b.err)
571      return b.err;
572
573    *memcache_p = memcache;
574
575    svn_pool_destroy(subpool);
576    return SVN_NO_ERROR;
577  }
578#else /* ! SVN_HAVE_MEMCACHE */
579  {
580    return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL);
581  }
582#endif /* SVN_HAVE_MEMCACHE */
583}
584