1/* 2 * cache-memcache.c: memcached caching for Subversion 3 * 4 * ==================================================================== 5 * Licensed to the Apache Software Foundation (ASF) under one 6 * or more contributor license agreements. See the NOTICE file 7 * distributed with this work for additional information 8 * regarding copyright ownership. The ASF licenses this file 9 * to you under the Apache License, Version 2.0 (the 10 * "License"); you may not use this file except in compliance 11 * with the License. You may obtain a copy of the License at 12 * 13 * http://www.apache.org/licenses/LICENSE-2.0 14 * 15 * Unless required by applicable law or agreed to in writing, 16 * software distributed under the License is distributed on an 17 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18 * KIND, either express or implied. See the License for the 19 * specific language governing permissions and limitations 20 * under the License. 21 * ==================================================================== 22 */ 23 24#include <apr_md5.h> 25 26#include "svn_pools.h" 27#include "svn_base64.h" 28#include "svn_path.h" 29 30#include "svn_private_config.h" 31#include "private/svn_cache.h" 32#include "private/svn_dep_compat.h" 33 34#include "cache.h" 35 36#ifdef SVN_HAVE_MEMCACHE 37 38#include <apr_memcache.h> 39 40/* A note on thread safety: 41 42 The apr_memcache_t object does its own mutex handling, and nothing 43 else in memcache_t is ever modified, so this implementation should 44 be fully thread-safe. 45*/ 46 47/* The (internal) cache object. */ 48typedef struct memcache_t { 49 /* The memcached server set we're using. */ 50 apr_memcache_t *memcache; 51 52 /* A prefix used to differentiate our data from any other data in 53 * the memcached (URI-encoded). */ 54 const char *prefix; 55 56 /* The size of the key: either a fixed number of bytes or 57 * APR_HASH_KEY_STRING. */ 58 apr_ssize_t klen; 59 60 61 /* Used to marshal values in and out of the cache. */ 62 svn_cache__serialize_func_t serialize_func; 63 svn_cache__deserialize_func_t deserialize_func; 64} memcache_t; 65 66/* The wrapper around apr_memcache_t. */ 67struct svn_memcache_t { 68 apr_memcache_t *c; 69}; 70 71 72/* The memcached protocol says the maximum key length is 250. Let's 73 just say 249, to be safe. */ 74#define MAX_MEMCACHED_KEY_LEN 249 75#define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \ 76 2 * APR_MD5_DIGESTSIZE) 77 78 79/* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated 80 in POOL. */ 81static svn_error_t * 82build_key(const char **mc_key, 83 memcache_t *cache, 84 const void *raw_key, 85 apr_pool_t *pool) 86{ 87 const char *encoded_suffix; 88 const char *long_key; 89 apr_size_t long_key_len; 90 91 if (cache->klen == APR_HASH_KEY_STRING) 92 encoded_suffix = svn_path_uri_encode(raw_key, pool); 93 else 94 { 95 const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool); 96 const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE, 97 pool); 98 encoded_suffix = encoded->data; 99 } 100 101 long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix, 102 (char *)NULL); 103 long_key_len = strlen(long_key); 104 105 /* We don't want to have a key that's too big. If it was going to 106 be too big, we MD5 the entire string, then replace the last bit 107 with the checksum. Note that APR_MD5_DIGESTSIZE is for the pure 108 binary digest; we have to double that when we convert to hex. 109 110 Every key we use will either be at most 111 MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly 112 MAX_MEMCACHED_KEY_LEN bytes long. */ 113 if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN) 114 { 115 svn_checksum_t *checksum; 116 SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len, 117 pool)); 118 119 long_key = apr_pstrcat(pool, 120 apr_pstrmemdup(pool, long_key, 121 MEMCACHED_KEY_UNHASHED_LEN), 122 svn_checksum_to_cstring_display(checksum, pool), 123 (char *)NULL); 124 } 125 126 *mc_key = long_key; 127 return SVN_NO_ERROR; 128} 129 130/* Core functionality of our getter functions: fetch DATA from the memcached 131 * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and 132 * use a tempoary sub-pool of POOL for allocations. 133 */ 134static svn_error_t * 135memcache_internal_get(char **data, 136 apr_size_t *size, 137 svn_boolean_t *found, 138 void *cache_void, 139 const void *key, 140 apr_pool_t *pool) 141{ 142 memcache_t *cache = cache_void; 143 apr_status_t apr_err; 144 const char *mc_key; 145 apr_pool_t *subpool; 146 147 if (key == NULL) 148 { 149 *found = FALSE; 150 return SVN_NO_ERROR; 151 } 152 153 subpool = svn_pool_create(pool); 154 SVN_ERR(build_key(&mc_key, cache, key, subpool)); 155 156 apr_err = apr_memcache_getp(cache->memcache, 157 pool, 158 mc_key, 159 data, 160 size, 161 NULL /* ignore flags */); 162 if (apr_err == APR_NOTFOUND) 163 { 164 *found = FALSE; 165 svn_pool_destroy(subpool); 166 return SVN_NO_ERROR; 167 } 168 else if (apr_err != APR_SUCCESS || !*data) 169 return svn_error_wrap_apr(apr_err, 170 _("Unknown memcached error while reading")); 171 172 *found = TRUE; 173 174 svn_pool_destroy(subpool); 175 return SVN_NO_ERROR; 176} 177 178 179static svn_error_t * 180memcache_get(void **value_p, 181 svn_boolean_t *found, 182 void *cache_void, 183 const void *key, 184 apr_pool_t *result_pool) 185{ 186 memcache_t *cache = cache_void; 187 char *data; 188 apr_size_t data_len; 189 SVN_ERR(memcache_internal_get(&data, 190 &data_len, 191 found, 192 cache_void, 193 key, 194 result_pool)); 195 196 /* If we found it, de-serialize it. */ 197 if (*found) 198 { 199 if (cache->deserialize_func) 200 { 201 SVN_ERR((cache->deserialize_func)(value_p, data, data_len, 202 result_pool)); 203 } 204 else 205 { 206 svn_string_t *value = apr_pcalloc(result_pool, sizeof(*value)); 207 value->data = data; 208 value->len = data_len; 209 *value_p = value; 210 } 211 } 212 213 return SVN_NO_ERROR; 214} 215 216/* Core functionality of our setter functions: store LENGH bytes of DATA 217 * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL 218 * for temporary allocations. 219 */ 220static svn_error_t * 221memcache_internal_set(void *cache_void, 222 const void *key, 223 const char *data, 224 apr_size_t len, 225 apr_pool_t *scratch_pool) 226{ 227 memcache_t *cache = cache_void; 228 const char *mc_key; 229 apr_status_t apr_err; 230 231 SVN_ERR(build_key(&mc_key, cache, key, scratch_pool)); 232 apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0); 233 234 /* ### Maybe write failures should be ignored (but logged)? */ 235 if (apr_err != APR_SUCCESS) 236 return svn_error_wrap_apr(apr_err, 237 _("Unknown memcached error while writing")); 238 239 return SVN_NO_ERROR; 240} 241 242 243static svn_error_t * 244memcache_set(void *cache_void, 245 const void *key, 246 void *value, 247 apr_pool_t *scratch_pool) 248{ 249 memcache_t *cache = cache_void; 250 apr_pool_t *subpool = svn_pool_create(scratch_pool); 251 void *data; 252 apr_size_t data_len; 253 svn_error_t *err; 254 255 if (key == NULL) 256 return SVN_NO_ERROR; 257 258 if (cache->serialize_func) 259 { 260 SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool)); 261 } 262 else 263 { 264 svn_stringbuf_t *value_str = value; 265 data = value_str->data; 266 data_len = value_str->len; 267 } 268 269 err = memcache_internal_set(cache_void, key, data, data_len, subpool); 270 271 svn_pool_destroy(subpool); 272 return err; 273} 274 275static svn_error_t * 276memcache_get_partial(void **value_p, 277 svn_boolean_t *found, 278 void *cache_void, 279 const void *key, 280 svn_cache__partial_getter_func_t func, 281 void *baton, 282 apr_pool_t *result_pool) 283{ 284 svn_error_t *err = SVN_NO_ERROR; 285 286 char *data; 287 apr_size_t size; 288 SVN_ERR(memcache_internal_get(&data, 289 &size, 290 found, 291 cache_void, 292 key, 293 result_pool)); 294 295 /* If we found it, de-serialize it. */ 296 return *found 297 ? func(value_p, data, size, baton, result_pool) 298 : err; 299} 300 301 302static svn_error_t * 303memcache_set_partial(void *cache_void, 304 const void *key, 305 svn_cache__partial_setter_func_t func, 306 void *baton, 307 apr_pool_t *scratch_pool) 308{ 309 svn_error_t *err = SVN_NO_ERROR; 310 311 void *data; 312 apr_size_t size; 313 svn_boolean_t found = FALSE; 314 315 apr_pool_t *subpool = svn_pool_create(scratch_pool); 316 SVN_ERR(memcache_internal_get((char **)&data, 317 &size, 318 &found, 319 cache_void, 320 key, 321 subpool)); 322 323 /* If we found it, modify it and write it back to cache */ 324 if (found) 325 { 326 SVN_ERR(func(&data, &size, baton, subpool)); 327 err = memcache_internal_set(cache_void, key, data, size, subpool); 328 } 329 330 svn_pool_destroy(subpool); 331 return err; 332} 333 334 335static svn_error_t * 336memcache_iter(svn_boolean_t *completed, 337 void *cache_void, 338 svn_iter_apr_hash_cb_t user_cb, 339 void *user_baton, 340 apr_pool_t *scratch_pool) 341{ 342 return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 343 _("Can't iterate a memcached cache")); 344} 345 346static svn_boolean_t 347memcache_is_cachable(void *unused, apr_size_t size) 348{ 349 (void)unused; /* silence gcc warning. */ 350 351 /* The memcached cutoff seems to be a bit (header length?) under a megabyte. 352 * We round down a little to be safe. 353 */ 354 return size < 1000000; 355} 356 357static svn_error_t * 358memcache_get_info(void *cache_void, 359 svn_cache__info_t *info, 360 svn_boolean_t reset, 361 apr_pool_t *result_pool) 362{ 363 memcache_t *cache = cache_void; 364 365 info->id = apr_pstrdup(result_pool, cache->prefix); 366 367 /* we don't have any memory allocation info */ 368 369 info->used_size = 0; 370 info->total_size = 0; 371 info->data_size = 0; 372 info->used_entries = 0; 373 info->total_entries = 0; 374 375 return SVN_NO_ERROR; 376} 377 378static svn_cache__vtable_t memcache_vtable = { 379 memcache_get, 380 memcache_set, 381 memcache_iter, 382 memcache_is_cachable, 383 memcache_get_partial, 384 memcache_set_partial, 385 memcache_get_info 386}; 387 388svn_error_t * 389svn_cache__create_memcache(svn_cache__t **cache_p, 390 svn_memcache_t *memcache, 391 svn_cache__serialize_func_t serialize_func, 392 svn_cache__deserialize_func_t deserialize_func, 393 apr_ssize_t klen, 394 const char *prefix, 395 apr_pool_t *pool) 396{ 397 svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper)); 398 memcache_t *cache = apr_pcalloc(pool, sizeof(*cache)); 399 400 cache->serialize_func = serialize_func; 401 cache->deserialize_func = deserialize_func; 402 cache->klen = klen; 403 cache->prefix = svn_path_uri_encode(prefix, pool); 404 cache->memcache = memcache->c; 405 406 wrapper->vtable = &memcache_vtable; 407 wrapper->cache_internal = cache; 408 wrapper->error_handler = 0; 409 wrapper->error_baton = 0; 410 411 *cache_p = wrapper; 412 return SVN_NO_ERROR; 413} 414 415 416/*** Creating apr_memcache_t from svn_config_t. ***/ 417 418/* Baton for add_memcache_server. */ 419struct ams_baton { 420 apr_memcache_t *memcache; 421 apr_pool_t *memcache_pool; 422 svn_error_t *err; 423}; 424 425/* Implements svn_config_enumerator2_t. */ 426static svn_boolean_t 427add_memcache_server(const char *name, 428 const char *value, 429 void *baton, 430 apr_pool_t *pool) 431{ 432 struct ams_baton *b = baton; 433 char *host, *scope; 434 apr_port_t port; 435 apr_status_t apr_err; 436 apr_memcache_server_t *server; 437 438 apr_err = apr_parse_addr_port(&host, &scope, &port, 439 value, pool); 440 if (apr_err != APR_SUCCESS) 441 { 442 b->err = svn_error_wrap_apr(apr_err, 443 _("Error parsing memcache server '%s'"), 444 name); 445 return FALSE; 446 } 447 448 if (scope) 449 { 450 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL, 451 _("Scope not allowed in memcache server " 452 "'%s'"), 453 name); 454 return FALSE; 455 } 456 if (!host || !port) 457 { 458 b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL, 459 _("Must specify host and port for memcache " 460 "server '%s'"), 461 name); 462 return FALSE; 463 } 464 465 /* Note: the four numbers here are only relevant when an 466 apr_memcache_t is being shared by multiple threads. */ 467 apr_err = apr_memcache_server_create(b->memcache_pool, 468 host, 469 port, 470 0, /* min connections */ 471 5, /* soft max connections */ 472 10, /* hard max connections */ 473 /* time to live (in microseconds) */ 474 apr_time_from_sec(50), 475 &server); 476 if (apr_err != APR_SUCCESS) 477 { 478 b->err = svn_error_wrap_apr(apr_err, 479 _("Unknown error creating memcache server")); 480 return FALSE; 481 } 482 483 apr_err = apr_memcache_add_server(b->memcache, server); 484 if (apr_err != APR_SUCCESS) 485 { 486 b->err = svn_error_wrap_apr(apr_err, 487 _("Unknown error adding server to memcache")); 488 return FALSE; 489 } 490 491 return TRUE; 492} 493 494#else /* ! SVN_HAVE_MEMCACHE */ 495 496/* Stubs for no apr memcache library. */ 497 498struct svn_memcache_t { 499 void *unused; /* Let's not have a size-zero struct. */ 500}; 501 502svn_error_t * 503svn_cache__create_memcache(svn_cache__t **cache_p, 504 svn_memcache_t *memcache, 505 svn_cache__serialize_func_t serialize_func, 506 svn_cache__deserialize_func_t deserialize_func, 507 apr_ssize_t klen, 508 const char *prefix, 509 apr_pool_t *pool) 510{ 511 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL); 512} 513 514#endif /* SVN_HAVE_MEMCACHE */ 515 516/* Implements svn_config_enumerator2_t. Just used for the 517 entry-counting return value of svn_config_enumerate2. */ 518static svn_boolean_t 519nop_enumerator(const char *name, 520 const char *value, 521 void *baton, 522 apr_pool_t *pool) 523{ 524 return TRUE; 525} 526 527svn_error_t * 528svn_cache__make_memcache_from_config(svn_memcache_t **memcache_p, 529 svn_config_t *config, 530 apr_pool_t *pool) 531{ 532 int server_count; 533 apr_pool_t *subpool = svn_pool_create(pool); 534 535 server_count = 536 svn_config_enumerate2(config, 537 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS, 538 nop_enumerator, NULL, subpool); 539 540 if (server_count == 0) 541 { 542 *memcache_p = NULL; 543 svn_pool_destroy(subpool); 544 return SVN_NO_ERROR; 545 } 546 547 if (server_count > APR_INT16_MAX) 548 return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL); 549 550#ifdef SVN_HAVE_MEMCACHE 551 { 552 struct ams_baton b; 553 svn_memcache_t *memcache = apr_pcalloc(pool, sizeof(*memcache)); 554 apr_status_t apr_err = apr_memcache_create(pool, 555 (apr_uint16_t)server_count, 556 0, /* flags */ 557 &(memcache->c)); 558 if (apr_err != APR_SUCCESS) 559 return svn_error_wrap_apr(apr_err, 560 _("Unknown error creating apr_memcache_t")); 561 562 b.memcache = memcache->c; 563 b.memcache_pool = pool; 564 b.err = SVN_NO_ERROR; 565 svn_config_enumerate2(config, 566 SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS, 567 add_memcache_server, &b, 568 subpool); 569 570 if (b.err) 571 return b.err; 572 573 *memcache_p = memcache; 574 575 svn_pool_destroy(subpool); 576 return SVN_NO_ERROR; 577 } 578#else /* ! SVN_HAVE_MEMCACHE */ 579 { 580 return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL); 581 } 582#endif /* SVN_HAVE_MEMCACHE */ 583} 584