1251881Speter/* 2251881Speter * cache-memcache.c: memcached caching for Subversion 3251881Speter * 4251881Speter * ==================================================================== 5251881Speter * Licensed to the Apache Software Foundation (ASF) under one 6251881Speter * or more contributor license agreements. See the NOTICE file 7251881Speter * distributed with this work for additional information 8251881Speter * regarding copyright ownership. The ASF licenses this file 9251881Speter * to you under the Apache License, Version 2.0 (the 10251881Speter * "License"); you may not use this file except in compliance 11251881Speter * with the License. You may obtain a copy of the License at 12251881Speter * 13251881Speter * http://www.apache.org/licenses/LICENSE-2.0 14251881Speter * 15251881Speter * Unless required by applicable law or agreed to in writing, 16251881Speter * software distributed under the License is distributed on an 17251881Speter * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 18251881Speter * KIND, either express or implied. See the License for the 19251881Speter * specific language governing permissions and limitations 20251881Speter * under the License. 21251881Speter * ==================================================================== 22251881Speter */ 23251881Speter 24251881Speter#include <apr_md5.h> 25251881Speter 26251881Speter#include "svn_pools.h" 27251881Speter#include "svn_base64.h" 28251881Speter#include "svn_path.h" 29251881Speter 30251881Speter#include "svn_private_config.h" 31251881Speter#include "private/svn_cache.h" 32251881Speter#include "private/svn_dep_compat.h" 33251881Speter 34251881Speter#include "cache.h" 35251881Speter 36251881Speter#ifdef SVN_HAVE_MEMCACHE 37251881Speter 38251881Speter#include <apr_memcache.h> 39251881Speter 40251881Speter/* A note on thread safety: 41251881Speter 42251881Speter The apr_memcache_t object does its own mutex handling, and nothing 43251881Speter else in memcache_t is ever modified, so this implementation should 44251881Speter be fully thread-safe. 45251881Speter*/ 46251881Speter 47251881Speter/* The (internal) cache object. */ 48251881Spetertypedef struct memcache_t { 49251881Speter /* The memcached server set we're using. */ 50251881Speter apr_memcache_t *memcache; 51251881Speter 52251881Speter /* A prefix used to differentiate our data from any other data in 53251881Speter * the memcached (URI-encoded). */ 54251881Speter const char *prefix; 55251881Speter 56251881Speter /* The size of the key: either a fixed number of bytes or 57251881Speter * APR_HASH_KEY_STRING. */ 58251881Speter apr_ssize_t klen; 59251881Speter 60251881Speter 61251881Speter /* Used to marshal values in and out of the cache. */ 62251881Speter svn_cache__serialize_func_t serialize_func; 63251881Speter svn_cache__deserialize_func_t deserialize_func; 64251881Speter} memcache_t; 65251881Speter 66251881Speter/* The wrapper around apr_memcache_t. */ 67251881Speterstruct svn_memcache_t { 68251881Speter apr_memcache_t *c; 69251881Speter}; 70251881Speter 71251881Speter 72251881Speter/* The memcached protocol says the maximum key length is 250. Let's 73251881Speter just say 249, to be safe. */ 74251881Speter#define MAX_MEMCACHED_KEY_LEN 249 75251881Speter#define MEMCACHED_KEY_UNHASHED_LEN (MAX_MEMCACHED_KEY_LEN - \ 76251881Speter 2 * APR_MD5_DIGESTSIZE) 77251881Speter 78251881Speter 79251881Speter/* Set *MC_KEY to a memcache key for the given key KEY for CACHE, allocated 80251881Speter in POOL. */ 81251881Speterstatic svn_error_t * 82251881Speterbuild_key(const char **mc_key, 83251881Speter memcache_t *cache, 84251881Speter const void *raw_key, 85251881Speter apr_pool_t *pool) 86251881Speter{ 87251881Speter const char *encoded_suffix; 88251881Speter const char *long_key; 89251881Speter apr_size_t long_key_len; 90251881Speter 91251881Speter if (cache->klen == APR_HASH_KEY_STRING) 92251881Speter encoded_suffix = svn_path_uri_encode(raw_key, pool); 93251881Speter else 94251881Speter { 95251881Speter const svn_string_t *raw = svn_string_ncreate(raw_key, cache->klen, pool); 96251881Speter const svn_string_t *encoded = svn_base64_encode_string2(raw, FALSE, 97251881Speter pool); 98251881Speter encoded_suffix = encoded->data; 99251881Speter } 100251881Speter 101251881Speter long_key = apr_pstrcat(pool, "SVN:", cache->prefix, ":", encoded_suffix, 102251881Speter (char *)NULL); 103251881Speter long_key_len = strlen(long_key); 104251881Speter 105251881Speter /* We don't want to have a key that's too big. If it was going to 106251881Speter be too big, we MD5 the entire string, then replace the last bit 107251881Speter with the checksum. Note that APR_MD5_DIGESTSIZE is for the pure 108251881Speter binary digest; we have to double that when we convert to hex. 109251881Speter 110251881Speter Every key we use will either be at most 111251881Speter MEMCACHED_KEY_UNHASHED_LEN bytes long, or be exactly 112251881Speter MAX_MEMCACHED_KEY_LEN bytes long. */ 113251881Speter if (long_key_len > MEMCACHED_KEY_UNHASHED_LEN) 114251881Speter { 115251881Speter svn_checksum_t *checksum; 116251881Speter SVN_ERR(svn_checksum(&checksum, svn_checksum_md5, long_key, long_key_len, 117251881Speter pool)); 118251881Speter 119251881Speter long_key = apr_pstrcat(pool, 120251881Speter apr_pstrmemdup(pool, long_key, 121251881Speter MEMCACHED_KEY_UNHASHED_LEN), 122251881Speter svn_checksum_to_cstring_display(checksum, pool), 123251881Speter (char *)NULL); 124251881Speter } 125251881Speter 126251881Speter *mc_key = long_key; 127251881Speter return SVN_NO_ERROR; 128251881Speter} 129251881Speter 130251881Speter/* Core functionality of our getter functions: fetch DATA from the memcached 131251881Speter * given by CACHE_VOID and identified by KEY. Indicate success in FOUND and 132251881Speter * use a tempoary sub-pool of POOL for allocations. 133251881Speter */ 134251881Speterstatic svn_error_t * 135251881Spetermemcache_internal_get(char **data, 136251881Speter apr_size_t *size, 137251881Speter svn_boolean_t *found, 138251881Speter void *cache_void, 139251881Speter const void *key, 140251881Speter apr_pool_t *pool) 141251881Speter{ 142251881Speter memcache_t *cache = cache_void; 143251881Speter apr_status_t apr_err; 144251881Speter const char *mc_key; 145251881Speter apr_pool_t *subpool; 146251881Speter 147251881Speter if (key == NULL) 148251881Speter { 149251881Speter *found = FALSE; 150251881Speter return SVN_NO_ERROR; 151251881Speter } 152251881Speter 153251881Speter subpool = svn_pool_create(pool); 154251881Speter SVN_ERR(build_key(&mc_key, cache, key, subpool)); 155251881Speter 156251881Speter apr_err = apr_memcache_getp(cache->memcache, 157251881Speter pool, 158251881Speter mc_key, 159251881Speter data, 160251881Speter size, 161251881Speter NULL /* ignore flags */); 162251881Speter if (apr_err == APR_NOTFOUND) 163251881Speter { 164251881Speter *found = FALSE; 165251881Speter svn_pool_destroy(subpool); 166251881Speter return SVN_NO_ERROR; 167251881Speter } 168251881Speter else if (apr_err != APR_SUCCESS || !*data) 169251881Speter return svn_error_wrap_apr(apr_err, 170251881Speter _("Unknown memcached error while reading")); 171251881Speter 172251881Speter *found = TRUE; 173251881Speter 174251881Speter svn_pool_destroy(subpool); 175251881Speter return SVN_NO_ERROR; 176251881Speter} 177251881Speter 178251881Speter 179251881Speterstatic svn_error_t * 180251881Spetermemcache_get(void **value_p, 181251881Speter svn_boolean_t *found, 182251881Speter void *cache_void, 183251881Speter const void *key, 184251881Speter apr_pool_t *result_pool) 185251881Speter{ 186251881Speter memcache_t *cache = cache_void; 187251881Speter char *data; 188251881Speter apr_size_t data_len; 189251881Speter SVN_ERR(memcache_internal_get(&data, 190251881Speter &data_len, 191251881Speter found, 192251881Speter cache_void, 193251881Speter key, 194251881Speter result_pool)); 195251881Speter 196251881Speter /* If we found it, de-serialize it. */ 197251881Speter if (*found) 198251881Speter { 199251881Speter if (cache->deserialize_func) 200251881Speter { 201251881Speter SVN_ERR((cache->deserialize_func)(value_p, data, data_len, 202251881Speter result_pool)); 203251881Speter } 204251881Speter else 205251881Speter { 206251881Speter svn_string_t *value = apr_pcalloc(result_pool, sizeof(*value)); 207251881Speter value->data = data; 208251881Speter value->len = data_len; 209251881Speter *value_p = value; 210251881Speter } 211251881Speter } 212251881Speter 213251881Speter return SVN_NO_ERROR; 214251881Speter} 215251881Speter 216251881Speter/* Core functionality of our setter functions: store LENGH bytes of DATA 217251881Speter * to be identified by KEY in the memcached given by CACHE_VOID. Use POOL 218251881Speter * for temporary allocations. 219251881Speter */ 220251881Speterstatic svn_error_t * 221251881Spetermemcache_internal_set(void *cache_void, 222251881Speter const void *key, 223251881Speter const char *data, 224251881Speter apr_size_t len, 225251881Speter apr_pool_t *scratch_pool) 226251881Speter{ 227251881Speter memcache_t *cache = cache_void; 228251881Speter const char *mc_key; 229251881Speter apr_status_t apr_err; 230251881Speter 231251881Speter SVN_ERR(build_key(&mc_key, cache, key, scratch_pool)); 232251881Speter apr_err = apr_memcache_set(cache->memcache, mc_key, (char *)data, len, 0, 0); 233251881Speter 234251881Speter /* ### Maybe write failures should be ignored (but logged)? */ 235251881Speter if (apr_err != APR_SUCCESS) 236251881Speter return svn_error_wrap_apr(apr_err, 237251881Speter _("Unknown memcached error while writing")); 238251881Speter 239251881Speter return SVN_NO_ERROR; 240251881Speter} 241251881Speter 242251881Speter 243251881Speterstatic svn_error_t * 244251881Spetermemcache_set(void *cache_void, 245251881Speter const void *key, 246251881Speter void *value, 247251881Speter apr_pool_t *scratch_pool) 248251881Speter{ 249251881Speter memcache_t *cache = cache_void; 250251881Speter apr_pool_t *subpool = svn_pool_create(scratch_pool); 251251881Speter void *data; 252251881Speter apr_size_t data_len; 253251881Speter svn_error_t *err; 254251881Speter 255251881Speter if (key == NULL) 256251881Speter return SVN_NO_ERROR; 257251881Speter 258251881Speter if (cache->serialize_func) 259251881Speter { 260251881Speter SVN_ERR((cache->serialize_func)(&data, &data_len, value, subpool)); 261251881Speter } 262251881Speter else 263251881Speter { 264251881Speter svn_stringbuf_t *value_str = value; 265251881Speter data = value_str->data; 266251881Speter data_len = value_str->len; 267251881Speter } 268251881Speter 269251881Speter err = memcache_internal_set(cache_void, key, data, data_len, subpool); 270251881Speter 271251881Speter svn_pool_destroy(subpool); 272251881Speter return err; 273251881Speter} 274251881Speter 275251881Speterstatic svn_error_t * 276251881Spetermemcache_get_partial(void **value_p, 277251881Speter svn_boolean_t *found, 278251881Speter void *cache_void, 279251881Speter const void *key, 280251881Speter svn_cache__partial_getter_func_t func, 281251881Speter void *baton, 282251881Speter apr_pool_t *result_pool) 283251881Speter{ 284251881Speter svn_error_t *err = SVN_NO_ERROR; 285251881Speter 286251881Speter char *data; 287251881Speter apr_size_t size; 288251881Speter SVN_ERR(memcache_internal_get(&data, 289251881Speter &size, 290251881Speter found, 291251881Speter cache_void, 292251881Speter key, 293251881Speter result_pool)); 294251881Speter 295251881Speter /* If we found it, de-serialize it. */ 296251881Speter return *found 297251881Speter ? func(value_p, data, size, baton, result_pool) 298251881Speter : err; 299251881Speter} 300251881Speter 301251881Speter 302251881Speterstatic svn_error_t * 303251881Spetermemcache_set_partial(void *cache_void, 304251881Speter const void *key, 305251881Speter svn_cache__partial_setter_func_t func, 306251881Speter void *baton, 307251881Speter apr_pool_t *scratch_pool) 308251881Speter{ 309251881Speter svn_error_t *err = SVN_NO_ERROR; 310251881Speter 311251881Speter void *data; 312251881Speter apr_size_t size; 313251881Speter svn_boolean_t found = FALSE; 314251881Speter 315251881Speter apr_pool_t *subpool = svn_pool_create(scratch_pool); 316251881Speter SVN_ERR(memcache_internal_get((char **)&data, 317251881Speter &size, 318251881Speter &found, 319251881Speter cache_void, 320251881Speter key, 321251881Speter subpool)); 322251881Speter 323251881Speter /* If we found it, modify it and write it back to cache */ 324251881Speter if (found) 325251881Speter { 326251881Speter SVN_ERR(func(&data, &size, baton, subpool)); 327251881Speter err = memcache_internal_set(cache_void, key, data, size, subpool); 328251881Speter } 329251881Speter 330251881Speter svn_pool_destroy(subpool); 331251881Speter return err; 332251881Speter} 333251881Speter 334251881Speter 335251881Speterstatic svn_error_t * 336251881Spetermemcache_iter(svn_boolean_t *completed, 337251881Speter void *cache_void, 338251881Speter svn_iter_apr_hash_cb_t user_cb, 339251881Speter void *user_baton, 340251881Speter apr_pool_t *scratch_pool) 341251881Speter{ 342251881Speter return svn_error_create(SVN_ERR_UNSUPPORTED_FEATURE, NULL, 343251881Speter _("Can't iterate a memcached cache")); 344251881Speter} 345251881Speter 346251881Speterstatic svn_boolean_t 347251881Spetermemcache_is_cachable(void *unused, apr_size_t size) 348251881Speter{ 349251881Speter (void)unused; /* silence gcc warning. */ 350251881Speter 351251881Speter /* The memcached cutoff seems to be a bit (header length?) under a megabyte. 352251881Speter * We round down a little to be safe. 353251881Speter */ 354251881Speter return size < 1000000; 355251881Speter} 356251881Speter 357251881Speterstatic svn_error_t * 358251881Spetermemcache_get_info(void *cache_void, 359251881Speter svn_cache__info_t *info, 360251881Speter svn_boolean_t reset, 361251881Speter apr_pool_t *result_pool) 362251881Speter{ 363251881Speter memcache_t *cache = cache_void; 364251881Speter 365251881Speter info->id = apr_pstrdup(result_pool, cache->prefix); 366251881Speter 367251881Speter /* we don't have any memory allocation info */ 368251881Speter 369251881Speter info->used_size = 0; 370251881Speter info->total_size = 0; 371251881Speter info->data_size = 0; 372251881Speter info->used_entries = 0; 373251881Speter info->total_entries = 0; 374251881Speter 375251881Speter return SVN_NO_ERROR; 376251881Speter} 377251881Speter 378251881Speterstatic svn_cache__vtable_t memcache_vtable = { 379251881Speter memcache_get, 380251881Speter memcache_set, 381251881Speter memcache_iter, 382251881Speter memcache_is_cachable, 383251881Speter memcache_get_partial, 384251881Speter memcache_set_partial, 385251881Speter memcache_get_info 386251881Speter}; 387251881Speter 388251881Spetersvn_error_t * 389251881Spetersvn_cache__create_memcache(svn_cache__t **cache_p, 390251881Speter svn_memcache_t *memcache, 391251881Speter svn_cache__serialize_func_t serialize_func, 392251881Speter svn_cache__deserialize_func_t deserialize_func, 393251881Speter apr_ssize_t klen, 394251881Speter const char *prefix, 395251881Speter apr_pool_t *pool) 396251881Speter{ 397251881Speter svn_cache__t *wrapper = apr_pcalloc(pool, sizeof(*wrapper)); 398251881Speter memcache_t *cache = apr_pcalloc(pool, sizeof(*cache)); 399251881Speter 400251881Speter cache->serialize_func = serialize_func; 401251881Speter cache->deserialize_func = deserialize_func; 402251881Speter cache->klen = klen; 403251881Speter cache->prefix = svn_path_uri_encode(prefix, pool); 404251881Speter cache->memcache = memcache->c; 405251881Speter 406251881Speter wrapper->vtable = &memcache_vtable; 407251881Speter wrapper->cache_internal = cache; 408251881Speter wrapper->error_handler = 0; 409251881Speter wrapper->error_baton = 0; 410251881Speter 411251881Speter *cache_p = wrapper; 412251881Speter return SVN_NO_ERROR; 413251881Speter} 414251881Speter 415251881Speter 416251881Speter/*** Creating apr_memcache_t from svn_config_t. ***/ 417251881Speter 418251881Speter/* Baton for add_memcache_server. */ 419251881Speterstruct ams_baton { 420251881Speter apr_memcache_t *memcache; 421251881Speter apr_pool_t *memcache_pool; 422251881Speter svn_error_t *err; 423251881Speter}; 424251881Speter 425251881Speter/* Implements svn_config_enumerator2_t. */ 426251881Speterstatic svn_boolean_t 427251881Speteradd_memcache_server(const char *name, 428251881Speter const char *value, 429251881Speter void *baton, 430251881Speter apr_pool_t *pool) 431251881Speter{ 432251881Speter struct ams_baton *b = baton; 433251881Speter char *host, *scope; 434251881Speter apr_port_t port; 435251881Speter apr_status_t apr_err; 436251881Speter apr_memcache_server_t *server; 437251881Speter 438251881Speter apr_err = apr_parse_addr_port(&host, &scope, &port, 439251881Speter value, pool); 440251881Speter if (apr_err != APR_SUCCESS) 441251881Speter { 442251881Speter b->err = svn_error_wrap_apr(apr_err, 443251881Speter _("Error parsing memcache server '%s'"), 444251881Speter name); 445251881Speter return FALSE; 446251881Speter } 447251881Speter 448251881Speter if (scope) 449251881Speter { 450251881Speter b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL, 451251881Speter _("Scope not allowed in memcache server " 452251881Speter "'%s'"), 453251881Speter name); 454251881Speter return FALSE; 455251881Speter } 456251881Speter if (!host || !port) 457251881Speter { 458251881Speter b->err = svn_error_createf(SVN_ERR_BAD_SERVER_SPECIFICATION, NULL, 459251881Speter _("Must specify host and port for memcache " 460251881Speter "server '%s'"), 461251881Speter name); 462251881Speter return FALSE; 463251881Speter } 464251881Speter 465251881Speter /* Note: the four numbers here are only relevant when an 466251881Speter apr_memcache_t is being shared by multiple threads. */ 467251881Speter apr_err = apr_memcache_server_create(b->memcache_pool, 468251881Speter host, 469251881Speter port, 470251881Speter 0, /* min connections */ 471251881Speter 5, /* soft max connections */ 472251881Speter 10, /* hard max connections */ 473251881Speter /* time to live (in microseconds) */ 474251881Speter apr_time_from_sec(50), 475251881Speter &server); 476251881Speter if (apr_err != APR_SUCCESS) 477251881Speter { 478251881Speter b->err = svn_error_wrap_apr(apr_err, 479251881Speter _("Unknown error creating memcache server")); 480251881Speter return FALSE; 481251881Speter } 482251881Speter 483251881Speter apr_err = apr_memcache_add_server(b->memcache, server); 484251881Speter if (apr_err != APR_SUCCESS) 485251881Speter { 486251881Speter b->err = svn_error_wrap_apr(apr_err, 487251881Speter _("Unknown error adding server to memcache")); 488251881Speter return FALSE; 489251881Speter } 490251881Speter 491251881Speter return TRUE; 492251881Speter} 493251881Speter 494251881Speter#else /* ! SVN_HAVE_MEMCACHE */ 495251881Speter 496251881Speter/* Stubs for no apr memcache library. */ 497251881Speter 498251881Speterstruct svn_memcache_t { 499251881Speter void *unused; /* Let's not have a size-zero struct. */ 500251881Speter}; 501251881Speter 502251881Spetersvn_error_t * 503251881Spetersvn_cache__create_memcache(svn_cache__t **cache_p, 504251881Speter svn_memcache_t *memcache, 505251881Speter svn_cache__serialize_func_t serialize_func, 506251881Speter svn_cache__deserialize_func_t deserialize_func, 507251881Speter apr_ssize_t klen, 508251881Speter const char *prefix, 509251881Speter apr_pool_t *pool) 510251881Speter{ 511251881Speter return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL); 512251881Speter} 513251881Speter 514251881Speter#endif /* SVN_HAVE_MEMCACHE */ 515251881Speter 516251881Speter/* Implements svn_config_enumerator2_t. Just used for the 517251881Speter entry-counting return value of svn_config_enumerate2. */ 518251881Speterstatic svn_boolean_t 519251881Speternop_enumerator(const char *name, 520251881Speter const char *value, 521251881Speter void *baton, 522251881Speter apr_pool_t *pool) 523251881Speter{ 524251881Speter return TRUE; 525251881Speter} 526251881Speter 527251881Spetersvn_error_t * 528251881Spetersvn_cache__make_memcache_from_config(svn_memcache_t **memcache_p, 529251881Speter svn_config_t *config, 530251881Speter apr_pool_t *pool) 531251881Speter{ 532251881Speter int server_count; 533251881Speter apr_pool_t *subpool = svn_pool_create(pool); 534251881Speter 535251881Speter server_count = 536251881Speter svn_config_enumerate2(config, 537251881Speter SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS, 538251881Speter nop_enumerator, NULL, subpool); 539251881Speter 540251881Speter if (server_count == 0) 541251881Speter { 542251881Speter *memcache_p = NULL; 543251881Speter svn_pool_destroy(subpool); 544251881Speter return SVN_NO_ERROR; 545251881Speter } 546251881Speter 547251881Speter if (server_count > APR_INT16_MAX) 548251881Speter return svn_error_create(SVN_ERR_TOO_MANY_MEMCACHED_SERVERS, NULL, NULL); 549251881Speter 550251881Speter#ifdef SVN_HAVE_MEMCACHE 551251881Speter { 552251881Speter struct ams_baton b; 553251881Speter svn_memcache_t *memcache = apr_pcalloc(pool, sizeof(*memcache)); 554251881Speter apr_status_t apr_err = apr_memcache_create(pool, 555251881Speter (apr_uint16_t)server_count, 556251881Speter 0, /* flags */ 557251881Speter &(memcache->c)); 558251881Speter if (apr_err != APR_SUCCESS) 559251881Speter return svn_error_wrap_apr(apr_err, 560251881Speter _("Unknown error creating apr_memcache_t")); 561251881Speter 562251881Speter b.memcache = memcache->c; 563251881Speter b.memcache_pool = pool; 564251881Speter b.err = SVN_NO_ERROR; 565251881Speter svn_config_enumerate2(config, 566251881Speter SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS, 567251881Speter add_memcache_server, &b, 568251881Speter subpool); 569251881Speter 570251881Speter if (b.err) 571251881Speter return b.err; 572251881Speter 573251881Speter *memcache_p = memcache; 574251881Speter 575251881Speter svn_pool_destroy(subpool); 576251881Speter return SVN_NO_ERROR; 577251881Speter } 578251881Speter#else /* ! SVN_HAVE_MEMCACHE */ 579251881Speter { 580251881Speter return svn_error_create(SVN_ERR_NO_APR_MEMCACHE, NULL, NULL); 581251881Speter } 582251881Speter#endif /* SVN_HAVE_MEMCACHE */ 583251881Speter} 584