1/* $NetBSD: rumpuser_pth.c,v 1.6 2010/12/01 17:22:51 pooka Exp $ */ 2 3/* 4 * Copyright (c) 2007-2010 Antti Kantee. All Rights Reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29#if !defined(lint) 30__RCSID("$NetBSD: rumpuser_pth.c,v 1.6 2010/12/01 17:22:51 pooka Exp $"); 31#endif /* !lint */ 32 33#ifdef __linux__ 34#define _XOPEN_SOURCE 500 35#define _BSD_SOURCE 36#define _FILE_OFFSET_BITS 64 37#endif 38 39#include <assert.h> 40#include <errno.h> 41#include <pthread.h> 42#include <stdlib.h> 43#include <stdio.h> 44#include <string.h> 45#include <stdint.h> 46#include <unistd.h> 47 48#include <rump/rumpuser.h> 49 50#include "rumpuser_int.h" 51 52static pthread_key_t curlwpkey; 53 54#define NOFAIL(a) do {if (!(a)) abort();} while (/*CONSTCOND*/0) 55#define NOFAIL_ERRNO(a) \ 56do { \ 57 int fail_rv = (a); \ 58 if (fail_rv) { \ 59 printf("panic: rumpuser fatal failure %d (%s)\n", \ 60 fail_rv, strerror(fail_rv)); \ 61 abort(); \ 62 } \ 63} while (/*CONSTCOND*/0) 64 65struct rumpuser_mtx { 66 pthread_mutex_t pthmtx; 67 struct lwp *owner; 68 int iskmutex; 69}; 70 71#define RURW_AMWRITER(rw) (rw->writer == rumpuser_get_curlwp() \ 72 && rw->readers == -1) 73#define RURW_HASREAD(rw) (rw->readers > 0) 74 75#define RURW_SETWRITE(rw) \ 76do { \ 77 assert(rw->readers == 0); \ 78 rw->writer = rumpuser_get_curlwp(); \ 79 rw->readers = -1; \ 80} while (/*CONSTCOND*/0) 81#define RURW_CLRWRITE(rw) \ 82do { \ 83 assert(rw->readers == -1 && RURW_AMWRITER(rw)); \ 84 rw->readers = 0; \ 85} while (/*CONSTCOND*/0) 86#define RURW_INCREAD(rw) \ 87do { \ 88 pthread_spin_lock(&rw->spin); \ 89 assert(rw->readers >= 0); \ 90 ++(rw)->readers; \ 91 pthread_spin_unlock(&rw->spin); \ 92} while (/*CONSTCOND*/0) 93#define RURW_DECREAD(rw) \ 94do { \ 95 pthread_spin_lock(&rw->spin); \ 96 assert(rw->readers > 0); \ 97 --(rw)->readers; \ 98 pthread_spin_unlock(&rw->spin); \ 99} while (/*CONSTCOND*/0) 100 101struct rumpuser_rw { 102 pthread_rwlock_t pthrw; 103 pthread_spinlock_t spin; 104 int readers; 105 struct lwp *writer; 106}; 107 108struct rumpuser_cv { 109 pthread_cond_t pthcv; 110 int nwaiters; 111}; 112 113struct rumpuser_mtx rumpuser_aio_mtx; 114struct rumpuser_cv rumpuser_aio_cv; 115int rumpuser_aio_head, rumpuser_aio_tail; 116struct rumpuser_aio rumpuser_aios[N_AIOS]; 117 118kernel_lockfn rumpuser__klock; 119kernel_unlockfn rumpuser__kunlock; 120int rumpuser__wantthreads; 121 122void 123/*ARGSUSED*/ 124rumpuser_biothread(void *arg) 125{ 126 struct rumpuser_aio *rua; 127 rump_biodone_fn biodone = arg; 128 ssize_t rv; 129 int error, dummy; 130 131 /* unschedule from CPU. we reschedule before running the interrupt */ 132 rumpuser__kunlock(0, &dummy, NULL); 133 assert(dummy == 0); 134 135 NOFAIL_ERRNO(pthread_mutex_lock(&rumpuser_aio_mtx.pthmtx)); 136 for (;;) { 137 while (rumpuser_aio_head == rumpuser_aio_tail) { 138 NOFAIL_ERRNO(pthread_cond_wait(&rumpuser_aio_cv.pthcv, 139 &rumpuser_aio_mtx.pthmtx)); 140 } 141 142 rua = &rumpuser_aios[rumpuser_aio_tail]; 143 assert(rua->rua_bp != NULL); 144 pthread_mutex_unlock(&rumpuser_aio_mtx.pthmtx); 145 146 if (rua->rua_op & RUA_OP_READ) { 147 error = 0; 148 rv = pread(rua->rua_fd, rua->rua_data, 149 rua->rua_dlen, rua->rua_off); 150 if (rv < 0) { 151 rv = 0; 152 error = errno; 153 } 154 } else { 155 error = 0; 156 rv = pwrite(rua->rua_fd, rua->rua_data, 157 rua->rua_dlen, rua->rua_off); 158 if (rv < 0) { 159 rv = 0; 160 error = errno; 161 } else if (rua->rua_op & RUA_OP_SYNC) { 162#ifdef __NetBSD__ 163 fsync_range(rua->rua_fd, FDATASYNC, 164 rua->rua_off, rua->rua_dlen); 165#else 166 fsync(rua->rua_fd); 167#endif 168 } 169 } 170 rumpuser__klock(0, NULL); 171 biodone(rua->rua_bp, (size_t)rv, error); 172 rumpuser__kunlock(0, &dummy, NULL); 173 174 rua->rua_bp = NULL; 175 176 NOFAIL_ERRNO(pthread_mutex_lock(&rumpuser_aio_mtx.pthmtx)); 177 rumpuser_aio_tail = (rumpuser_aio_tail+1) % N_AIOS; 178 pthread_cond_signal(&rumpuser_aio_cv.pthcv); 179 } 180 181 /*NOTREACHED*/ 182 fprintf(stderr, "error: rumpuser_biothread reached unreachable\n"); 183 abort(); 184} 185 186void 187rumpuser_thrinit(kernel_lockfn lockfn, kernel_unlockfn unlockfn, int threads) 188{ 189 190 pthread_mutex_init(&rumpuser_aio_mtx.pthmtx, NULL); 191 pthread_cond_init(&rumpuser_aio_cv.pthcv, NULL); 192 193 pthread_key_create(&curlwpkey, NULL); 194 195 rumpuser__klock = lockfn; 196 rumpuser__kunlock = unlockfn; 197 rumpuser__wantthreads = threads; 198} 199 200#if 0 201void 202rumpuser__thrdestroy(void) 203{ 204 205 pthread_key_delete(curlwpkey); 206} 207#endif 208 209int 210rumpuser_thread_create(void *(*f)(void *), void *arg, const char *thrname, 211 int joinable, void **ptcookie) 212{ 213 pthread_t ptid; 214 pthread_t *ptidp; 215 pthread_attr_t pattr; 216 int rv; 217 218 if ((rv = pthread_attr_init(&pattr)) != 0) 219 return rv; 220 221 if (joinable) { 222 NOFAIL(ptidp = malloc(sizeof(*ptidp))); 223 pthread_attr_setdetachstate(&pattr, PTHREAD_CREATE_JOINABLE); 224 } else { 225 ptidp = &ptid; 226 pthread_attr_setdetachstate(&pattr, PTHREAD_CREATE_DETACHED); 227 } 228 229 rv = pthread_create(ptidp, &pattr, f, arg); 230#ifdef __NetBSD__ 231 if (rv == 0 && thrname) 232 pthread_setname_np(ptid, thrname, NULL); 233#endif 234 235 if (joinable) { 236 assert(ptcookie); 237 *ptcookie = ptidp; 238 } 239 240 pthread_attr_destroy(&pattr); 241 242 return rv; 243} 244 245__dead void 246rumpuser_thread_exit(void) 247{ 248 249 pthread_exit(NULL); 250} 251 252int 253rumpuser_thread_join(void *ptcookie) 254{ 255 pthread_t *pt = ptcookie; 256 int rv; 257 258 KLOCK_WRAP((rv = pthread_join(*pt, NULL))); 259 if (rv == 0) 260 free(pt); 261 262 return rv; 263} 264 265void 266rumpuser_mutex_init(struct rumpuser_mtx **mtx) 267{ 268 pthread_mutexattr_t att; 269 270 NOFAIL(*mtx = malloc(sizeof(struct rumpuser_mtx))); 271 272 pthread_mutexattr_init(&att); 273 pthread_mutexattr_settype(&att, PTHREAD_MUTEX_ERRORCHECK); 274 NOFAIL_ERRNO(pthread_mutex_init(&((*mtx)->pthmtx), &att)); 275 pthread_mutexattr_destroy(&att); 276 277 (*mtx)->owner = NULL; 278 (*mtx)->iskmutex = 0; 279} 280 281void 282rumpuser_mutex_init_kmutex(struct rumpuser_mtx **mtx) 283{ 284 285 rumpuser_mutex_init(mtx); 286 (*mtx)->iskmutex = 1; 287} 288 289static void 290mtxenter(struct rumpuser_mtx *mtx) 291{ 292 293 if (!mtx->iskmutex) 294 return; 295 296 assert(mtx->owner == NULL); 297 mtx->owner = rumpuser_get_curlwp(); 298} 299 300static void 301mtxexit(struct rumpuser_mtx *mtx) 302{ 303 304 if (!mtx->iskmutex) 305 return; 306 307 assert(mtx->owner != NULL); 308 mtx->owner = NULL; 309} 310 311void 312rumpuser_mutex_enter(struct rumpuser_mtx *mtx) 313{ 314 315 if (pthread_mutex_trylock(&mtx->pthmtx) != 0) 316 KLOCK_WRAP(NOFAIL_ERRNO(pthread_mutex_lock(&mtx->pthmtx))); 317 mtxenter(mtx); 318} 319 320void 321rumpuser_mutex_enter_nowrap(struct rumpuser_mtx *mtx) 322{ 323 324 NOFAIL_ERRNO(pthread_mutex_lock(&mtx->pthmtx)); 325 mtxenter(mtx); 326} 327 328int 329rumpuser_mutex_tryenter(struct rumpuser_mtx *mtx) 330{ 331 int rv; 332 333 rv = pthread_mutex_trylock(&mtx->pthmtx); 334 if (rv == 0) { 335 mtxenter(mtx); 336 } 337 338 return rv == 0; 339} 340 341void 342rumpuser_mutex_exit(struct rumpuser_mtx *mtx) 343{ 344 345 mtxexit(mtx); 346 NOFAIL_ERRNO(pthread_mutex_unlock(&mtx->pthmtx)); 347} 348 349void 350rumpuser_mutex_destroy(struct rumpuser_mtx *mtx) 351{ 352 353 NOFAIL_ERRNO(pthread_mutex_destroy(&mtx->pthmtx)); 354 free(mtx); 355} 356 357struct lwp * 358rumpuser_mutex_owner(struct rumpuser_mtx *mtx) 359{ 360 361 if (__predict_false(!mtx->iskmutex)) { 362 printf("panic: rumpuser_mutex_held unsupported on non-kmtx\n"); 363 abort(); 364 } 365 366 return mtx->owner; 367} 368 369void 370rumpuser_rw_init(struct rumpuser_rw **rw) 371{ 372 373 NOFAIL(*rw = malloc(sizeof(struct rumpuser_rw))); 374 NOFAIL_ERRNO(pthread_rwlock_init(&((*rw)->pthrw), NULL)); 375 NOFAIL_ERRNO(pthread_spin_init(&((*rw)->spin), PTHREAD_PROCESS_SHARED)); 376 (*rw)->readers = 0; 377 (*rw)->writer = NULL; 378} 379 380void 381rumpuser_rw_enter(struct rumpuser_rw *rw, int iswrite) 382{ 383 384 if (iswrite) { 385 if (pthread_rwlock_trywrlock(&rw->pthrw) != 0) 386 KLOCK_WRAP(NOFAIL_ERRNO( 387 pthread_rwlock_wrlock(&rw->pthrw))); 388 RURW_SETWRITE(rw); 389 } else { 390 if (pthread_rwlock_tryrdlock(&rw->pthrw) != 0) 391 KLOCK_WRAP(NOFAIL_ERRNO( 392 pthread_rwlock_rdlock(&rw->pthrw))); 393 RURW_INCREAD(rw); 394 } 395} 396 397int 398rumpuser_rw_tryenter(struct rumpuser_rw *rw, int iswrite) 399{ 400 int rv; 401 402 if (iswrite) { 403 rv = pthread_rwlock_trywrlock(&rw->pthrw); 404 if (rv == 0) 405 RURW_SETWRITE(rw); 406 } else { 407 rv = pthread_rwlock_tryrdlock(&rw->pthrw); 408 if (rv == 0) 409 RURW_INCREAD(rw); 410 } 411 412 return rv == 0; 413} 414 415void 416rumpuser_rw_exit(struct rumpuser_rw *rw) 417{ 418 419 if (RURW_HASREAD(rw)) 420 RURW_DECREAD(rw); 421 else 422 RURW_CLRWRITE(rw); 423 NOFAIL_ERRNO(pthread_rwlock_unlock(&rw->pthrw)); 424} 425 426void 427rumpuser_rw_destroy(struct rumpuser_rw *rw) 428{ 429 430 NOFAIL_ERRNO(pthread_rwlock_destroy(&rw->pthrw)); 431 NOFAIL_ERRNO(pthread_spin_destroy(&rw->spin)); 432 free(rw); 433} 434 435int 436rumpuser_rw_held(struct rumpuser_rw *rw) 437{ 438 439 return rw->readers != 0; 440} 441 442int 443rumpuser_rw_rdheld(struct rumpuser_rw *rw) 444{ 445 446 return RURW_HASREAD(rw); 447} 448 449int 450rumpuser_rw_wrheld(struct rumpuser_rw *rw) 451{ 452 453 return RURW_AMWRITER(rw); 454} 455 456void 457rumpuser_cv_init(struct rumpuser_cv **cv) 458{ 459 460 NOFAIL(*cv = malloc(sizeof(struct rumpuser_cv))); 461 NOFAIL_ERRNO(pthread_cond_init(&((*cv)->pthcv), NULL)); 462 (*cv)->nwaiters = 0; 463} 464 465void 466rumpuser_cv_destroy(struct rumpuser_cv *cv) 467{ 468 469 NOFAIL_ERRNO(pthread_cond_destroy(&cv->pthcv)); 470 free(cv); 471} 472 473void 474rumpuser_cv_wait(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx) 475{ 476 int nlocks; 477 478 cv->nwaiters++; 479 rumpuser__kunlock(0, &nlocks, mtx); 480 mtxexit(mtx); 481 NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx)); 482 mtxenter(mtx); 483 rumpuser__klock(nlocks, mtx); 484 cv->nwaiters--; 485} 486 487void 488rumpuser_cv_wait_nowrap(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx) 489{ 490 491 cv->nwaiters++; 492 mtxexit(mtx); 493 NOFAIL_ERRNO(pthread_cond_wait(&cv->pthcv, &mtx->pthmtx)); 494 mtxenter(mtx); 495 cv->nwaiters--; 496} 497 498int 499rumpuser_cv_timedwait(struct rumpuser_cv *cv, struct rumpuser_mtx *mtx, 500 int64_t sec, int64_t nsec) 501{ 502 struct timespec ts; 503 int rv, nlocks; 504 505 /* LINTED */ 506 ts.tv_sec = sec; ts.tv_nsec = nsec; 507 508 cv->nwaiters++; 509 rumpuser__kunlock(0, &nlocks, mtx); 510 mtxexit(mtx); 511 rv = pthread_cond_timedwait(&cv->pthcv, &mtx->pthmtx, &ts); 512 mtxenter(mtx); 513 rumpuser__klock(nlocks, mtx); 514 cv->nwaiters--; 515 if (rv != 0 && rv != ETIMEDOUT) 516 abort(); 517 518 return rv == ETIMEDOUT; 519} 520 521void 522rumpuser_cv_signal(struct rumpuser_cv *cv) 523{ 524 525 NOFAIL_ERRNO(pthread_cond_signal(&cv->pthcv)); 526} 527 528void 529rumpuser_cv_broadcast(struct rumpuser_cv *cv) 530{ 531 532 NOFAIL_ERRNO(pthread_cond_broadcast(&cv->pthcv)); 533} 534 535int 536rumpuser_cv_has_waiters(struct rumpuser_cv *cv) 537{ 538 539 return cv->nwaiters; 540} 541 542/* 543 * curlwp 544 */ 545 546void 547rumpuser_set_curlwp(struct lwp *l) 548{ 549 550 assert(pthread_getspecific(curlwpkey) == NULL || l == NULL); 551 pthread_setspecific(curlwpkey, l); 552} 553 554struct lwp * 555rumpuser_get_curlwp(void) 556{ 557 558 return pthread_getspecific(curlwpkey); 559} 560