syncservice.path Date: Wed May 18 11:59:18 EST 2005 Author: Jeremy Kerr Title: Implement a fsync daemon to flush file-backed pages every 30s Status: Development Index: lib/libc/io/FileLinuxFile.C =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/lib/libc/io/FileLinuxFile.C,v retrieving revision 1.136 diff -u -u -r1.136 FileLinuxFile.C --- lib/libc/io/FileLinuxFile.C 15 Apr 2005 17:39:34 -0000 1.136 +++ lib/libc/io/FileLinuxFile.C 18 May 2005 02:13:49 -0000 @@ -625,8 +625,10 @@ if (_FAILURE(rc)) return (_SCLSCD(rc) == 1) ? 0 : rc; } +#ifndef ENABLE_SYNCSERVICE SysStatus rc = locked_flush(); tassertMsg(_SUCCESS(rc), "?"); +#endif delete buffer; Index: os/kernel/ObjectRefsKern.H =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/os/kernel/ObjectRefsKern.H,v retrieving revision 1.24 diff -u -u -r1.24 ObjectRefsKern.H --- os/kernel/ObjectRefsKern.H 11 Jul 2004 21:59:26 -0000 1.24 +++ os/kernel/ObjectRefsKern.H 18 May 2005 02:13:51 -0000 @@ -38,6 +38,7 @@ class FSSwap; class KernelInfoMgr; class HWPerfMon; +class SyncService; /* internal references to kernel objects */ typedef HAT **HATRef; @@ -53,6 +54,7 @@ typedef FSSwap **FSSwapRef; typedef KernelInfoMgr **KernelInfoMgrRef; typedef HWPerfMon **HWPerfMonRef; +typedef SyncService **SyncServiceRef; #define GOBJK(OBJ) (CObjGlobalsKern::OBJ()) #define DREFGOBJK(OBJ) (DREF(GOBJK(OBJ))) Index: os/kernel/bilge/CObjGlobalsKern.H =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/os/kernel/bilge/CObjGlobalsKern.H,v retrieving revision 1.1 diff -u -u -r1.1 CObjGlobalsKern.H --- os/kernel/bilge/CObjGlobalsKern.H 11 Jul 2004 21:59:26 -0000 1.1 +++ os/kernel/bilge/CObjGlobalsKern.H 18 May 2005 02:13:51 -0000 @@ -33,7 +33,8 @@ KernelInfoMgrIndex =FSSwapIndex+1, ResourceManagerIndex =KernelInfoMgrIndex+1, HWPerfMonIndex =ResourceManagerIndex+1, - numReservedEntriesKern =HWPerfMonIndex+1 + SyncServiceIndex =HWPerfMonIndex+1, + numReservedEntriesKern =SyncServiceIndex+1 }; // just type cheat the normal application's process ref @@ -67,6 +68,9 @@ static HWPerfMonRef TheHWPerfMonRef() {return (HWPerfMonRef)COSMgr::indexToRef(HWPerfMonIndex);} + + static SyncServiceRef TheSyncServiceRef() + {return (SyncServiceRef)COSMgr::indexToRef(SyncServiceIndex);} static FileLinuxRef TheConsoleRef() { return (FileLinuxRef)COSMgr::indexToRef(ConsoleIndex);} Index: os/kernel/defines/paging.H =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/os/kernel/defines/paging.H,v retrieving revision 1.19 diff -u -u -r1.19 paging.H --- os/kernel/defines/paging.H 13 Nov 2004 21:07:40 -0000 1.19 +++ os/kernel/defines/paging.H 18 May 2005 02:13:51 -0000 @@ -70,4 +70,9 @@ #define LARGE_PAGES_NON_PAGEABLE #endif +/** + * Enables the SyncService daemon, which flushes FCMFiles to disk every 30s + */ +#define ENABLE_SYNCSERVICE + #endif /* #ifndef __PAGING_H_ */ Index: os/kernel/init/KernelInit.C =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/os/kernel/init/KernelInit.C,v retrieving revision 1.577 diff -u -u -r1.577 KernelInit.C --- os/kernel/init/KernelInit.C 24 Apr 2005 03:32:59 -0000 1.577 +++ os/kernel/init/KernelInit.C 18 May 2005 02:13:53 -0000 @@ -61,6 +61,7 @@ #include "mem/HardwareSpecificRegions.H" #include "mem/RegionRedZone.H" #include "mem/PMRoot.H" +#include "mem/SyncService.H" #include "misc/linkage.H" #include "bilge/FSRamSwap.H" #include "bilge/TestSwitch.H" @@ -1851,6 +1853,10 @@ NetDev::ClassInit(vp); KernelPagingTransportPA::ClassInit(vp); KernelPagingTransportVA::ClassInit(vp); + +#ifdef ENABLE_SYNCSERVICE + SyncService::ClassInit(vp); +#endif /* ENABLE_SYNCSERVICE */ ProcessSetKern::ClassInit(vp); // maintains info about processes Index: os/kernel/mem/FCMFile.C =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/os/kernel/mem/FCMFile.C,v retrieving revision 1.1 diff -u -u -r1.1 FCMFile.C --- os/kernel/mem/FCMFile.C 20 Oct 2004 18:10:29 -0000 1.1 +++ os/kernel/mem/FCMFile.C 18 May 2005 02:13:53 -0000 @@ -15,36 +15,48 @@ #include "kernIncs.H" #include "FCMFile.H" #include "mem/FCMDefaultMultiRep.H" +#include "mem/SyncService.H" #include #include /* static */ SysStatus FCMFile::CreateDefault(FCMRef &ref, FRRef frRefArg, - uval pageable, uval preFetchPages, - uval maxPages) { + uval pageable, uval preFetchPages, + uval maxPages) { SysStatus rc; if (KernelInfo::ControlFlagIsSet(KernelInfo::USE_MULTI_REP_FCMS)) { rc = FCMDefaultMultiRep::Create(ref, frRefArg, pageable); } else { - rc = FCMFile::Create(ref, frRefArg, pageable, preFetchPages, - maxPages); + rc = FCMFile::Create(ref, frRefArg, pageable, preFetchPages, + maxPages); } return rc; } /* static */ SysStatus FCMFile::Create(FCMRef &ref, FRRef frRefArg, uval pageable, - uval preFetchPages, uval maxPages) + uval preFetchPages, uval maxPages) { SysStatus rc; FCMFile *fcm; - fcm = new FCMFile; if (fcm == NULL) return -1; rc = fcm->init(ref, frRefArg, PAGE_SIZE, pageable, 0, preFetchPages, maxPages); + +#ifdef ENABLE_SYNCSERVICE + /** @todo Orran - we shouldn't need to detect this. It looks like we're + * getting FRPlaceHolders backing FCMFiles, which cause problems + * when the SyncService calls fsync(). -jk */ + if (pageable) { + DREFGOBJK(TheSyncServiceRef)->attachFCM(ref); + } else { + err_printf("Didn't attach FCMFile to SyncService - not pageable\n"); + } +#endif /* ENABLE_SYNCSERVICE */ + TraceOSMemFCMDefaultCreate((uval)ref); return rc; } @@ -84,6 +96,19 @@ // if doingIO then is already being written back if (pg->dirty && !pg->doingIO) { if (numCollected >= listSize) break; + /* + * FIXME: in the non-force case, we should not be + * paying the cost to unmap at all, we should only + * unmap when the app explicitly fsyncs... + * + * Linux semantics are actually don't unmap it. + * So... we could do the same, but currently we have + * an invarient in K42 that a page that is doing I/O + * is not mapped. Its not clear which is better, since + * we avoid potential extra writes linux must do, since + * by not unmapping its not unsetting dirty bit in the + * page tables. + */ if (pg->mapped) { unmapPage(pg); wasmapped = 1; @@ -112,6 +137,10 @@ for (i = 0; i < numCollected; i++) { // err_printf("startput for %lx/%lx - %ld\n", // paddrList[i], offsetList[i], i); + /* + * FIXME: we should queue up our IORestartRequest + * and resume the fsync when we get called back + */ rc = DREF(frRef)->startPutPage(paddrList[i], offsetList[i]); // what should we do on errors; we need to at least remove doingIO // flag and wake up anyone waiting @@ -130,3 +159,11 @@ return 0; } +SysStatus +FCMFile::doDestroy() +{ +#ifdef ENABLE_SYNCSERVICE + DREFGOBJK(TheSyncServiceRef)->detachFCM(getRef()); +#endif /* ENABLE_SYNCSERVICE */ + return FCMDefault::doDestroy(); +} Index: os/kernel/mem/FCMFile.H =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/os/kernel/mem/FCMFile.H,v retrieving revision 1.1 diff -u -u -r1.1 FCMFile.H --- os/kernel/mem/FCMFile.H 20 Oct 2004 18:10:29 -0000 1.1 +++ os/kernel/mem/FCMFile.H 18 May 2005 02:13:54 -0000 @@ -10,16 +10,16 @@ * * $Id: FCMFile.H,v 1.1 2004/10/20 18:10:29 okrieg Exp $ *****************************************************************************/ -/***************************************************************************** - * Module Description: Basic FCM for mapping a file representative. - * **************************************************************************/ +/** + * @class FCMFile + * FCM that is backed by a real file. Has to have a real + * implementation of fsync... + */ #include "mem/FCMDefault.H" +class FCMFile; +typedef FCMFile** FCMFileRef; -/// FCM backed by a real file -/** FCM that is backed by a real file. Has to have a real - * implementation of fsync... - */ class FCMFile : public FCMDefault { public: @@ -32,6 +32,11 @@ uval maxPages = FCM_MAX_NUMPAGES); virtual SysStatus fsync(uval force); + + virtual SysStatus doDestroy(); + + DEFINE_REFS(FCMFile); }; + #endif /* #ifndef __FCMDEFAULTFILE_H_ */ Index: os/kernel/mem/Makefile =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/os/kernel/mem/Makefile,v retrieving revision 1.100 diff -u -u -r1.100 Makefile --- os/kernel/mem/Makefile 2 Mar 2005 05:27:57 -0000 1.100 +++ os/kernel/mem/Makefile 18 May 2005 02:13:54 -0000 @@ -62,6 +62,7 @@ SharedBufferProducer.C \ KernelPagingTransport.C KernelPagingTransportPA.C \ KernelPagingTransportVA.C PMLeafExp.C \ + SyncService.C \ $(SERVE:%=X%.C) #FIXME BIG Workaround to fix the compiler Index: os/kernel/mem/SyncService.C =================================================================== RCS file: os/kernel/mem/SyncService.C diff -N os/kernel/mem/SyncService.C --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ os/kernel/mem/SyncService.C 18 May 2005 02:13:54 -0000 @@ -0,0 +1,142 @@ +/****************************************************************************** + * K42: (C) Copyright IBM Corp. 2000-2003. + * All Rights Reserved + * + * This file is distributed under the GNU LGPL. You should have + * received a copy of the license along with K42; see the file LICENSE.html + * in the top-level directory for more details. + * + * $Id: PMRoot.C,v 1.59 2004/12/14 18:54:55 butrico Exp $ + *****************************************************************************/ +#include "kernIncs.H" +#include "SyncService.H" +#include "cobj/sys/COSMgrObject.H" + +/** + * Root class for distributed SyncService objects + */ +class SyncService::MyRoot : public CObjRootMultiRepPinned { +protected: + SyncService *reps[Scheduler::VPLimit]; +public: + virtual CObjRep * createRep(VPNum vp) { + reps[vp] = new SyncService(vp); + tassert(reps[vp], + err_printf("No memory for SyncSerivce rep on vp %ld", vp)); + return reps[vp]; + } + + virtual SyncService *locateRep(FCMFileRef fcm); + + DEFINE_PINNEDGLOBALPADDED_NEW(SyncService::MyRoot); + MyRoot(RepRef rep); +}; + +SyncService::MyRoot::MyRoot(RepRef rep) + : CObjRootMultiRepPinned(rep) +{ + memset(reps, 0, sizeof(*reps)); +} + +SyncService * +SyncService::MyRoot::locateRep(FCMFileRef fcm) +{ + VPNum vp = COSMgrObject::refToVP((CORef)fcm); + return reps[vp]; +} + +SyncService::SyncService(VPNum vp) +{ + fcmHashLock.init(); + timer.start(); +} + + +/**************************************************************************** +* Given that this is a kernel system wide object we instantiate all +* reps when the system is started this allows us to not worry about +* the rep set changing during the life time of the object (= system) +* Further we do not need to worry about destruction issues as it should +* never go away. Note there are things that could break these assumptions: +* Hot swapping of Hardware +* Rep migration / restructuring +*****************************************************************************/ +/* static */ void +SyncService::ClassInit(VPNum vp) +{ + if (vp == 0) { + MyRoot *myRoot; + + myRoot = new MyRoot((RepRef)(GOBJK(TheSyncServiceRef))); + passert(myRoot!=NULL, err_printf("No mem for SyncService\n")); + } + /* To make life easier we instantiate all reps on all vp at startup + * by making an external call to the single instance + */ + DREF(((SyncService **)GOBJK(TheSyncServiceRef)))->establishRep(); +} + +SysStatus +SyncService::attachFCM(FCMFileRef fcm) +{ + SyncService *rep = COGLOBAL(locateRep(fcm)); + tassert(rep, err_printf("unable to locate a rep for fcm=%p\n", fcm)); + return rep->repAttachFCM(fcm); +} + +SysStatus +SyncService::detachFCM(FCMFileRef fcm) +{ + SyncService *rep = COGLOBAL(locateRep(fcm)); + tassert(rep, err_printf("Unable to locate a rep for fcm=%p\n", fcm)); + return rep->repDetachFCM(fcm); +} + +SysStatus +SyncService::repAttachFCM(FCMFileRef fcm) +{ + uval tmp = 0; + AutoLock al(&fcmHashLock); // locks now, unlocks on return + tassert(!fcmHash.find(fcm, tmp), err_printf("re-attach\n")); + fcmHash.add(fcm, tmp); + numFCM++; + return 0; +} + +SysStatus +SyncService::repDetachFCM(FCMFileRef fcm) +{ + uval tmp; + AutoLock al(&fcmHashLock); // locks now, unlocks on return + if (fcmHash.remove(fcm, tmp)) { + numFCM--; + return 0; + } + tassert(0, err_printf("No FCM (%p) to detach from SyncService\n", fcm)); + /** @todo: correct error code? */ + return 0; +} + +/* static */ void +SyncService::HandleTimer(uval p) +{ + DREFGOBJK(TheSyncServiceRef)->handleTimer(); +} + +void +SyncService::handleTimer() +{ + syncFCMs(); + timer.start(); +} + +void +SyncService::syncFCMs() +{ + FCMFileRef fcm = NULL; + uval tmp; + SysStatus rc; + + for (rc=fcmHash.getFirst(fcm, tmp); rc; rc=fcmHash.getNextWithFF(fcm, tmp)) + DREF(fcm)->fsync(0); +} Index: os/kernel/mem/SyncService.H =================================================================== RCS file: os/kernel/mem/SyncService.H diff -N os/kernel/mem/SyncService.H --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ os/kernel/mem/SyncService.H 18 May 2005 02:13:54 -0000 @@ -0,0 +1,125 @@ +#ifndef __SYNCSERVICE_H_ +#define __SYNCSERVICE_H_ +/****************************************************************************** + * K42: (C) Copyright IBM Corp. 2005 + * All Rights Reserved + * + * This file is distributed under the GNU LGPL. You should have + * received a copy of the license along with K42; see the file LICENSE.html + * in the top-level directory for more details. + * + * $Id: PMRoot.H,v 1.34 2004/12/14 18:54:55 butrico Exp $ + *****************************************************************************/ +/***************************************************************************** + * @class SyncService + * FCMFile register with this class to have themselves called periodically + * for syncing. + * **************************************************************************/ + +#include +#include +#include "mem/FCMFile.H" + +class SyncService : public Obj { + typedef BLock LockType; + class MyRoot; + class TimerEventSS; + friend class MyRoot; + + MyRoot* root() {return (MyRoot*)(myRoot);} + + /** protects the FCMHash */ + LockType fcmHashLock; + + /** hash of the FCMs that are attached to this SyncService */ + HashSimple fcmHash; + + /** count of attached FCMs */ + uval numFCM; + + /** Called when the timer has expired */ + static void HandleTimer(uval p); + void handleTimer(); + + /** + * A implementation of TimerEvent to trigger the regular fsync + */ + class SyncService::TimerEventSS : protected TimerEvent { + /** number of seconds between syncs */ + static const uval SYNC_INTERVAL = 30; + public: + DEFINE_LOCALSTRICT_NEW(TimerEventSS); + /** + * Called when the timer expires + */ + virtual void handleEvent() { + SysStatus rc = + Scheduler::DisabledScheduleFunction(SyncService::HandleTimer, + (uval)0); + tassert(_SUCCESS(rc), err_printf("oops\n")); + } + /** + * Start the timer to expire in SYNC_INTERVAL seconds + */ + void start() { + scheduleEvent(SYNC_INTERVAL * Scheduler::TicksPerSecond(), + TimerEvent::relative); + } + }; + + /** + * The TimerEvent that will call us every 30s + */ + TimerEventSS timer; + + /** + * Construct a SyncService representative on the specified vp + * @param vp the vp number of this representative + */ + SyncService(VPNum vp); + +public: + DEFINE_PINNEDGLOBALPADDED_NEW(SyncService); + + static void ClassInit(VPNum vp); + + /** + * null function called to instantiate the rep + */ + virtual SysStatus establishRep() { return 0; } + + /** + * Attach a FCM (FCMFile) to the SyncService. This will find an appropriate + * SyncService rep and call it to do the attaching + * @param fcm The FCMFile to attach + */ + virtual SysStatus attachFCM(FCMFileRef fcm); + + /** + * Attach a FCM (FCMFile) to the SyncService + * @param fcm The FCMFile to attach + */ + virtual SysStatus detachFCM(FCMFileRef fcm); + +protected: + /** + * Do the actual attachment. The fcm must exist on the same VP as the + * SyncService + * @param fcm The FCMFile to attach + */ + virtual SysStatus repAttachFCM(FCMFileRef fcm); + + /** + * Do the actual detach. The fcm must exist on the same VP as the + * SyncService + * @param fcm The FCMFile to attach + */ + virtual SysStatus repDetachFCM(FCMFileRef fcm); + + /** + * Call fsync on each of the attached FCMs + */ + virtual void syncFCMs(); +}; + +#endif /* #ifndef __SYNCSERVICE_H_ */ Index: lib/libc/misc/HashSimple.C =================================================================== RCS file: /u/kitchawa/cvsroot/kitch-core/lib/libc/misc/HashSimple.C,v retrieving revision 1.14 diff -u -r1.14 HashSimple.C --- lib/libc/misc/HashSimple.C 8 Oct 2004 21:40:06 -0000 1.14 +++ lib/libc/misc/HashSimple.C 19 May 2005 03:48:49 -0000 @@ -500,6 +500,7 @@ template class HashSimpleBase; template class HashSimpleBase; template class HashSimpleLockedBase; +template class HashSimpleBase; template class HashSimpleBase; template class HashSimpleLockedBase; template class HashSimpleBase;