source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resourcewatcher.h"
   8
   9 using namespace std;
  10
  11 namespace Msp {
  12 namespace GL {
  13
  14 resource_load_error::resource_load_error(const string &name, const string &err):
  15         runtime_error(format("%s: %s", name, err))
  16 { }
  17
  18 resource_load_error::resource_load_error(const string &name, const exception &exc):
  19         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  20 { }
  21
  22
  23 ResourceManager::ResourceManager():
  24         policy(LOAD_ON_DEMAND),
  25         async_loads(true),
  26         size_limit(0),
  27         frame(0),
  28         min_retain_frames(30),
  29         max_retain_frames(0),
  30         next_unload(0)
  31 { }
  32
  33 ResourceManager::~ResourceManager()
  34 {
  35         thread.terminate();
  36
  37         while(!resources.empty())
  38                 resources.begin()->second.resource->set_manager(0);
  39 }
  40
  41 void ResourceManager::set_loading_policy(LoadingPolicy p)
  42 {
  43         policy = p;
  44 }
  45
  46 void ResourceManager::set_async_loads(bool a)
  47 {
  48         async_loads = a;
  49 }
  50
  51 void ResourceManager::set_size_limit(UInt64 s)
  52 {
  53         size_limit = s;
  54 }
  55
  56 void ResourceManager::set_min_retain_frames(unsigned f)
  57 {
  58         min_retain_frames = f;
  59 }
  60
  61 void ResourceManager::set_max_retain_frames(unsigned f)
  62 {
  63         max_retain_frames = f;
  64 }
  65
  66 void ResourceManager::add_resource(Resource &r)
  67 {
  68         insert_unique(resources, &r, ManagedResource(r));
  69 }
  70
  71 void *ResourceManager::get_data_for_resource(const Resource &r)
  72 {
  73         return &get_item(resources, &r);
  74 }
  75
  76 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  77 {
  78         set_resource_location(r, ResourceLocation(c, n));
  79 }
  80
  81 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  82 {
  83         ManagedResource &managed = get_item(resources, &r);
  84         managed.location = l;
  85
  86         if(policy==LOAD_IMMEDIATELY)
  87                 load_resource(r);
  88 }
  89
  90 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
  91 {
  92         const ManagedResource &managed = get_item(resources, &r);
  93         return managed.location.collection ? &managed.location : 0;
  94 }
  95
  96 void ResourceManager::load_resource(Resource &r)
  97 {
  98         ManagedResource &managed = get_item(resources, &r);
  99         if(!managed.location.collection)
 100                 throw runtime_error("no location");
 101
 102         if(managed.state!=ManagedResource::NOT_LOADED)
 103                 return;
 104
 105         if(async_loads)
 106         {
 107                 managed.state = ManagedResource::LOAD_QUEUED;
 108                 queue.push_back(&managed);
 109         }
 110         else
 111         {
 112                 managed.start_loading();
 113                 while(!managed.loader->process()) ;
 114                 managed.finish_loading(true);
 115         }
 116 }
 117
 118 bool ResourceManager::is_resource_loaded(const Resource &r) const
 119 {
 120         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 121         return managed ? managed->state==ManagedResource::LOADED : false;
 122 }
 123
 124 void ResourceManager::resource_used(const Resource &r)
 125 {
 126         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 127         if(!managed)
 128                 return;
 129         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 130                 load_resource(*managed->resource);
 131
 132         managed->last_used = frame;
 133         if(max_retain_frames && !next_unload)
 134                 next_unload = frame+max_retain_frames+1;
 135 }
 136
 137 void ResourceManager::remove_resource(Resource &r)
 138 {
 139         ManagedResource &managed = get_item(resources, &r);
 140         ManagedResource::State state = managed.state;
 141         if(state==ManagedResource::LOAD_QUEUED)
 142         {
 143                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 144                 if(i!=queue.end())
 145                         queue.erase(i);
 146         }
 147         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 148                 thread.remove_resource(managed);
 149         remove_existing(resources, &r);
 150 }
 151
 152 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 153 {
 154         get_item(resources, &r).add_watcher(w);
 155 }
 156
 157 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 158 {
 159         get_item(resources, &r).remove_watcher(w);
 160 }
 161
 162 void ResourceManager::tick()
 163 {
 164         ++frame;
 165
 166         bool do_unload = (frame>=next_unload);
 167         if(thread.sync())
 168                 do_unload = true;
 169
 170         if(thread.needs_work() && !queue.empty())
 171                 dispatch_work();
 172
 173         if(do_unload)
 174         {
 175                 if(max_retain_frames && frame>=next_unload)
 176                 {
 177                         unload_by_age();
 178
 179                         next_unload = frame;
 180                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 181                                 if(i->second.state==ManagedResource::LOADED)
 182                                         next_unload = min(next_unload, i->second.last_used);
 183                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 184                 }
 185
 186                 if(size_limit)
 187                         unload_by_size();
 188         }
 189 }
 190
 191 void ResourceManager::dispatch_work()
 192 {
 193         queue.sort(age_order);
 194
 195         if(queue.front()->last_used+10<frame)
 196         {
 197                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 198                         (*i)->state = ManagedResource::NOT_LOADED;
 199                 queue.clear();
 200                 return;
 201         }
 202
 203         while(thread.needs_work() && !queue.empty())
 204         {
 205                 ManagedResource *managed = queue.front();
 206                 queue.pop_front();
 207                 thread.add_resource(*managed);
 208         }
 209 }
 210
 211 void ResourceManager::unload_by_age()
 212 {
 213         unsigned unload_limit = frame-max_retain_frames;
 214
 215         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 216                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 217                         i->second.unload();
 218 }
 219
 220 void ResourceManager::unload_by_size()
 221 {
 222         unsigned unload_limit = frame-min_retain_frames;
 223
 224         while(get_total_data_size()>size_limit)
 225         {
 226                 ManagedResource *best = 0;
 227                 UInt64 best_impact = 0;
 228                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 229                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 230                         {
 231                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 232                                 if(!best || impact>best_impact)
 233                                 {
 234                                         best = &i->second;
 235                                         best_impact = impact;
 236                                 }
 237                         }
 238
 239                 if(!best)
 240                         break;
 241
 242                 best->unload();
 243         }
 244 }
 245
 246 UInt64 ResourceManager::get_total_data_size() const
 247 {
 248         UInt64 total = 0;
 249         for(ResourceMap::const_iterator i=resources.begin(); i!=resources.end(); ++i)
 250                 if(i->second.state==ManagedResource::LOADED)
 251                         total += i->second.data_size;
 252         return total;
 253 }
 254
 255 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 256 {
 257         return mr1->last_used>mr2->last_used;
 258 }
 259
 260
 261 ResourceManager::ResourceLocation::ResourceLocation():
 262         collection(0)
 263 { }
 264
 265 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 266         collection(&c),
 267         name(n)
 268 { }
 269
 270
 271 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 272         resource(&r),
 273         io(0),
 274         loader(0),
 275         state(NOT_LOADED),
 276         last_used(0),
 277         data_size(0)
 278 { }
 279
 280 void ResourceManager::ManagedResource::start_loading()
 281 {
 282         io = location.collection->open_raw(location.name);
 283         if(!io)
 284                 throw resource_load_error(location.name, "open failed");
 285
 286         loader = resource->load(*io);
 287         if(!loader)
 288         {
 289                 delete io;
 290                 io = 0;
 291                 throw logic_error("no loader created");
 292         }
 293         state = LOADING;
 294 }
 295
 296 bool ResourceManager::ManagedResource::process(bool sync)
 297 {
 298         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 299                 if(loader->process())
 300                         state = LOAD_FINISHED;
 301
 302         return state==LOAD_FINISHED;
 303 }
 304
 305 void ResourceManager::ManagedResource::finish_loading(bool successful)
 306 {
 307         delete loader;
 308         loader = 0;
 309         delete io;
 310         io = 0;
 311
 312         if(successful)
 313         {
 314                 state = LOADED;
 315                 data_size = resource->get_data_size();
 316
 317                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 318                         (*i)->resource_loaded(*resource);
 319         }
 320         else
 321         {
 322                 resource->unload();
 323                 state = NOT_LOADED;
 324         }
 325 }
 326
 327 void ResourceManager::ManagedResource::finish_loading()
 328 {
 329         finish_loading(state==LOAD_FINISHED);
 330 }
 331
 332 void ResourceManager::ManagedResource::unload()
 333 {
 334         resource->unload();
 335         state = NOT_LOADED;
 336
 337         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 338                 (*i)->resource_unloaded(*resource);
 339 }
 340
 341 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 342 {
 343         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 344                 watchers.push_back(&w);
 345 }
 346
 347 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 348 {
 349         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 350         if(end!=watchers.end())
 351                 watchers.erase(end, watchers.end());
 352 }
 353
 354
 355 ResourceManager::LoadingThread::LoadingThread():
 356         sem(1),
 357         capacity(2),
 358         size(0),
 359         done(false)
 360 {
 361         launch();
 362 }
 363
 364 void ResourceManager::LoadingThread::main()
 365 {
 366         bool wait_for_work = false;
 367         while(!done)
 368         {
 369                 if(wait_for_work)
 370                         sem.wait();
 371
 372                 if(ManagedResource *managed = front(async_queue))
 373                 {
 374                         try
 375                         {
 376                                 managed->process(false);
 377                         }
 378                         catch(const exception &e)
 379                         {
 380                                 MutexLock lock(queue_mutex);
 381                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 382                                 managed->state = ManagedResource::LOAD_ERROR;
 383                         }
 384
 385                         MutexLock lock(queue_mutex);
 386                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 387                         wait_for_work = async_queue.empty();
 388                 }
 389                 else
 390                         wait_for_work = true;
 391         }
 392 }
 393
 394 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 395 {
 396         MutexLock lock(queue_mutex);
 397         if(queue.empty())
 398                 return 0;
 399
 400         return queue.front();
 401 }
 402
 403 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 404 {
 405         r.start_loading();
 406
 407         MutexLock lock(queue_mutex);
 408         if(r.loader->needs_sync())
 409                 sync_queue.push_back(&r);
 410         else
 411         {
 412                 bool was_empty = async_queue.empty();
 413                 async_queue.push_back(&r);
 414                 if(was_empty)
 415                         sem.signal();
 416         }
 417
 418         ++size;
 419 }
 420
 421 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 422 {
 423         while(!try_remove_resource(r))
 424                 Time::sleep(Time::msec);
 425
 426         r.finish_loading();
 427 }
 428
 429 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 430 {
 431         MutexLock lock(queue_mutex);
 432
 433         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 434         if(i==async_queue.end())
 435         {
 436                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 437                 if(i!=sync_queue.end())
 438                         sync_queue.erase(i);
 439         }
 440         else if(i==async_queue.begin())
 441                 return false;
 442         else
 443                 async_queue.erase(i);
 444
 445         return true;
 446 }
 447
 448 bool ResourceManager::LoadingThread::sync()
 449 {
 450         {
 451                 MutexLock lock(queue_mutex);
 452
 453                 if(!error_queue.empty())
 454                 {
 455                         resource_load_error err = error_queue.front();
 456                         error_queue.pop_front();
 457                         throw err;
 458                 }
 459
 460                 unsigned async_size = async_queue.size();
 461                 if(async_size==0 && size==capacity)
 462                         ++capacity;
 463                 else if(async_size>2 && capacity>2)
 464                         --capacity;
 465         }
 466
 467         bool any_finished = false;
 468         while(ManagedResource *managed = front(sync_queue))
 469         {
 470                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 471                 {
 472                         managed->finish_loading();
 473                         any_finished = true;
 474                         --size;
 475
 476                         MutexLock lock(queue_mutex);
 477                         sync_queue.pop_front();
 478                 }
 479                 else
 480                 {
 481                         MutexLock lock(queue_mutex);
 482                         bool was_empty = async_queue.empty();
 483                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 484                         if(was_empty)
 485                                 sem.signal();
 486                 }
 487         }
 488
 489         return any_finished;
 490 }
 491
 492 void ResourceManager::LoadingThread::terminate()
 493 {
 494         done = true;
 495         sem.signal();
 496         join();
 497         async_queue.clear();
 498         sync_queue.clear();
 499 }
 500
 501 } // namespace GL
 502 } // namespace Msp