source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resources.h"
   8 #include "resourcewatcher.h"
   9
  10 using namespace std;
  11
  12 namespace Msp {
  13 namespace GL {
  14
  15 resource_load_error::resource_load_error(const string &name, const string &err):
  16         runtime_error(format("%s: %s", name, err))
  17 { }
  18
  19 resource_load_error::resource_load_error(const string &name, const exception &exc):
  20         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  21 { }
  22
  23
  24 ResourceManager::ResourceManager():
  25         policy(LOAD_ON_DEMAND),
  26         async_loads(true),
  27         size_limit(0),
  28         frame(0),
  29         min_retain_frames(30),
  30         max_retain_frames(0),
  31         next_unload(0)
  32 { }
  33
  34 ResourceManager::~ResourceManager()
  35 {
  36         thread.terminate();
  37
  38         while(!resources.empty())
  39                 resources.begin()->second.resource->set_manager(0);
  40 }
  41
  42 void ResourceManager::set_loading_policy(LoadingPolicy p)
  43 {
  44         policy = p;
  45 }
  46
  47 void ResourceManager::set_async_loads(bool a)
  48 {
  49         async_loads = a;
  50 }
  51
  52 void ResourceManager::set_size_limit(UInt64 s)
  53 {
  54         size_limit = s;
  55 }
  56
  57 void ResourceManager::set_min_retain_frames(unsigned f)
  58 {
  59         min_retain_frames = f;
  60 }
  61
  62 void ResourceManager::set_max_retain_frames(unsigned f)
  63 {
  64         max_retain_frames = f;
  65 }
  66
  67 void ResourceManager::add_resource(Resource &r)
  68 {
  69         MutexLock lock(map_mutex);
  70         insert_unique(resources, &r, ManagedResource(r));
  71 }
  72
  73 const ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r) const
  74 {
  75         MutexLock lock(map_mutex);
  76         return get_item(resources, &r);
  77 }
  78
  79 ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r)
  80 {
  81         MutexLock lock(map_mutex);
  82         return get_item(resources, &r);
  83 }
  84
  85 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  86 {
  87         set_resource_location(r, ResourceLocation(c, n));
  88 }
  89
  90 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  91 {
  92         {
  93                 MutexLock lock(map_mutex);
  94                 ManagedResource &managed = get_item(resources, &r);
  95                 managed.location = l;
  96         }
  97
  98         if(policy==LOAD_IMMEDIATELY)
  99                 load_resource(r);
 100 }
 101
 102 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
 103 {
 104         const ManagedResource &managed = get_managed_resource(r);
 105         return managed.location.collection ? &managed.location : 0;
 106 }
 107
 108 void ResourceManager::load_resource(Resource &r)
 109 {
 110         ManagedResource &managed = get_managed_resource(r);
 111         if(!managed.location.collection)
 112                 throw runtime_error("no location");
 113
 114         if(managed.state!=ManagedResource::NOT_LOADED)
 115                 return;
 116
 117         if(async_loads)
 118         {
 119                 managed.state = ManagedResource::LOAD_QUEUED;
 120                 LoadQueue::iterator i;
 121                 for(i=queue.begin(); (i!=queue.end() && (*i)->load_priority>=managed.load_priority); ++i) ;
 122                 queue.insert(i, &managed);
 123         }
 124         else
 125         {
 126                 managed.start_loading();
 127                 while(!managed.loader->process()) ;
 128                 managed.finish_loading(true);
 129                 total_data_size += managed.data_size;
 130         }
 131 }
 132
 133 bool ResourceManager::is_resource_loaded(const Resource &r) const
 134 {
 135         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 136         return managed ? managed->state==ManagedResource::LOADED : false;
 137 }
 138
 139 void ResourceManager::resource_used(const Resource &r)
 140 {
 141         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 142         if(!managed)
 143                 return;
 144         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 145                 load_resource(*managed->resource);
 146
 147         managed->last_used = frame;
 148         if(max_retain_frames && !next_unload)
 149                 next_unload = frame+max_retain_frames+1;
 150 }
 151
 152 void ResourceManager::remove_resource(Resource &r)
 153 {
 154         ManagedResource &managed = get_managed_resource(r);
 155         ManagedResource::State state = managed.state;
 156         if(state==ManagedResource::LOAD_QUEUED)
 157         {
 158                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 159                 if(i!=queue.end())
 160                         queue.erase(i);
 161         }
 162         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 163                 thread.remove_resource(managed);
 164
 165         MutexLock lock(map_mutex);
 166         remove_existing(resources, &r);
 167 }
 168
 169 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 170 {
 171         get_managed_resource(r).add_watcher(w);
 172 }
 173
 174 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 175 {
 176         get_managed_resource(r).remove_watcher(w);
 177 }
 178
 179 void ResourceManager::tick()
 180 {
 181         ++frame;
 182
 183         bool do_unload = (frame>=next_unload);
 184         if(thread.sync())
 185         {
 186                 total_data_size += thread.get_and_reset_loaded_data_size();
 187                 do_unload = true;
 188         }
 189
 190         if(thread.needs_work() && !queue.empty())
 191                 dispatch_work();
 192
 193         if(do_unload)
 194         {
 195                 MutexLock lock(map_mutex);
 196                 if(max_retain_frames && frame>=next_unload)
 197                 {
 198                         unload_by_age();
 199
 200                         next_unload = frame;
 201                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 202                                 if(i->second.state==ManagedResource::LOADED)
 203                                         next_unload = min(next_unload, i->second.last_used);
 204                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 205                 }
 206
 207                 if(size_limit)
 208                         unload_by_size();
 209         }
 210 }
 211
 212 void ResourceManager::dispatch_work()
 213 {
 214         queue.sort(age_order);
 215
 216         if(queue.front()->last_used+10<frame)
 217         {
 218                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 219                         (*i)->state = ManagedResource::NOT_LOADED;
 220                 queue.clear();
 221                 return;
 222         }
 223
 224         while(thread.needs_work() && !queue.empty())
 225         {
 226                 ManagedResource *managed = queue.front();
 227                 queue.pop_front();
 228                 thread.add_resource(*managed);
 229         }
 230 }
 231
 232 void ResourceManager::unload_by_age()
 233 {
 234         unsigned unload_limit = frame-max_retain_frames;
 235
 236         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 237                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 238                 {
 239                         i->second.unload();
 240                         total_data_size -= i->second.data_size;
 241                 }
 242 }
 243
 244 void ResourceManager::unload_by_size()
 245 {
 246         unsigned unload_limit = frame-min_retain_frames;
 247
 248         while(total_data_size>size_limit)
 249         {
 250                 ManagedResource *best = 0;
 251                 UInt64 best_impact = 0;
 252                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 253                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 254                         {
 255                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 256                                 if(!best || impact>best_impact)
 257                                 {
 258                                         best = &i->second;
 259                                         best_impact = impact;
 260                                 }
 261                         }
 262
 263                 if(!best)
 264                         break;
 265
 266                 best->unload();
 267                 total_data_size -= best->data_size;
 268         }
 269 }
 270
 271 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 272 {
 273         return mr1->last_used>mr2->last_used;
 274 }
 275
 276
 277 ResourceManager::ResourceLocation::ResourceLocation():
 278         collection(0)
 279 { }
 280
 281 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 282         collection(&c),
 283         name(n)
 284 { }
 285
 286
 287 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 288         resource(&r),
 289         load_priority(r.get_load_priority()),
 290         io(0),
 291         loader(0),
 292         state(NOT_LOADED),
 293         last_used(0),
 294         data_size(0)
 295 { }
 296
 297 void ResourceManager::ManagedResource::start_loading()
 298 {
 299         io = location.collection->open_raw(location.name);
 300         if(!io)
 301                 throw resource_load_error(location.name, "open failed");
 302
 303         const Resources *res = dynamic_cast<Resources *>(location.collection);
 304         loader = resource->load(*io, res);
 305         if(!loader)
 306         {
 307                 delete io;
 308                 io = 0;
 309                 throw logic_error("no loader created");
 310         }
 311         state = LOADING;
 312 }
 313
 314 bool ResourceManager::ManagedResource::process(bool sync)
 315 {
 316         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 317                 if(loader->process())
 318                         state = LOAD_FINISHED;
 319
 320         return state==LOAD_FINISHED;
 321 }
 322
 323 void ResourceManager::ManagedResource::finish_loading(bool successful)
 324 {
 325         delete loader;
 326         loader = 0;
 327         delete io;
 328         io = 0;
 329
 330         if(successful)
 331         {
 332                 state = LOADED;
 333                 data_size = resource->get_data_size();
 334
 335                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 336                         (*i)->resource_loaded(*resource);
 337         }
 338         else
 339         {
 340                 resource->unload();
 341                 state = NOT_LOADED;
 342         }
 343 }
 344
 345 void ResourceManager::ManagedResource::finish_loading()
 346 {
 347         finish_loading(state==LOAD_FINISHED);
 348 }
 349
 350 void ResourceManager::ManagedResource::unload()
 351 {
 352         resource->unload();
 353         state = NOT_LOADED;
 354
 355         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 356                 (*i)->resource_unloaded(*resource);
 357 }
 358
 359 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 360 {
 361         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 362                 watchers.push_back(&w);
 363 }
 364
 365 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 366 {
 367         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 368         if(end!=watchers.end())
 369                 watchers.erase(end, watchers.end());
 370 }
 371
 372
 373 ResourceManager::LoadingThread::LoadingThread():
 374         sem(1),
 375         capacity(2),
 376         size(0),
 377         done(false)
 378 {
 379         launch();
 380 }
 381
 382 void ResourceManager::LoadingThread::main()
 383 {
 384         bool wait_for_work = false;
 385         while(!done)
 386         {
 387                 if(wait_for_work)
 388                         sem.wait();
 389
 390                 if(ManagedResource *managed = front(async_queue))
 391                 {
 392                         try
 393                         {
 394                                 managed->process(false);
 395                         }
 396                         catch(const exception &e)
 397                         {
 398                                 MutexLock lock(queue_mutex);
 399                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 400                                 managed->state = ManagedResource::LOAD_ERROR;
 401                         }
 402
 403                         MutexLock lock(queue_mutex);
 404                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 405                         wait_for_work = async_queue.empty();
 406                 }
 407                 else
 408                         wait_for_work = true;
 409         }
 410 }
 411
 412 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 413 {
 414         MutexLock lock(queue_mutex);
 415         if(queue.empty())
 416                 return 0;
 417
 418         return queue.front();
 419 }
 420
 421 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 422 {
 423         r.start_loading();
 424
 425         MutexLock lock(queue_mutex);
 426         if(r.loader->needs_sync())
 427                 sync_queue.push_back(&r);
 428         else
 429         {
 430                 bool was_empty = async_queue.empty();
 431                 async_queue.push_back(&r);
 432                 if(was_empty)
 433                         sem.signal();
 434         }
 435
 436         ++size;
 437 }
 438
 439 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 440 {
 441         while(!try_remove_resource(r))
 442                 Time::sleep(Time::msec);
 443
 444         r.finish_loading();
 445         if(r.state==ManagedResource::LOADED)
 446         {
 447                 MutexLock lock(data_size_mutex);
 448                 loaded_data_size += r.data_size;
 449         }
 450 }
 451
 452 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 453 {
 454         MutexLock lock(queue_mutex);
 455
 456         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 457         if(i==async_queue.end())
 458         {
 459                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 460                 if(i!=sync_queue.end())
 461                 {
 462                         sync_queue.erase(i);
 463                         --size;
 464                 }
 465         }
 466         else if(i==async_queue.begin())
 467                 return false;
 468         else
 469         {
 470                 async_queue.erase(i);
 471                 --size;
 472         }
 473
 474         return true;
 475 }
 476
 477 bool ResourceManager::LoadingThread::sync()
 478 {
 479         {
 480                 MutexLock lock(queue_mutex);
 481
 482                 if(!error_queue.empty())
 483                 {
 484                         resource_load_error err = error_queue.front();
 485                         error_queue.pop_front();
 486                         throw err;
 487                 }
 488
 489                 unsigned async_size = async_queue.size();
 490                 if(async_size==0 && size==capacity)
 491                         ++capacity;
 492                 else if(async_size>2 && capacity>2)
 493                         --capacity;
 494         }
 495
 496         bool any_finished = false;
 497         while(ManagedResource *managed = front(sync_queue))
 498         {
 499                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 500                 {
 501                         managed->finish_loading();
 502                         if(managed->state==ManagedResource::LOADED)
 503                         {
 504                                 MutexLock lock(data_size_mutex);
 505                                 loaded_data_size += managed->data_size;
 506                         }
 507                         any_finished = true;
 508                         --size;
 509
 510                         MutexLock lock(queue_mutex);
 511                         sync_queue.pop_front();
 512                 }
 513                 else
 514                 {
 515                         MutexLock lock(queue_mutex);
 516                         bool was_empty = async_queue.empty();
 517                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 518                         if(was_empty)
 519                                 sem.signal();
 520                 }
 521         }
 522
 523         return any_finished;
 524 }
 525
 526 UInt64 ResourceManager::LoadingThread::get_and_reset_loaded_data_size()
 527 {
 528         MutexLock lock(data_size_mutex);
 529         UInt64 result = loaded_data_size;
 530         loaded_data_size = 0;
 531         return result;
 532 }
 533
 534 void ResourceManager::LoadingThread::terminate()
 535 {
 536         done = true;
 537         sem.signal();
 538         join();
 539         async_queue.clear();
 540         sync_queue.clear();
 541 }
 542
 543 } // namespace GL
 544 } // namespace Msp