source/resources/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resources.h"
   8 #include "resourceobserver.h"
   9
  10 using namespace std;
  11
  12 namespace Msp {
  13 namespace GL {
  14
  15 resource_load_error::resource_load_error(const string &name, const string &err):
  16         runtime_error(format("%s: %s", name, err))
  17 { }
  18
  19 resource_load_error::resource_load_error(const string &name, const exception &exc):
  20         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  21 { }
  22
  23
  24 ResourceManager::ResourceManager():
  25         policy(LOAD_ON_DEMAND),
  26         async_loads(true),
  27         total_data_size(0),
  28         size_limit(0),
  29         frame(0),
  30         min_retain_frames(30),
  31         max_retain_frames(0),
  32         next_unload(0)
  33 { }
  34
  35 ResourceManager::~ResourceManager()
  36 {
  37         thread.terminate();
  38
  39         while(!resources.empty())
  40                 resources.begin()->second.resource->set_manager(0);
  41 }
  42
  43 void ResourceManager::set_loading_policy(LoadingPolicy p)
  44 {
  45         policy = p;
  46 }
  47
  48 void ResourceManager::set_async_loads(bool a)
  49 {
  50         async_loads = a;
  51 }
  52
  53 void ResourceManager::set_size_limit(UInt64 s)
  54 {
  55         size_limit = s;
  56 }
  57
  58 void ResourceManager::set_min_retain_frames(unsigned f)
  59 {
  60         min_retain_frames = f;
  61 }
  62
  63 void ResourceManager::set_max_retain_frames(unsigned f)
  64 {
  65         max_retain_frames = f;
  66 }
  67
  68 void ResourceManager::add_resource(Resource &r)
  69 {
  70         MutexLock lock(map_mutex);
  71         insert_unique(resources, &r, ManagedResource(r));
  72 }
  73
  74 const ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r) const
  75 {
  76         MutexLock lock(map_mutex);
  77         return get_item(resources, &r);
  78 }
  79
  80 ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r)
  81 {
  82         MutexLock lock(map_mutex);
  83         return get_item(resources, &r);
  84 }
  85
  86 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  87 {
  88         set_resource_location(r, ResourceLocation(c, n));
  89 }
  90
  91 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  92 {
  93         {
  94                 MutexLock lock(map_mutex);
  95                 ManagedResource &managed = get_item(resources, &r);
  96                 managed.location = l;
  97         }
  98
  99         if(policy==LOAD_IMMEDIATELY)
 100                 load_resource(r);
 101 }
 102
 103 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
 104 {
 105         const ManagedResource &managed = get_managed_resource(r);
 106         return managed.location.collection ? &managed.location : 0;
 107 }
 108
 109 void ResourceManager::load_resource(Resource &r)
 110 {
 111         ManagedResource &managed = get_managed_resource(r);
 112         if(!managed.location.collection)
 113                 throw runtime_error("no location");
 114
 115         if(managed.state!=ManagedResource::NOT_LOADED)
 116                 return;
 117
 118         if(async_loads)
 119         {
 120                 managed.state = ManagedResource::LOAD_QUEUED;
 121                 LoadQueue::iterator i;
 122                 for(i=queue.begin(); (i!=queue.end() && (*i)->load_priority>=managed.load_priority); ++i) ;
 123                 queue.insert(i, &managed);
 124         }
 125         else
 126         {
 127                 managed.start_loading();
 128                 while(!managed.loader->process()) ;
 129                 managed.finish_loading(true);
 130                 total_data_size += managed.data_size;
 131         }
 132 }
 133
 134 bool ResourceManager::is_resource_loaded(const Resource &r) const
 135 {
 136         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 137         return managed ? managed->state==ManagedResource::LOADED : false;
 138 }
 139
 140 void ResourceManager::resource_used(const Resource &r)
 141 {
 142         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 143         if(!managed)
 144                 return;
 145         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 146                 load_resource(*managed->resource);
 147
 148         managed->last_used = frame;
 149         if(max_retain_frames && !next_unload)
 150                 next_unload = frame+max_retain_frames+1;
 151 }
 152
 153 void ResourceManager::remove_resource(Resource &r)
 154 {
 155         ManagedResource &managed = get_managed_resource(r);
 156         ManagedResource::State state = managed.state;
 157         if(state==ManagedResource::LOAD_QUEUED)
 158         {
 159                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 160                 if(i!=queue.end())
 161                         queue.erase(i);
 162         }
 163         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 164                 thread.remove_resource(managed);
 165
 166         for(vector<ResourceObserver *>::const_iterator i=managed.observers.begin(); i!=managed.observers.end(); ++i)
 167                 (*i)->resource_removed(r);
 168
 169         MutexLock lock(map_mutex);
 170         remove_existing(resources, &r);
 171 }
 172
 173 void ResourceManager::observe_resource(const Resource &r, ResourceObserver &w)
 174 {
 175         get_managed_resource(r).add_observer(w);
 176 }
 177
 178 void ResourceManager::unobserve_resource(const Resource &r, ResourceObserver &w)
 179 {
 180         get_managed_resource(r).remove_observer(w);
 181 }
 182
 183 void ResourceManager::tick()
 184 {
 185         ++frame;
 186
 187         bool do_unload = (frame>=next_unload);
 188         if(thread.sync())
 189         {
 190                 total_data_size += thread.get_and_reset_loaded_data_size();
 191                 do_unload = true;
 192         }
 193
 194         if(thread.needs_work() && !queue.empty())
 195                 dispatch_work();
 196
 197         if(do_unload)
 198         {
 199                 MutexLock lock(map_mutex);
 200                 if(max_retain_frames && frame>=next_unload)
 201                 {
 202                         unload_by_age();
 203
 204                         next_unload = frame;
 205                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 206                                 if(i->second.state==ManagedResource::LOADED)
 207                                         next_unload = min(next_unload, i->second.last_used);
 208                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 209                 }
 210
 211                 if(size_limit)
 212                         unload_by_size();
 213         }
 214 }
 215
 216 void ResourceManager::dispatch_work()
 217 {
 218         queue.sort(age_order);
 219
 220         if(queue.front()->last_used+min_retain_frames<frame)
 221         {
 222                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 223                         (*i)->state = ManagedResource::NOT_LOADED;
 224                 queue.clear();
 225                 return;
 226         }
 227
 228         while(thread.needs_work() && !queue.empty())
 229         {
 230                 ManagedResource *managed = queue.front();
 231                 queue.pop_front();
 232                 thread.add_resource(*managed);
 233         }
 234 }
 235
 236 void ResourceManager::unload_by_age()
 237 {
 238         unsigned unload_limit = frame-max_retain_frames;
 239
 240         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 241                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 242                 {
 243                         i->second.unload();
 244                         total_data_size -= i->second.data_size;
 245                 }
 246 }
 247
 248 void ResourceManager::unload_by_size()
 249 {
 250         unsigned unload_limit = frame-min_retain_frames;
 251
 252         while(total_data_size>size_limit)
 253         {
 254                 ManagedResource *best = 0;
 255                 UInt64 best_impact = 0;
 256                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 257                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 258                         {
 259                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 260                                 if(!best || impact>best_impact)
 261                                 {
 262                                         best = &i->second;
 263                                         best_impact = impact;
 264                                 }
 265                         }
 266
 267                 if(!best)
 268                         break;
 269
 270                 best->unload();
 271                 total_data_size -= best->data_size;
 272         }
 273 }
 274
 275 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 276 {
 277         return mr1->last_used>mr2->last_used;
 278 }
 279
 280
 281 ResourceManager::ResourceLocation::ResourceLocation():
 282         collection(0)
 283 { }
 284
 285 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 286         collection(&c),
 287         name(n)
 288 { }
 289
 290
 291 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 292         resource(&r),
 293         load_priority(r.get_load_priority()),
 294         io(0),
 295         loader(0),
 296         state(NOT_LOADED),
 297         last_used(0),
 298         data_size(0)
 299 { }
 300
 301 void ResourceManager::ManagedResource::start_loading()
 302 {
 303         io = location.collection->open_raw(location.name);
 304         if(!io)
 305                 throw resource_load_error(location.name, "open failed");
 306
 307         const Resources *res = dynamic_cast<Resources *>(location.collection);
 308         loader = resource->load(*io, res);
 309         if(!loader)
 310         {
 311                 delete io;
 312                 io = 0;
 313                 throw logic_error("no loader created");
 314         }
 315         state = LOADING;
 316 }
 317
 318 bool ResourceManager::ManagedResource::process(bool sync)
 319 {
 320         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 321                 if(loader->process())
 322                         state = LOAD_FINISHED;
 323
 324         return state==LOAD_FINISHED;
 325 }
 326
 327 void ResourceManager::ManagedResource::finish_loading(bool successful)
 328 {
 329         delete loader;
 330         loader = 0;
 331         delete io;
 332         io = 0;
 333
 334         if(successful)
 335         {
 336                 state = LOADED;
 337                 data_size = resource->get_data_size();
 338
 339                 for(vector<ResourceObserver *>::const_iterator i=observers.begin(); i!=observers.end(); ++i)
 340                         (*i)->resource_loaded(*resource);
 341         }
 342         else
 343         {
 344                 resource->unload();
 345                 state = NOT_LOADED;
 346         }
 347 }
 348
 349 void ResourceManager::ManagedResource::finish_loading()
 350 {
 351         finish_loading(state==LOAD_FINISHED);
 352 }
 353
 354 void ResourceManager::ManagedResource::unload()
 355 {
 356         resource->unload();
 357         state = NOT_LOADED;
 358
 359         for(vector<ResourceObserver *>::const_iterator i=observers.begin(); i!=observers.end(); ++i)
 360                 (*i)->resource_unloaded(*resource);
 361 }
 362
 363 void ResourceManager::ManagedResource::add_observer(ResourceObserver &w)
 364 {
 365         if(find(observers.begin(), observers.end(), &w)==observers.end())
 366                 observers.push_back(&w);
 367 }
 368
 369 void ResourceManager::ManagedResource::remove_observer(ResourceObserver &w)
 370 {
 371         vector<ResourceObserver *>::iterator end = remove(observers.begin(), observers.end(), &w);
 372         if(end!=observers.end())
 373                 observers.erase(end, observers.end());
 374 }
 375
 376
 377 ResourceManager::LoadingThread::LoadingThread():
 378         Thread("ResourceManager"),
 379         sem(1),
 380         capacity(2),
 381         size(0),
 382         loaded_data_size(0),
 383         done(false)
 384 {
 385         launch();
 386 }
 387
 388 void ResourceManager::LoadingThread::main()
 389 {
 390         bool wait_for_work = false;
 391         while(!done)
 392         {
 393                 if(wait_for_work)
 394                         sem.wait();
 395
 396                 if(ManagedResource *managed = front(async_queue))
 397                 {
 398                         try
 399                         {
 400                                 managed->process(false);
 401                         }
 402                         catch(const exception &e)
 403                         {
 404                                 MutexLock lock(queue_mutex);
 405                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 406                                 managed->state = ManagedResource::LOAD_ERROR;
 407                         }
 408
 409                         MutexLock lock(queue_mutex);
 410                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 411                         wait_for_work = async_queue.empty();
 412                 }
 413                 else
 414                         wait_for_work = true;
 415         }
 416 }
 417
 418 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &que)
 419 {
 420         MutexLock lock(queue_mutex);
 421         if(que.empty())
 422                 return 0;
 423
 424         return que.front();
 425 }
 426
 427 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 428 {
 429         r.start_loading();
 430
 431         MutexLock lock(queue_mutex);
 432         if(r.loader->needs_sync())
 433                 sync_queue.push_back(&r);
 434         else
 435         {
 436                 bool was_empty = async_queue.empty();
 437                 async_queue.push_back(&r);
 438                 if(was_empty)
 439                         sem.signal();
 440         }
 441
 442         ++size;
 443 }
 444
 445 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 446 {
 447         while(!try_remove_resource(r))
 448                 Time::sleep(Time::msec);
 449
 450         r.finish_loading();
 451         if(r.state==ManagedResource::LOADED)
 452         {
 453                 MutexLock lock(data_size_mutex);
 454                 loaded_data_size += r.data_size;
 455         }
 456 }
 457
 458 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 459 {
 460         MutexLock lock(queue_mutex);
 461
 462         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 463         if(i==async_queue.end())
 464         {
 465                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 466                 if(i!=sync_queue.end())
 467                 {
 468                         sync_queue.erase(i);
 469                         --size;
 470                 }
 471         }
 472         else if(i==async_queue.begin())
 473                 return false;
 474         else
 475         {
 476                 async_queue.erase(i);
 477                 --size;
 478         }
 479
 480         return true;
 481 }
 482
 483 bool ResourceManager::LoadingThread::sync()
 484 {
 485         {
 486                 MutexLock lock(queue_mutex);
 487
 488                 if(!error_queue.empty())
 489                 {
 490                         resource_load_error err = error_queue.front();
 491                         error_queue.pop_front();
 492                         throw err;
 493                 }
 494
 495                 unsigned async_size = async_queue.size();
 496                 if(async_size==0 && size==capacity)
 497                         ++capacity;
 498                 else if(async_size>2 && capacity>2)
 499                         --capacity;
 500         }
 501
 502         bool any_finished = false;
 503         while(ManagedResource *managed = front(sync_queue))
 504         {
 505                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 506                 {
 507                         managed->finish_loading();
 508                         if(managed->state==ManagedResource::LOADED)
 509                         {
 510                                 MutexLock lock(data_size_mutex);
 511                                 loaded_data_size += managed->data_size;
 512                         }
 513                         any_finished = true;
 514                         --size;
 515
 516                         MutexLock lock(queue_mutex);
 517                         sync_queue.pop_front();
 518                 }
 519                 else
 520                 {
 521                         MutexLock lock(queue_mutex);
 522                         bool was_empty = async_queue.empty();
 523                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 524                         if(was_empty)
 525                                 sem.signal();
 526                 }
 527         }
 528
 529         return any_finished;
 530 }
 531
 532 UInt64 ResourceManager::LoadingThread::get_and_reset_loaded_data_size()
 533 {
 534         MutexLock lock(data_size_mutex);
 535         UInt64 result = loaded_data_size;
 536         loaded_data_size = 0;
 537         return result;
 538 }
 539
 540 void ResourceManager::LoadingThread::terminate()
 541 {
 542         done = true;
 543         sem.signal();
 544         join();
 545         async_queue.clear();
 546         sync_queue.clear();
 547 }
 548
 549 } // namespace GL
 550 } // namespace Msp