source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resources.h"
   8 #include "resourcewatcher.h"
   9
  10 using namespace std;
  11
  12 namespace Msp {
  13 namespace GL {
  14
  15 resource_load_error::resource_load_error(const string &name, const string &err):
  16         runtime_error(format("%s: %s", name, err))
  17 { }
  18
  19 resource_load_error::resource_load_error(const string &name, const exception &exc):
  20         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  21 { }
  22
  23
  24 ResourceManager::ResourceManager():
  25         policy(LOAD_ON_DEMAND),
  26         async_loads(true),
  27         total_data_size(0),
  28         size_limit(0),
  29         frame(0),
  30         min_retain_frames(30),
  31         max_retain_frames(0),
  32         next_unload(0)
  33 { }
  34
  35 ResourceManager::~ResourceManager()
  36 {
  37         thread.terminate();
  38
  39         while(!resources.empty())
  40                 resources.begin()->second.resource->set_manager(0);
  41 }
  42
  43 void ResourceManager::set_loading_policy(LoadingPolicy p)
  44 {
  45         policy = p;
  46 }
  47
  48 void ResourceManager::set_async_loads(bool a)
  49 {
  50         async_loads = a;
  51 }
  52
  53 void ResourceManager::set_size_limit(UInt64 s)
  54 {
  55         size_limit = s;
  56 }
  57
  58 void ResourceManager::set_min_retain_frames(unsigned f)
  59 {
  60         min_retain_frames = f;
  61 }
  62
  63 void ResourceManager::set_max_retain_frames(unsigned f)
  64 {
  65         max_retain_frames = f;
  66 }
  67
  68 void ResourceManager::add_resource(Resource &r)
  69 {
  70         MutexLock lock(map_mutex);
  71         insert_unique(resources, &r, ManagedResource(r));
  72 }
  73
  74 const ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r) const
  75 {
  76         MutexLock lock(map_mutex);
  77         return get_item(resources, &r);
  78 }
  79
  80 ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r)
  81 {
  82         MutexLock lock(map_mutex);
  83         return get_item(resources, &r);
  84 }
  85
  86 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  87 {
  88         set_resource_location(r, ResourceLocation(c, n));
  89 }
  90
  91 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  92 {
  93         {
  94                 MutexLock lock(map_mutex);
  95                 ManagedResource &managed = get_item(resources, &r);
  96                 managed.location = l;
  97         }
  98
  99         if(policy==LOAD_IMMEDIATELY)
 100                 load_resource(r);
 101 }
 102
 103 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
 104 {
 105         const ManagedResource &managed = get_managed_resource(r);
 106         return managed.location.collection ? &managed.location : 0;
 107 }
 108
 109 void ResourceManager::load_resource(Resource &r)
 110 {
 111         ManagedResource &managed = get_managed_resource(r);
 112         if(!managed.location.collection)
 113                 throw runtime_error("no location");
 114
 115         if(managed.state!=ManagedResource::NOT_LOADED)
 116                 return;
 117
 118         if(async_loads)
 119         {
 120                 managed.state = ManagedResource::LOAD_QUEUED;
 121                 LoadQueue::iterator i;
 122                 for(i=queue.begin(); (i!=queue.end() && (*i)->load_priority>=managed.load_priority); ++i) ;
 123                 queue.insert(i, &managed);
 124         }
 125         else
 126         {
 127                 managed.start_loading();
 128                 while(!managed.loader->process()) ;
 129                 managed.finish_loading(true);
 130                 total_data_size += managed.data_size;
 131         }
 132 }
 133
 134 bool ResourceManager::is_resource_loaded(const Resource &r) const
 135 {
 136         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 137         return managed ? managed->state==ManagedResource::LOADED : false;
 138 }
 139
 140 void ResourceManager::resource_used(const Resource &r)
 141 {
 142         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 143         if(!managed)
 144                 return;
 145         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 146                 load_resource(*managed->resource);
 147
 148         managed->last_used = frame;
 149         if(max_retain_frames && !next_unload)
 150                 next_unload = frame+max_retain_frames+1;
 151 }
 152
 153 void ResourceManager::remove_resource(Resource &r)
 154 {
 155         ManagedResource &managed = get_managed_resource(r);
 156         ManagedResource::State state = managed.state;
 157         if(state==ManagedResource::LOAD_QUEUED)
 158         {
 159                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 160                 if(i!=queue.end())
 161                         queue.erase(i);
 162         }
 163         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 164                 thread.remove_resource(managed);
 165
 166         MutexLock lock(map_mutex);
 167         remove_existing(resources, &r);
 168 }
 169
 170 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 171 {
 172         get_managed_resource(r).add_watcher(w);
 173 }
 174
 175 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 176 {
 177         get_managed_resource(r).remove_watcher(w);
 178 }
 179
 180 void ResourceManager::tick()
 181 {
 182         ++frame;
 183
 184         bool do_unload = (frame>=next_unload);
 185         if(thread.sync())
 186         {
 187                 total_data_size += thread.get_and_reset_loaded_data_size();
 188                 do_unload = true;
 189         }
 190
 191         if(thread.needs_work() && !queue.empty())
 192                 dispatch_work();
 193
 194         if(do_unload)
 195         {
 196                 MutexLock lock(map_mutex);
 197                 if(max_retain_frames && frame>=next_unload)
 198                 {
 199                         unload_by_age();
 200
 201                         next_unload = frame;
 202                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 203                                 if(i->second.state==ManagedResource::LOADED)
 204                                         next_unload = min(next_unload, i->second.last_used);
 205                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 206                 }
 207
 208                 if(size_limit)
 209                         unload_by_size();
 210         }
 211 }
 212
 213 void ResourceManager::dispatch_work()
 214 {
 215         queue.sort(age_order);
 216
 217         if(queue.front()->last_used+10<frame)
 218         {
 219                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 220                         (*i)->state = ManagedResource::NOT_LOADED;
 221                 queue.clear();
 222                 return;
 223         }
 224
 225         while(thread.needs_work() && !queue.empty())
 226         {
 227                 ManagedResource *managed = queue.front();
 228                 queue.pop_front();
 229                 thread.add_resource(*managed);
 230         }
 231 }
 232
 233 void ResourceManager::unload_by_age()
 234 {
 235         unsigned unload_limit = frame-max_retain_frames;
 236
 237         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 238                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 239                 {
 240                         i->second.unload();
 241                         total_data_size -= i->second.data_size;
 242                 }
 243 }
 244
 245 void ResourceManager::unload_by_size()
 246 {
 247         unsigned unload_limit = frame-min_retain_frames;
 248
 249         while(total_data_size>size_limit)
 250         {
 251                 ManagedResource *best = 0;
 252                 UInt64 best_impact = 0;
 253                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 254                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 255                         {
 256                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 257                                 if(!best || impact>best_impact)
 258                                 {
 259                                         best = &i->second;
 260                                         best_impact = impact;
 261                                 }
 262                         }
 263
 264                 if(!best)
 265                         break;
 266
 267                 best->unload();
 268                 total_data_size -= best->data_size;
 269         }
 270 }
 271
 272 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 273 {
 274         return mr1->last_used>mr2->last_used;
 275 }
 276
 277
 278 ResourceManager::ResourceLocation::ResourceLocation():
 279         collection(0)
 280 { }
 281
 282 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 283         collection(&c),
 284         name(n)
 285 { }
 286
 287
 288 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 289         resource(&r),
 290         load_priority(r.get_load_priority()),
 291         io(0),
 292         loader(0),
 293         state(NOT_LOADED),
 294         last_used(0),
 295         data_size(0)
 296 { }
 297
 298 void ResourceManager::ManagedResource::start_loading()
 299 {
 300         io = location.collection->open_raw(location.name);
 301         if(!io)
 302                 throw resource_load_error(location.name, "open failed");
 303
 304         const Resources *res = dynamic_cast<Resources *>(location.collection);
 305         loader = resource->load(*io, res);
 306         if(!loader)
 307         {
 308                 delete io;
 309                 io = 0;
 310                 throw logic_error("no loader created");
 311         }
 312         state = LOADING;
 313 }
 314
 315 bool ResourceManager::ManagedResource::process(bool sync)
 316 {
 317         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 318                 if(loader->process())
 319                         state = LOAD_FINISHED;
 320
 321         return state==LOAD_FINISHED;
 322 }
 323
 324 void ResourceManager::ManagedResource::finish_loading(bool successful)
 325 {
 326         delete loader;
 327         loader = 0;
 328         delete io;
 329         io = 0;
 330
 331         if(successful)
 332         {
 333                 state = LOADED;
 334                 data_size = resource->get_data_size();
 335
 336                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 337                         (*i)->resource_loaded(*resource);
 338         }
 339         else
 340         {
 341                 resource->unload();
 342                 state = NOT_LOADED;
 343         }
 344 }
 345
 346 void ResourceManager::ManagedResource::finish_loading()
 347 {
 348         finish_loading(state==LOAD_FINISHED);
 349 }
 350
 351 void ResourceManager::ManagedResource::unload()
 352 {
 353         resource->unload();
 354         state = NOT_LOADED;
 355
 356         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 357                 (*i)->resource_unloaded(*resource);
 358 }
 359
 360 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 361 {
 362         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 363                 watchers.push_back(&w);
 364 }
 365
 366 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 367 {
 368         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 369         if(end!=watchers.end())
 370                 watchers.erase(end, watchers.end());
 371 }
 372
 373
 374 ResourceManager::LoadingThread::LoadingThread():
 375         sem(1),
 376         capacity(2),
 377         size(0),
 378         loaded_data_size(0),
 379         done(false)
 380 {
 381         launch();
 382 }
 383
 384 void ResourceManager::LoadingThread::main()
 385 {
 386         bool wait_for_work = false;
 387         while(!done)
 388         {
 389                 if(wait_for_work)
 390                         sem.wait();
 391
 392                 if(ManagedResource *managed = front(async_queue))
 393                 {
 394                         try
 395                         {
 396                                 managed->process(false);
 397                         }
 398                         catch(const exception &e)
 399                         {
 400                                 MutexLock lock(queue_mutex);
 401                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 402                                 managed->state = ManagedResource::LOAD_ERROR;
 403                         }
 404
 405                         MutexLock lock(queue_mutex);
 406                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 407                         wait_for_work = async_queue.empty();
 408                 }
 409                 else
 410                         wait_for_work = true;
 411         }
 412 }
 413
 414 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 415 {
 416         MutexLock lock(queue_mutex);
 417         if(queue.empty())
 418                 return 0;
 419
 420         return queue.front();
 421 }
 422
 423 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 424 {
 425         r.start_loading();
 426
 427         MutexLock lock(queue_mutex);
 428         if(r.loader->needs_sync())
 429                 sync_queue.push_back(&r);
 430         else
 431         {
 432                 bool was_empty = async_queue.empty();
 433                 async_queue.push_back(&r);
 434                 if(was_empty)
 435                         sem.signal();
 436         }
 437
 438         ++size;
 439 }
 440
 441 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 442 {
 443         while(!try_remove_resource(r))
 444                 Time::sleep(Time::msec);
 445
 446         r.finish_loading();
 447         if(r.state==ManagedResource::LOADED)
 448         {
 449                 MutexLock lock(data_size_mutex);
 450                 loaded_data_size += r.data_size;
 451         }
 452 }
 453
 454 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 455 {
 456         MutexLock lock(queue_mutex);
 457
 458         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 459         if(i==async_queue.end())
 460         {
 461                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 462                 if(i!=sync_queue.end())
 463                 {
 464                         sync_queue.erase(i);
 465                         --size;
 466                 }
 467         }
 468         else if(i==async_queue.begin())
 469                 return false;
 470         else
 471         {
 472                 async_queue.erase(i);
 473                 --size;
 474         }
 475
 476         return true;
 477 }
 478
 479 bool ResourceManager::LoadingThread::sync()
 480 {
 481         {
 482                 MutexLock lock(queue_mutex);
 483
 484                 if(!error_queue.empty())
 485                 {
 486                         resource_load_error err = error_queue.front();
 487                         error_queue.pop_front();
 488                         throw err;
 489                 }
 490
 491                 unsigned async_size = async_queue.size();
 492                 if(async_size==0 && size==capacity)
 493                         ++capacity;
 494                 else if(async_size>2 && capacity>2)
 495                         --capacity;
 496         }
 497
 498         bool any_finished = false;
 499         while(ManagedResource *managed = front(sync_queue))
 500         {
 501                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 502                 {
 503                         managed->finish_loading();
 504                         if(managed->state==ManagedResource::LOADED)
 505                         {
 506                                 MutexLock lock(data_size_mutex);
 507                                 loaded_data_size += managed->data_size;
 508                         }
 509                         any_finished = true;
 510                         --size;
 511
 512                         MutexLock lock(queue_mutex);
 513                         sync_queue.pop_front();
 514                 }
 515                 else
 516                 {
 517                         MutexLock lock(queue_mutex);
 518                         bool was_empty = async_queue.empty();
 519                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 520                         if(was_empty)
 521                                 sem.signal();
 522                 }
 523         }
 524
 525         return any_finished;
 526 }
 527
 528 UInt64 ResourceManager::LoadingThread::get_and_reset_loaded_data_size()
 529 {
 530         MutexLock lock(data_size_mutex);
 531         UInt64 result = loaded_data_size;
 532         loaded_data_size = 0;
 533         return result;
 534 }
 535
 536 void ResourceManager::LoadingThread::terminate()
 537 {
 538         done = true;
 539         sem.signal();
 540         join();
 541         async_queue.clear();
 542         sync_queue.clear();
 543 }
 544
 545 } // namespace GL
 546 } // namespace Msp