source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resourcewatcher.h"
   8
   9 using namespace std;
  10
  11 namespace Msp {
  12 namespace GL {
  13
  14 resource_load_error::resource_load_error(const string &name, const exception &exc):
  15         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  16 { }
  17
  18
  19 ResourceManager::ResourceManager():
  20         policy(LOAD_ON_DEMAND),
  21         async_loads(true),
  22         size_limit(0),
  23         frame(0),
  24         min_retain_frames(30),
  25         max_retain_frames(0),
  26         next_unload(0)
  27 { }
  28
  29 ResourceManager::~ResourceManager()
  30 {
  31         thread.terminate();
  32
  33         while(!resources.empty())
  34                 resources.begin()->second.resource->set_manager(0);
  35 }
  36
  37 void ResourceManager::set_loading_policy(LoadingPolicy p)
  38 {
  39         policy = p;
  40 }
  41
  42 void ResourceManager::set_async_loads(bool a)
  43 {
  44         async_loads = a;
  45 }
  46
  47 void ResourceManager::set_size_limit(UInt64 s)
  48 {
  49         size_limit = s;
  50 }
  51
  52 void ResourceManager::set_min_retain_frames(unsigned f)
  53 {
  54         min_retain_frames = f;
  55 }
  56
  57 void ResourceManager::set_max_retain_frames(unsigned f)
  58 {
  59         max_retain_frames = f;
  60 }
  61
  62 void ResourceManager::add_resource(Resource &r)
  63 {
  64         insert_unique(resources, &r, ManagedResource(r));
  65 }
  66
  67 void *ResourceManager::get_data_for_resource(const Resource &r)
  68 {
  69         return &get_item(resources, &r);
  70 }
  71
  72 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  73 {
  74         set_resource_location(r, ResourceLocation(c, n));
  75 }
  76
  77 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  78 {
  79         ManagedResource &managed = get_item(resources, &r);
  80         managed.location = l;
  81
  82         if(policy==LOAD_IMMEDIATELY)
  83                 load_resource(r);
  84 }
  85
  86 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
  87 {
  88         const ManagedResource &managed = get_item(resources, &r);
  89         return managed.location.collection ? &managed.location : 0;
  90 }
  91
  92 void ResourceManager::load_resource(Resource &r)
  93 {
  94         ManagedResource &managed = get_item(resources, &r);
  95         if(!managed.location.collection)
  96                 throw runtime_error("no location");
  97
  98         if(managed.state!=ManagedResource::NOT_LOADED)
  99                 return;
 100
 101         if(async_loads)
 102         {
 103                 managed.state = ManagedResource::LOAD_QUEUED;
 104                 queue.push_back(&managed);
 105         }
 106         else
 107         {
 108                 managed.start_loading();
 109                 while(!managed.loader->process()) ;
 110                 managed.finish_loading(true);
 111         }
 112 }
 113
 114 bool ResourceManager::is_resource_loaded(const Resource &r) const
 115 {
 116         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 117         return managed ? managed->state==ManagedResource::LOADED : false;
 118 }
 119
 120 void ResourceManager::resource_used(const Resource &r)
 121 {
 122         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 123         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 124                 load_resource(*managed->resource);
 125
 126         managed->last_used = frame;
 127         if(max_retain_frames && !next_unload)
 128                 next_unload = frame+max_retain_frames+1;
 129 }
 130
 131 void ResourceManager::remove_resource(Resource &r)
 132 {
 133         ManagedResource &managed = get_item(resources, &r);
 134         ManagedResource::State state = managed.state;
 135         if(state==ManagedResource::LOAD_QUEUED)
 136         {
 137                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 138                 if(i!=queue.end())
 139                         queue.erase(i);
 140         }
 141         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 142                 thread.remove_resource(managed);
 143         remove_existing(resources, &r);
 144 }
 145
 146 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 147 {
 148         get_item(resources, &r).add_watcher(w);
 149 }
 150
 151 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 152 {
 153         get_item(resources, &r).remove_watcher(w);
 154 }
 155
 156 void ResourceManager::tick()
 157 {
 158         ++frame;
 159
 160         bool do_unload = (frame>=next_unload);
 161         if(thread.sync())
 162                 do_unload = true;
 163
 164         if(thread.needs_work() && !queue.empty())
 165                 dispatch_work();
 166
 167         if(do_unload)
 168         {
 169                 if(max_retain_frames && frame>=next_unload)
 170                 {
 171                         unload_by_age();
 172
 173                         next_unload = frame;
 174                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 175                                 if(i->second.state==ManagedResource::LOADED)
 176                                         next_unload = min(next_unload, i->second.last_used);
 177                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 178                 }
 179
 180                 if(size_limit)
 181                         unload_by_size();
 182         }
 183 }
 184
 185 void ResourceManager::dispatch_work()
 186 {
 187         queue.sort(age_order);
 188
 189         if(queue.front()->last_used+10<frame)
 190         {
 191                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 192                         (*i)->state = ManagedResource::NOT_LOADED;
 193                 queue.clear();
 194                 return;
 195         }
 196
 197         while(thread.needs_work() && !queue.empty())
 198         {
 199                 ManagedResource *managed = queue.front();
 200                 queue.pop_front();
 201                 thread.add_resource(*managed);
 202         }
 203 }
 204
 205 void ResourceManager::unload_by_age()
 206 {
 207         unsigned unload_limit = frame-max_retain_frames;
 208
 209         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 210                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 211                         i->second.unload();
 212 }
 213
 214 void ResourceManager::unload_by_size()
 215 {
 216         unsigned unload_limit = frame-min_retain_frames;
 217
 218         while(get_total_data_size()>size_limit)
 219         {
 220                 ManagedResource *best = 0;
 221                 UInt64 best_impact = 0;
 222                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 223                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 224                         {
 225                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 226                                 if(!best || impact>best_impact)
 227                                 {
 228                                         best = &i->second;
 229                                         best_impact = impact;
 230                                 }
 231                         }
 232
 233                 if(!best)
 234                         break;
 235
 236                 best->unload();
 237         }
 238 }
 239
 240 UInt64 ResourceManager::get_total_data_size() const
 241 {
 242         UInt64 total = 0;
 243         for(ResourceMap::const_iterator i=resources.begin(); i!=resources.end(); ++i)
 244                 if(i->second.state==ManagedResource::LOADED)
 245                         total += i->second.data_size;
 246         return total;
 247 }
 248
 249 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 250 {
 251         return mr1->last_used>mr2->last_used;
 252 }
 253
 254
 255 ResourceManager::ResourceLocation::ResourceLocation():
 256         collection(0)
 257 { }
 258
 259 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 260         collection(&c),
 261         name(n)
 262 { }
 263
 264
 265 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 266         resource(&r),
 267         io(0),
 268         loader(0),
 269         state(NOT_LOADED),
 270         last_used(0),
 271         data_size(0)
 272 { }
 273
 274 void ResourceManager::ManagedResource::start_loading()
 275 {
 276         io = location.collection->open_raw(location.name);
 277         loader = resource->load(*io);
 278         if(!loader)
 279         {
 280                 delete io;
 281                 io = 0;
 282                 throw logic_error("no loader created");
 283         }
 284         state = LOADING;
 285 }
 286
 287 bool ResourceManager::ManagedResource::process(bool sync)
 288 {
 289         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 290                 if(loader->process())
 291                         state = LOAD_FINISHED;
 292
 293         return state==LOAD_FINISHED;
 294 }
 295
 296 void ResourceManager::ManagedResource::finish_loading(bool successful)
 297 {
 298         delete loader;
 299         loader = 0;
 300         delete io;
 301         io = 0;
 302
 303         if(successful)
 304         {
 305                 state = LOADED;
 306                 data_size = resource->get_data_size();
 307
 308                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 309                         (*i)->resource_loaded(*resource);
 310         }
 311         else
 312         {
 313                 resource->unload();
 314                 state = NOT_LOADED;
 315         }
 316 }
 317
 318 void ResourceManager::ManagedResource::finish_loading()
 319 {
 320         finish_loading(state==LOAD_FINISHED);
 321 }
 322
 323 void ResourceManager::ManagedResource::unload()
 324 {
 325         resource->unload();
 326         state = NOT_LOADED;
 327
 328         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 329                 (*i)->resource_unloaded(*resource);
 330 }
 331
 332 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 333 {
 334         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 335                 watchers.push_back(&w);
 336 }
 337
 338 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 339 {
 340         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 341         if(end!=watchers.end())
 342                 watchers.erase(end, watchers.end());
 343 }
 344
 345
 346 ResourceManager::LoadingThread::LoadingThread():
 347         sem(1),
 348         capacity(2),
 349         size(0),
 350         done(false)
 351 {
 352         launch();
 353 }
 354
 355 void ResourceManager::LoadingThread::main()
 356 {
 357         bool wait_for_work = false;
 358         while(!done)
 359         {
 360                 if(wait_for_work)
 361                         sem.wait();
 362
 363                 if(ManagedResource *managed = front(async_queue))
 364                 {
 365                         try
 366                         {
 367                                 managed->process(false);
 368                         }
 369                         catch(const exception &e)
 370                         {
 371                                 MutexLock lock(queue_mutex);
 372                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 373                                 managed->state = ManagedResource::LOAD_ERROR;
 374                         }
 375
 376                         MutexLock lock(queue_mutex);
 377                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 378                         wait_for_work = async_queue.empty();
 379                 }
 380                 else
 381                         wait_for_work = true;
 382         }
 383 }
 384
 385 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 386 {
 387         MutexLock lock(queue_mutex);
 388         if(queue.empty())
 389                 return 0;
 390
 391         return queue.front();
 392 }
 393
 394 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 395 {
 396         r.start_loading();
 397
 398         MutexLock lock(queue_mutex);
 399         if(r.loader->needs_sync())
 400                 sync_queue.push_back(&r);
 401         else
 402         {
 403                 bool was_empty = async_queue.empty();
 404                 async_queue.push_back(&r);
 405                 if(was_empty)
 406                         sem.signal();
 407         }
 408
 409         ++size;
 410 }
 411
 412 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 413 {
 414         while(!try_remove_resource(r))
 415                 Time::sleep(Time::msec);
 416
 417         r.finish_loading();
 418 }
 419
 420 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 421 {
 422         MutexLock lock(queue_mutex);
 423
 424         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 425         if(i==async_queue.end())
 426         {
 427                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 428                 if(i!=sync_queue.end())
 429                         sync_queue.erase(i);
 430         }
 431         else if(i==async_queue.begin())
 432                 return false;
 433         else
 434                 async_queue.erase(i);
 435
 436         return true;
 437 }
 438
 439 bool ResourceManager::LoadingThread::sync()
 440 {
 441         {
 442                 MutexLock lock(queue_mutex);
 443
 444                 if(!error_queue.empty())
 445                 {
 446                         resource_load_error err = error_queue.front();
 447                         error_queue.pop_front();
 448                         throw err;
 449                 }
 450
 451                 unsigned async_size = async_queue.size();
 452                 if(async_size==0 && size==capacity)
 453                         ++capacity;
 454                 else if(async_size>2 && capacity>2)
 455                         --capacity;
 456         }
 457
 458         bool any_finished = false;
 459         while(ManagedResource *managed = front(sync_queue))
 460         {
 461                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 462                 {
 463                         managed->finish_loading();
 464                         any_finished = true;
 465                         --size;
 466
 467                         MutexLock lock(queue_mutex);
 468                         sync_queue.pop_front();
 469                 }
 470                 else
 471                 {
 472                         MutexLock lock(queue_mutex);
 473                         bool was_empty = async_queue.empty();
 474                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 475                         if(was_empty)
 476                                 sem.signal();
 477                 }
 478         }
 479
 480         return any_finished;
 481 }
 482
 483 void ResourceManager::LoadingThread::terminate()
 484 {
 485         done = true;
 486         sem.signal();
 487         join();
 488         async_queue.clear();
 489         sync_queue.clear();
 490 }
 491
 492 } // namespace GL
 493 } // namespace Msp