source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resourcewatcher.h"
   8
   9 using namespace std;
  10
  11 namespace Msp {
  12 namespace GL {
  13
  14 resource_load_error::resource_load_error(const string &name, const exception &exc):
  15         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  16 { }
  17
  18
  19 ResourceManager::ResourceManager():
  20         policy(LOAD_ON_DEMAND),
  21         async_loads(true),
  22         size_limit(0),
  23         frame(0),
  24         min_retain_frames(30),
  25         max_retain_frames(0),
  26         next_unload(0)
  27 { }
  28
  29 ResourceManager::~ResourceManager()
  30 {
  31         thread.terminate();
  32
  33         while(!resources.empty())
  34                 resources.begin()->second.resource->set_manager(0);
  35 }
  36
  37 void ResourceManager::set_loading_policy(LoadingPolicy p)
  38 {
  39         policy = p;
  40 }
  41
  42 void ResourceManager::set_async_loads(bool a)
  43 {
  44         async_loads = a;
  45 }
  46
  47 void ResourceManager::set_size_limit(UInt64 s)
  48 {
  49         size_limit = s;
  50 }
  51
  52 void ResourceManager::set_min_retain_frames(unsigned f)
  53 {
  54         min_retain_frames = f;
  55 }
  56
  57 void ResourceManager::set_max_retain_frames(unsigned f)
  58 {
  59         max_retain_frames = f;
  60 }
  61
  62 void ResourceManager::add_resource(Resource &r)
  63 {
  64         insert_unique(resources, &r, ManagedResource(r));
  65 }
  66
  67 void *ResourceManager::get_data_for_resource(const Resource &r)
  68 {
  69         return &get_item(resources, &r);
  70 }
  71
  72 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  73 {
  74         set_resource_location(r, ResourceLocation(c, n));
  75 }
  76
  77 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  78 {
  79         ManagedResource &managed = get_item(resources, &r);
  80         managed.location = l;
  81
  82         if(policy==LOAD_IMMEDIATELY)
  83                 load_resource(r);
  84 }
  85
  86 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
  87 {
  88         const ManagedResource &managed = get_item(resources, &r);
  89         return managed.location.collection ? &managed.location : 0;
  90 }
  91
  92 void ResourceManager::load_resource(Resource &r)
  93 {
  94         ManagedResource &managed = get_item(resources, &r);
  95         if(!managed.location.collection)
  96                 throw runtime_error("no location");
  97
  98         if(managed.state!=ManagedResource::NOT_LOADED)
  99                 return;
 100
 101         if(async_loads)
 102         {
 103                 managed.state = ManagedResource::LOAD_QUEUED;
 104                 queue.push_back(&managed);
 105         }
 106         else
 107         {
 108                 managed.start_loading();
 109                 while(!managed.loader->process()) ;
 110                 managed.finish_loading(true);
 111         }
 112 }
 113
 114 bool ResourceManager::is_resource_loaded(const Resource &r) const
 115 {
 116         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 117         return managed ? managed->state==ManagedResource::LOADED : false;
 118 }
 119
 120 void ResourceManager::resource_used(const Resource &r)
 121 {
 122         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 123         if(!managed)
 124                 return;
 125         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 126                 load_resource(*managed->resource);
 127
 128         managed->last_used = frame;
 129         if(max_retain_frames && !next_unload)
 130                 next_unload = frame+max_retain_frames+1;
 131 }
 132
 133 void ResourceManager::remove_resource(Resource &r)
 134 {
 135         ManagedResource &managed = get_item(resources, &r);
 136         ManagedResource::State state = managed.state;
 137         if(state==ManagedResource::LOAD_QUEUED)
 138         {
 139                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 140                 if(i!=queue.end())
 141                         queue.erase(i);
 142         }
 143         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 144                 thread.remove_resource(managed);
 145         remove_existing(resources, &r);
 146 }
 147
 148 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 149 {
 150         get_item(resources, &r).add_watcher(w);
 151 }
 152
 153 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 154 {
 155         get_item(resources, &r).remove_watcher(w);
 156 }
 157
 158 void ResourceManager::tick()
 159 {
 160         ++frame;
 161
 162         bool do_unload = (frame>=next_unload);
 163         if(thread.sync())
 164                 do_unload = true;
 165
 166         if(thread.needs_work() && !queue.empty())
 167                 dispatch_work();
 168
 169         if(do_unload)
 170         {
 171                 if(max_retain_frames && frame>=next_unload)
 172                 {
 173                         unload_by_age();
 174
 175                         next_unload = frame;
 176                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 177                                 if(i->second.state==ManagedResource::LOADED)
 178                                         next_unload = min(next_unload, i->second.last_used);
 179                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 180                 }
 181
 182                 if(size_limit)
 183                         unload_by_size();
 184         }
 185 }
 186
 187 void ResourceManager::dispatch_work()
 188 {
 189         queue.sort(age_order);
 190
 191         if(queue.front()->last_used+10<frame)
 192         {
 193                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 194                         (*i)->state = ManagedResource::NOT_LOADED;
 195                 queue.clear();
 196                 return;
 197         }
 198
 199         while(thread.needs_work() && !queue.empty())
 200         {
 201                 ManagedResource *managed = queue.front();
 202                 queue.pop_front();
 203                 thread.add_resource(*managed);
 204         }
 205 }
 206
 207 void ResourceManager::unload_by_age()
 208 {
 209         unsigned unload_limit = frame-max_retain_frames;
 210
 211         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 212                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 213                         i->second.unload();
 214 }
 215
 216 void ResourceManager::unload_by_size()
 217 {
 218         unsigned unload_limit = frame-min_retain_frames;
 219
 220         while(get_total_data_size()>size_limit)
 221         {
 222                 ManagedResource *best = 0;
 223                 UInt64 best_impact = 0;
 224                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 225                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 226                         {
 227                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 228                                 if(!best || impact>best_impact)
 229                                 {
 230                                         best = &i->second;
 231                                         best_impact = impact;
 232                                 }
 233                         }
 234
 235                 if(!best)
 236                         break;
 237
 238                 best->unload();
 239         }
 240 }
 241
 242 UInt64 ResourceManager::get_total_data_size() const
 243 {
 244         UInt64 total = 0;
 245         for(ResourceMap::const_iterator i=resources.begin(); i!=resources.end(); ++i)
 246                 if(i->second.state==ManagedResource::LOADED)
 247                         total += i->second.data_size;
 248         return total;
 249 }
 250
 251 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 252 {
 253         return mr1->last_used>mr2->last_used;
 254 }
 255
 256
 257 ResourceManager::ResourceLocation::ResourceLocation():
 258         collection(0)
 259 { }
 260
 261 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 262         collection(&c),
 263         name(n)
 264 { }
 265
 266
 267 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 268         resource(&r),
 269         io(0),
 270         loader(0),
 271         state(NOT_LOADED),
 272         last_used(0),
 273         data_size(0)
 274 { }
 275
 276 void ResourceManager::ManagedResource::start_loading()
 277 {
 278         io = location.collection->open_raw(location.name);
 279         loader = resource->load(*io);
 280         if(!loader)
 281         {
 282                 delete io;
 283                 io = 0;
 284                 throw logic_error("no loader created");
 285         }
 286         state = LOADING;
 287 }
 288
 289 bool ResourceManager::ManagedResource::process(bool sync)
 290 {
 291         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 292                 if(loader->process())
 293                         state = LOAD_FINISHED;
 294
 295         return state==LOAD_FINISHED;
 296 }
 297
 298 void ResourceManager::ManagedResource::finish_loading(bool successful)
 299 {
 300         delete loader;
 301         loader = 0;
 302         delete io;
 303         io = 0;
 304
 305         if(successful)
 306         {
 307                 state = LOADED;
 308                 data_size = resource->get_data_size();
 309
 310                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 311                         (*i)->resource_loaded(*resource);
 312         }
 313         else
 314         {
 315                 resource->unload();
 316                 state = NOT_LOADED;
 317         }
 318 }
 319
 320 void ResourceManager::ManagedResource::finish_loading()
 321 {
 322         finish_loading(state==LOAD_FINISHED);
 323 }
 324
 325 void ResourceManager::ManagedResource::unload()
 326 {
 327         resource->unload();
 328         state = NOT_LOADED;
 329
 330         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 331                 (*i)->resource_unloaded(*resource);
 332 }
 333
 334 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 335 {
 336         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 337                 watchers.push_back(&w);
 338 }
 339
 340 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 341 {
 342         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 343         if(end!=watchers.end())
 344                 watchers.erase(end, watchers.end());
 345 }
 346
 347
 348 ResourceManager::LoadingThread::LoadingThread():
 349         sem(1),
 350         capacity(2),
 351         size(0),
 352         done(false)
 353 {
 354         launch();
 355 }
 356
 357 void ResourceManager::LoadingThread::main()
 358 {
 359         bool wait_for_work = false;
 360         while(!done)
 361         {
 362                 if(wait_for_work)
 363                         sem.wait();
 364
 365                 if(ManagedResource *managed = front(async_queue))
 366                 {
 367                         try
 368                         {
 369                                 managed->process(false);
 370                         }
 371                         catch(const exception &e)
 372                         {
 373                                 MutexLock lock(queue_mutex);
 374                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 375                                 managed->state = ManagedResource::LOAD_ERROR;
 376                         }
 377
 378                         MutexLock lock(queue_mutex);
 379                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 380                         wait_for_work = async_queue.empty();
 381                 }
 382                 else
 383                         wait_for_work = true;
 384         }
 385 }
 386
 387 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 388 {
 389         MutexLock lock(queue_mutex);
 390         if(queue.empty())
 391                 return 0;
 392
 393         return queue.front();
 394 }
 395
 396 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 397 {
 398         r.start_loading();
 399
 400         MutexLock lock(queue_mutex);
 401         if(r.loader->needs_sync())
 402                 sync_queue.push_back(&r);
 403         else
 404         {
 405                 bool was_empty = async_queue.empty();
 406                 async_queue.push_back(&r);
 407                 if(was_empty)
 408                         sem.signal();
 409         }
 410
 411         ++size;
 412 }
 413
 414 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 415 {
 416         while(!try_remove_resource(r))
 417                 Time::sleep(Time::msec);
 418
 419         r.finish_loading();
 420 }
 421
 422 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 423 {
 424         MutexLock lock(queue_mutex);
 425
 426         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 427         if(i==async_queue.end())
 428         {
 429                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 430                 if(i!=sync_queue.end())
 431                         sync_queue.erase(i);
 432         }
 433         else if(i==async_queue.begin())
 434                 return false;
 435         else
 436                 async_queue.erase(i);
 437
 438         return true;
 439 }
 440
 441 bool ResourceManager::LoadingThread::sync()
 442 {
 443         {
 444                 MutexLock lock(queue_mutex);
 445
 446                 if(!error_queue.empty())
 447                 {
 448                         resource_load_error err = error_queue.front();
 449                         error_queue.pop_front();
 450                         throw err;
 451                 }
 452
 453                 unsigned async_size = async_queue.size();
 454                 if(async_size==0 && size==capacity)
 455                         ++capacity;
 456                 else if(async_size>2 && capacity>2)
 457                         --capacity;
 458         }
 459
 460         bool any_finished = false;
 461         while(ManagedResource *managed = front(sync_queue))
 462         {
 463                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 464                 {
 465                         managed->finish_loading();
 466                         any_finished = true;
 467                         --size;
 468
 469                         MutexLock lock(queue_mutex);
 470                         sync_queue.pop_front();
 471                 }
 472                 else
 473                 {
 474                         MutexLock lock(queue_mutex);
 475                         bool was_empty = async_queue.empty();
 476                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 477                         if(was_empty)
 478                                 sem.signal();
 479                 }
 480         }
 481
 482         return any_finished;
 483 }
 484
 485 void ResourceManager::LoadingThread::terminate()
 486 {
 487         done = true;
 488         sem.signal();
 489         join();
 490         async_queue.clear();
 491         sync_queue.clear();
 492 }
 493
 494 } // namespace GL
 495 } // namespace Msp