source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resourcewatcher.h"
   8
   9 using namespace std;
  10
  11 namespace Msp {
  12 namespace GL {
  13
  14 resource_load_error::resource_load_error(const string &name, const exception &exc):
  15         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  16 { }
  17
  18
  19 ResourceManager::ResourceManager():
  20         policy(LOAD_ON_DEMAND),
  21         async_loads(true),
  22         size_limit(0),
  23         frame(0),
  24         min_retain_frames(30),
  25         max_retain_frames(0),
  26         next_unload(0)
  27 { }
  28
  29 ResourceManager::~ResourceManager()
  30 {
  31         thread.terminate();
  32
  33         while(!resources.empty())
  34                 resources.begin()->second.resource->set_manager(0);
  35 }
  36
  37 void ResourceManager::set_loading_policy(LoadingPolicy p)
  38 {
  39         policy = p;
  40 }
  41
  42 void ResourceManager::set_async_loads(bool a)
  43 {
  44         async_loads = a;
  45 }
  46
  47 void ResourceManager::set_size_limit(UInt64 s)
  48 {
  49         size_limit = s;
  50 }
  51
  52 void ResourceManager::set_min_retain_frames(unsigned f)
  53 {
  54         min_retain_frames = f;
  55 }
  56
  57 void ResourceManager::set_max_retain_frames(unsigned f)
  58 {
  59         max_retain_frames = f;
  60 }
  61
  62 void ResourceManager::add_resource(Resource &r)
  63 {
  64         insert_unique(resources, &r, ManagedResource(r));
  65 }
  66
  67 void *ResourceManager::get_data_for_resource(const Resource &r)
  68 {
  69         return &get_item(resources, &r);
  70 }
  71
  72 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  73 {
  74         ManagedResource &managed = get_item(resources, &r);
  75         managed.collection = &c;
  76         managed.name = n;
  77
  78         if(policy==LOAD_IMMEDIATELY)
  79                 load_resource(r);
  80 }
  81
  82 void ResourceManager::load_resource(Resource &r)
  83 {
  84         ManagedResource &managed = get_item(resources, &r);
  85         if(!managed.collection)
  86                 throw runtime_error("no location");
  87
  88         if(managed.state!=ManagedResource::NOT_LOADED)
  89                 return;
  90
  91         if(async_loads)
  92         {
  93                 managed.state = ManagedResource::LOAD_QUEUED;
  94                 queue.push_back(&managed);
  95         }
  96         else
  97         {
  98                 managed.start_loading();
  99                 while(!managed.loader->process()) ;
 100                 managed.finish_loading(true);
 101         }
 102 }
 103
 104 bool ResourceManager::is_resource_loaded(const Resource &r)
 105 {
 106         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 107         return managed ? managed->state==ManagedResource::LOADED : false;
 108 }
 109
 110 void ResourceManager::resource_used(const Resource &r)
 111 {
 112         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 113         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 114                 load_resource(*managed->resource);
 115
 116         managed->last_used = frame;
 117         if(max_retain_frames && !next_unload)
 118                 next_unload = frame+max_retain_frames+1;
 119 }
 120
 121 void ResourceManager::remove_resource(Resource &r)
 122 {
 123         ManagedResource &managed = get_item(resources, &r);
 124         ManagedResource::State state = managed.state;
 125         if(state==ManagedResource::LOAD_QUEUED)
 126         {
 127                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 128                 if(i!=queue.end())
 129                         queue.erase(i);
 130         }
 131         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 132                 thread.remove_resource(managed);
 133         remove_existing(resources, &r);
 134 }
 135
 136 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 137 {
 138         get_item(resources, &r).add_watcher(w);
 139 }
 140
 141 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 142 {
 143         get_item(resources, &r).remove_watcher(w);
 144 }
 145
 146 void ResourceManager::tick()
 147 {
 148         ++frame;
 149
 150         bool do_unload = (frame>=next_unload);
 151         if(thread.sync())
 152                 do_unload = true;
 153
 154         if(thread.needs_work() && !queue.empty())
 155                 dispatch_work();
 156
 157         if(do_unload)
 158         {
 159                 if(max_retain_frames && frame>=next_unload)
 160                 {
 161                         unload_by_age();
 162
 163                         next_unload = frame;
 164                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 165                                 if(i->second.state==ManagedResource::LOADED)
 166                                         next_unload = min(next_unload, i->second.last_used);
 167                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 168                 }
 169
 170                 if(size_limit)
 171                         unload_by_size();
 172         }
 173 }
 174
 175 void ResourceManager::dispatch_work()
 176 {
 177         queue.sort(age_order);
 178
 179         if(queue.front()->last_used+10<frame)
 180         {
 181                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 182                         (*i)->state = ManagedResource::NOT_LOADED;
 183                 queue.clear();
 184                 return;
 185         }
 186
 187         while(thread.needs_work() && !queue.empty())
 188         {
 189                 ManagedResource *managed = queue.front();
 190                 queue.pop_front();
 191                 thread.add_resource(*managed);
 192         }
 193 }
 194
 195 void ResourceManager::unload_by_age()
 196 {
 197         unsigned unload_limit = frame-max_retain_frames;
 198
 199         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 200                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 201                         i->second.unload();
 202 }
 203
 204 void ResourceManager::unload_by_size()
 205 {
 206         unsigned unload_limit = frame-min_retain_frames;
 207
 208         while(get_total_data_size()>size_limit)
 209         {
 210                 ManagedResource *best = 0;
 211                 UInt64 best_impact = 0;
 212                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 213                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 214                         {
 215                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 216                                 if(!best || impact>best_impact)
 217                                 {
 218                                         best = &i->second;
 219                                         best_impact = impact;
 220                                 }
 221                         }
 222
 223                 if(!best)
 224                         break;
 225
 226                 best->unload();
 227         }
 228 }
 229
 230 UInt64 ResourceManager::get_total_data_size() const
 231 {
 232         UInt64 total = 0;
 233         for(ResourceMap::const_iterator i=resources.begin(); i!=resources.end(); ++i)
 234                 if(i->second.state==ManagedResource::LOADED)
 235                         total += i->second.data_size;
 236         return total;
 237 }
 238
 239 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 240 {
 241         return mr1->last_used>mr2->last_used;
 242 }
 243
 244
 245 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 246         resource(&r),
 247         collection(0),
 248         io(0),
 249         loader(0),
 250         state(NOT_LOADED),
 251         last_used(0),
 252         data_size(0)
 253 { }
 254
 255 void ResourceManager::ManagedResource::start_loading()
 256 {
 257         io = collection->open_raw(name);
 258         loader = resource->load(*io);
 259         if(!loader)
 260         {
 261                 delete io;
 262                 io = 0;
 263                 throw logic_error("no loader created");
 264         }
 265         state = LOADING;
 266 }
 267
 268 bool ResourceManager::ManagedResource::process(bool sync)
 269 {
 270         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 271                 if(loader->process())
 272                         state = LOAD_FINISHED;
 273
 274         return state==LOAD_FINISHED;
 275 }
 276
 277 void ResourceManager::ManagedResource::finish_loading(bool successful)
 278 {
 279         delete loader;
 280         loader = 0;
 281         delete io;
 282         io = 0;
 283
 284         if(successful)
 285         {
 286                 state = LOADED;
 287                 data_size = resource->get_data_size();
 288
 289                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 290                         (*i)->resource_loaded(*resource);
 291         }
 292         else
 293         {
 294                 resource->unload();
 295                 state = NOT_LOADED;
 296         }
 297 }
 298
 299 void ResourceManager::ManagedResource::finish_loading()
 300 {
 301         finish_loading(state==LOAD_FINISHED);
 302 }
 303
 304 void ResourceManager::ManagedResource::unload()
 305 {
 306         resource->unload();
 307         state = NOT_LOADED;
 308
 309         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 310                 (*i)->resource_unloaded(*resource);
 311 }
 312
 313 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 314 {
 315         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 316                 watchers.push_back(&w);
 317 }
 318
 319 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 320 {
 321         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 322         if(end!=watchers.end())
 323                 watchers.erase(end, watchers.end());
 324 }
 325
 326
 327 ResourceManager::LoadingThread::LoadingThread():
 328         sem(1),
 329         capacity(2),
 330         size(0),
 331         done(false)
 332 {
 333         launch();
 334 }
 335
 336 void ResourceManager::LoadingThread::main()
 337 {
 338         bool wait_for_work = false;
 339         while(!done)
 340         {
 341                 if(wait_for_work)
 342                         sem.wait();
 343
 344                 if(ManagedResource *managed = front(async_queue))
 345                 {
 346                         try
 347                         {
 348                                 managed->process(false);
 349                         }
 350                         catch(const exception &e)
 351                         {
 352                                 MutexLock lock(queue_mutex);
 353                                 error_queue.push_back(resource_load_error(managed->name, e));
 354                                 managed->state = ManagedResource::LOAD_ERROR;
 355                         }
 356
 357                         MutexLock lock(queue_mutex);
 358                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 359                         wait_for_work = async_queue.empty();
 360                 }
 361                 else
 362                         wait_for_work = true;
 363         }
 364 }
 365
 366 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 367 {
 368         MutexLock lock(queue_mutex);
 369         if(queue.empty())
 370                 return 0;
 371
 372         return queue.front();
 373 }
 374
 375 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 376 {
 377         r.start_loading();
 378
 379         MutexLock lock(queue_mutex);
 380         if(r.loader->needs_sync())
 381                 sync_queue.push_back(&r);
 382         else
 383         {
 384                 bool was_empty = async_queue.empty();
 385                 async_queue.push_back(&r);
 386                 if(was_empty)
 387                         sem.signal();
 388         }
 389
 390         ++size;
 391 }
 392
 393 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 394 {
 395         while(!try_remove_resource(r))
 396                 Time::sleep(Time::msec);
 397
 398         r.finish_loading();
 399 }
 400
 401 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 402 {
 403         MutexLock lock(queue_mutex);
 404
 405         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 406         if(i==async_queue.end())
 407         {
 408                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 409                 if(i!=sync_queue.end())
 410                         sync_queue.erase(i);
 411         }
 412         else if(i==async_queue.begin())
 413                 return false;
 414         else
 415                 async_queue.erase(i);
 416
 417         return true;
 418 }
 419
 420 bool ResourceManager::LoadingThread::sync()
 421 {
 422         {
 423                 MutexLock lock(queue_mutex);
 424
 425                 if(!error_queue.empty())
 426                 {
 427                         resource_load_error err = error_queue.front();
 428                         error_queue.pop_front();
 429                         throw err;
 430                 }
 431
 432                 unsigned async_size = async_queue.size();
 433                 if(async_size==0 && size==capacity)
 434                         ++capacity;
 435                 else if(async_size>2 && capacity>2)
 436                         --capacity;
 437         }
 438
 439         bool any_finished = false;
 440         while(ManagedResource *managed = front(sync_queue))
 441         {
 442                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 443                 {
 444                         managed->finish_loading();
 445                         any_finished = true;
 446                         --size;
 447
 448                         MutexLock lock(queue_mutex);
 449                         sync_queue.pop_front();
 450                 }
 451                 else
 452                 {
 453                         MutexLock lock(queue_mutex);
 454                         bool was_empty = async_queue.empty();
 455                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 456                         if(was_empty)
 457                                 sem.signal();
 458                 }
 459         }
 460
 461         return any_finished;
 462 }
 463
 464 void ResourceManager::LoadingThread::terminate()
 465 {
 466         done = true;
 467         sem.signal();
 468         join();
 469         async_queue.clear();
 470         sync_queue.clear();
 471 }
 472
 473 } // namespace GL
 474 } // namespace Msp