source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resources.h"
   8 #include "resourcewatcher.h"
   9
  10 using namespace std;
  11
  12 namespace Msp {
  13 namespace GL {
  14
  15 resource_load_error::resource_load_error(const string &name, const string &err):
  16         runtime_error(format("%s: %s", name, err))
  17 { }
  18
  19 resource_load_error::resource_load_error(const string &name, const exception &exc):
  20         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  21 { }
  22
  23
  24 ResourceManager::ResourceManager():
  25         policy(LOAD_ON_DEMAND),
  26         async_loads(true),
  27         size_limit(0),
  28         frame(0),
  29         min_retain_frames(30),
  30         max_retain_frames(0),
  31         next_unload(0)
  32 { }
  33
  34 ResourceManager::~ResourceManager()
  35 {
  36         thread.terminate();
  37
  38         while(!resources.empty())
  39                 resources.begin()->second.resource->set_manager(0);
  40 }
  41
  42 void ResourceManager::set_loading_policy(LoadingPolicy p)
  43 {
  44         policy = p;
  45 }
  46
  47 void ResourceManager::set_async_loads(bool a)
  48 {
  49         async_loads = a;
  50 }
  51
  52 void ResourceManager::set_size_limit(UInt64 s)
  53 {
  54         size_limit = s;
  55 }
  56
  57 void ResourceManager::set_min_retain_frames(unsigned f)
  58 {
  59         min_retain_frames = f;
  60 }
  61
  62 void ResourceManager::set_max_retain_frames(unsigned f)
  63 {
  64         max_retain_frames = f;
  65 }
  66
  67 void ResourceManager::add_resource(Resource &r)
  68 {
  69         insert_unique(resources, &r, ManagedResource(r));
  70 }
  71
  72 void *ResourceManager::get_data_for_resource(const Resource &r)
  73 {
  74         return &get_item(resources, &r);
  75 }
  76
  77 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  78 {
  79         set_resource_location(r, ResourceLocation(c, n));
  80 }
  81
  82 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  83 {
  84         ManagedResource &managed = get_item(resources, &r);
  85         managed.location = l;
  86
  87         if(policy==LOAD_IMMEDIATELY)
  88                 load_resource(r);
  89 }
  90
  91 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
  92 {
  93         const ManagedResource &managed = get_item(resources, &r);
  94         return managed.location.collection ? &managed.location : 0;
  95 }
  96
  97 void ResourceManager::load_resource(Resource &r)
  98 {
  99         ManagedResource &managed = get_item(resources, &r);
 100         if(!managed.location.collection)
 101                 throw runtime_error("no location");
 102
 103         if(managed.state!=ManagedResource::NOT_LOADED)
 104                 return;
 105
 106         if(async_loads)
 107         {
 108                 managed.state = ManagedResource::LOAD_QUEUED;
 109                 queue.push_back(&managed);
 110         }
 111         else
 112         {
 113                 managed.start_loading();
 114                 while(!managed.loader->process()) ;
 115                 managed.finish_loading(true);
 116         }
 117 }
 118
 119 bool ResourceManager::is_resource_loaded(const Resource &r) const
 120 {
 121         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 122         return managed ? managed->state==ManagedResource::LOADED : false;
 123 }
 124
 125 void ResourceManager::resource_used(const Resource &r)
 126 {
 127         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 128         if(!managed)
 129                 return;
 130         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 131                 load_resource(*managed->resource);
 132
 133         managed->last_used = frame;
 134         if(max_retain_frames && !next_unload)
 135                 next_unload = frame+max_retain_frames+1;
 136 }
 137
 138 void ResourceManager::remove_resource(Resource &r)
 139 {
 140         ManagedResource &managed = get_item(resources, &r);
 141         ManagedResource::State state = managed.state;
 142         if(state==ManagedResource::LOAD_QUEUED)
 143         {
 144                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 145                 if(i!=queue.end())
 146                         queue.erase(i);
 147         }
 148         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 149                 thread.remove_resource(managed);
 150         remove_existing(resources, &r);
 151 }
 152
 153 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 154 {
 155         get_item(resources, &r).add_watcher(w);
 156 }
 157
 158 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 159 {
 160         get_item(resources, &r).remove_watcher(w);
 161 }
 162
 163 void ResourceManager::tick()
 164 {
 165         ++frame;
 166
 167         bool do_unload = (frame>=next_unload);
 168         if(thread.sync())
 169                 do_unload = true;
 170
 171         if(thread.needs_work() && !queue.empty())
 172                 dispatch_work();
 173
 174         if(do_unload)
 175         {
 176                 if(max_retain_frames && frame>=next_unload)
 177                 {
 178                         unload_by_age();
 179
 180                         next_unload = frame;
 181                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 182                                 if(i->second.state==ManagedResource::LOADED)
 183                                         next_unload = min(next_unload, i->second.last_used);
 184                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 185                 }
 186
 187                 if(size_limit)
 188                         unload_by_size();
 189         }
 190 }
 191
 192 void ResourceManager::dispatch_work()
 193 {
 194         queue.sort(age_order);
 195
 196         if(queue.front()->last_used+10<frame)
 197         {
 198                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 199                         (*i)->state = ManagedResource::NOT_LOADED;
 200                 queue.clear();
 201                 return;
 202         }
 203
 204         while(thread.needs_work() && !queue.empty())
 205         {
 206                 ManagedResource *managed = queue.front();
 207                 queue.pop_front();
 208                 thread.add_resource(*managed);
 209         }
 210 }
 211
 212 void ResourceManager::unload_by_age()
 213 {
 214         unsigned unload_limit = frame-max_retain_frames;
 215
 216         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 217                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 218                         i->second.unload();
 219 }
 220
 221 void ResourceManager::unload_by_size()
 222 {
 223         unsigned unload_limit = frame-min_retain_frames;
 224
 225         while(get_total_data_size()>size_limit)
 226         {
 227                 ManagedResource *best = 0;
 228                 UInt64 best_impact = 0;
 229                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 230                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 231                         {
 232                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 233                                 if(!best || impact>best_impact)
 234                                 {
 235                                         best = &i->second;
 236                                         best_impact = impact;
 237                                 }
 238                         }
 239
 240                 if(!best)
 241                         break;
 242
 243                 best->unload();
 244         }
 245 }
 246
 247 UInt64 ResourceManager::get_total_data_size() const
 248 {
 249         UInt64 total = 0;
 250         for(ResourceMap::const_iterator i=resources.begin(); i!=resources.end(); ++i)
 251                 if(i->second.state==ManagedResource::LOADED)
 252                         total += i->second.data_size;
 253         return total;
 254 }
 255
 256 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 257 {
 258         return mr1->last_used>mr2->last_used;
 259 }
 260
 261
 262 ResourceManager::ResourceLocation::ResourceLocation():
 263         collection(0)
 264 { }
 265
 266 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 267         collection(&c),
 268         name(n)
 269 { }
 270
 271
 272 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 273         resource(&r),
 274         io(0),
 275         loader(0),
 276         state(NOT_LOADED),
 277         last_used(0),
 278         data_size(0)
 279 { }
 280
 281 void ResourceManager::ManagedResource::start_loading()
 282 {
 283         io = location.collection->open_raw(location.name);
 284         if(!io)
 285                 throw resource_load_error(location.name, "open failed");
 286
 287         const Resources *res = dynamic_cast<Resources *>(location.collection);
 288         loader = resource->load(*io, res);
 289         if(!loader)
 290         {
 291                 delete io;
 292                 io = 0;
 293                 throw logic_error("no loader created");
 294         }
 295         state = LOADING;
 296 }
 297
 298 bool ResourceManager::ManagedResource::process(bool sync)
 299 {
 300         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 301                 if(loader->process())
 302                         state = LOAD_FINISHED;
 303
 304         return state==LOAD_FINISHED;
 305 }
 306
 307 void ResourceManager::ManagedResource::finish_loading(bool successful)
 308 {
 309         delete loader;
 310         loader = 0;
 311         delete io;
 312         io = 0;
 313
 314         if(successful)
 315         {
 316                 state = LOADED;
 317                 data_size = resource->get_data_size();
 318
 319                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 320                         (*i)->resource_loaded(*resource);
 321         }
 322         else
 323         {
 324                 resource->unload();
 325                 state = NOT_LOADED;
 326         }
 327 }
 328
 329 void ResourceManager::ManagedResource::finish_loading()
 330 {
 331         finish_loading(state==LOAD_FINISHED);
 332 }
 333
 334 void ResourceManager::ManagedResource::unload()
 335 {
 336         resource->unload();
 337         state = NOT_LOADED;
 338
 339         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 340                 (*i)->resource_unloaded(*resource);
 341 }
 342
 343 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 344 {
 345         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 346                 watchers.push_back(&w);
 347 }
 348
 349 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 350 {
 351         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 352         if(end!=watchers.end())
 353                 watchers.erase(end, watchers.end());
 354 }
 355
 356
 357 ResourceManager::LoadingThread::LoadingThread():
 358         sem(1),
 359         capacity(2),
 360         size(0),
 361         done(false)
 362 {
 363         launch();
 364 }
 365
 366 void ResourceManager::LoadingThread::main()
 367 {
 368         bool wait_for_work = false;
 369         while(!done)
 370         {
 371                 if(wait_for_work)
 372                         sem.wait();
 373
 374                 if(ManagedResource *managed = front(async_queue))
 375                 {
 376                         try
 377                         {
 378                                 managed->process(false);
 379                         }
 380                         catch(const exception &e)
 381                         {
 382                                 MutexLock lock(queue_mutex);
 383                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 384                                 managed->state = ManagedResource::LOAD_ERROR;
 385                         }
 386
 387                         MutexLock lock(queue_mutex);
 388                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 389                         wait_for_work = async_queue.empty();
 390                 }
 391                 else
 392                         wait_for_work = true;
 393         }
 394 }
 395
 396 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 397 {
 398         MutexLock lock(queue_mutex);
 399         if(queue.empty())
 400                 return 0;
 401
 402         return queue.front();
 403 }
 404
 405 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 406 {
 407         r.start_loading();
 408
 409         MutexLock lock(queue_mutex);
 410         if(r.loader->needs_sync())
 411                 sync_queue.push_back(&r);
 412         else
 413         {
 414                 bool was_empty = async_queue.empty();
 415                 async_queue.push_back(&r);
 416                 if(was_empty)
 417                         sem.signal();
 418         }
 419
 420         ++size;
 421 }
 422
 423 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 424 {
 425         while(!try_remove_resource(r))
 426                 Time::sleep(Time::msec);
 427
 428         r.finish_loading();
 429 }
 430
 431 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 432 {
 433         MutexLock lock(queue_mutex);
 434
 435         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 436         if(i==async_queue.end())
 437         {
 438                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 439                 if(i!=sync_queue.end())
 440                         sync_queue.erase(i);
 441         }
 442         else if(i==async_queue.begin())
 443                 return false;
 444         else
 445                 async_queue.erase(i);
 446
 447         return true;
 448 }
 449
 450 bool ResourceManager::LoadingThread::sync()
 451 {
 452         {
 453                 MutexLock lock(queue_mutex);
 454
 455                 if(!error_queue.empty())
 456                 {
 457                         resource_load_error err = error_queue.front();
 458                         error_queue.pop_front();
 459                         throw err;
 460                 }
 461
 462                 unsigned async_size = async_queue.size();
 463                 if(async_size==0 && size==capacity)
 464                         ++capacity;
 465                 else if(async_size>2 && capacity>2)
 466                         --capacity;
 467         }
 468
 469         bool any_finished = false;
 470         while(ManagedResource *managed = front(sync_queue))
 471         {
 472                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 473                 {
 474                         managed->finish_loading();
 475                         any_finished = true;
 476                         --size;
 477
 478                         MutexLock lock(queue_mutex);
 479                         sync_queue.pop_front();
 480                 }
 481                 else
 482                 {
 483                         MutexLock lock(queue_mutex);
 484                         bool was_empty = async_queue.empty();
 485                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 486                         if(was_empty)
 487                                 sem.signal();
 488                 }
 489         }
 490
 491         return any_finished;
 492 }
 493
 494 void ResourceManager::LoadingThread::terminate()
 495 {
 496         done = true;
 497         sem.signal();
 498         join();
 499         async_queue.clear();
 500         sync_queue.clear();
 501 }
 502
 503 } // namespace GL
 504 } // namespace Msp