source/resourcemanager.cpp

   1 #include <algorithm>
   2 #include <typeinfo>
   3 #include <msp/debug/demangle.h>
   4 #include <msp/strings/format.h>
   5 #include <msp/time/utils.h>
   6 #include "resourcemanager.h"
   7 #include "resources.h"
   8 #include "resourcewatcher.h"
   9
  10 using namespace std;
  11
  12 namespace Msp {
  13 namespace GL {
  14
  15 resource_load_error::resource_load_error(const string &name, const string &err):
  16         runtime_error(format("%s: %s", name, err))
  17 { }
  18
  19 resource_load_error::resource_load_error(const string &name, const exception &exc):
  20         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
  21 { }
  22
  23
  24 ResourceManager::ResourceManager():
  25         policy(LOAD_ON_DEMAND),
  26         async_loads(true),
  27         size_limit(0),
  28         frame(0),
  29         min_retain_frames(30),
  30         max_retain_frames(0),
  31         next_unload(0)
  32 { }
  33
  34 ResourceManager::~ResourceManager()
  35 {
  36         thread.terminate();
  37
  38         while(!resources.empty())
  39                 resources.begin()->second.resource->set_manager(0);
  40 }
  41
  42 void ResourceManager::set_loading_policy(LoadingPolicy p)
  43 {
  44         policy = p;
  45 }
  46
  47 void ResourceManager::set_async_loads(bool a)
  48 {
  49         async_loads = a;
  50 }
  51
  52 void ResourceManager::set_size_limit(UInt64 s)
  53 {
  54         size_limit = s;
  55 }
  56
  57 void ResourceManager::set_min_retain_frames(unsigned f)
  58 {
  59         min_retain_frames = f;
  60 }
  61
  62 void ResourceManager::set_max_retain_frames(unsigned f)
  63 {
  64         max_retain_frames = f;
  65 }
  66
  67 void ResourceManager::add_resource(Resource &r)
  68 {
  69         insert_unique(resources, &r, ManagedResource(r));
  70 }
  71
  72 void *ResourceManager::get_data_for_resource(const Resource &r)
  73 {
  74         return &get_item(resources, &r);
  75 }
  76
  77 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
  78 {
  79         set_resource_location(r, ResourceLocation(c, n));
  80 }
  81
  82 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
  83 {
  84         ManagedResource &managed = get_item(resources, &r);
  85         managed.location = l;
  86
  87         if(policy==LOAD_IMMEDIATELY)
  88                 load_resource(r);
  89 }
  90
  91 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
  92 {
  93         const ManagedResource &managed = get_item(resources, &r);
  94         return managed.location.collection ? &managed.location : 0;
  95 }
  96
  97 void ResourceManager::load_resource(Resource &r)
  98 {
  99         ManagedResource &managed = get_item(resources, &r);
 100         if(!managed.location.collection)
 101                 throw runtime_error("no location");
 102
 103         if(managed.state!=ManagedResource::NOT_LOADED)
 104                 return;
 105
 106         if(async_loads)
 107         {
 108                 managed.state = ManagedResource::LOAD_QUEUED;
 109                 LoadQueue::iterator i;
 110                 for(i=queue.begin(); (i!=queue.end() && (*i)->load_priority>=managed.load_priority); ++i) ;
 111                 queue.insert(i, &managed);
 112         }
 113         else
 114         {
 115                 managed.start_loading();
 116                 while(!managed.loader->process()) ;
 117                 managed.finish_loading(true);
 118         }
 119 }
 120
 121 bool ResourceManager::is_resource_loaded(const Resource &r) const
 122 {
 123         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 124         return managed ? managed->state==ManagedResource::LOADED : false;
 125 }
 126
 127 void ResourceManager::resource_used(const Resource &r)
 128 {
 129         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
 130         if(!managed)
 131                 return;
 132         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
 133                 load_resource(*managed->resource);
 134
 135         managed->last_used = frame;
 136         if(max_retain_frames && !next_unload)
 137                 next_unload = frame+max_retain_frames+1;
 138 }
 139
 140 void ResourceManager::remove_resource(Resource &r)
 141 {
 142         ManagedResource &managed = get_item(resources, &r);
 143         ManagedResource::State state = managed.state;
 144         if(state==ManagedResource::LOAD_QUEUED)
 145         {
 146                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
 147                 if(i!=queue.end())
 148                         queue.erase(i);
 149         }
 150         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
 151                 thread.remove_resource(managed);
 152         remove_existing(resources, &r);
 153 }
 154
 155 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
 156 {
 157         get_item(resources, &r).add_watcher(w);
 158 }
 159
 160 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
 161 {
 162         get_item(resources, &r).remove_watcher(w);
 163 }
 164
 165 void ResourceManager::tick()
 166 {
 167         ++frame;
 168
 169         bool do_unload = (frame>=next_unload);
 170         if(thread.sync())
 171                 do_unload = true;
 172
 173         if(thread.needs_work() && !queue.empty())
 174                 dispatch_work();
 175
 176         if(do_unload)
 177         {
 178                 if(max_retain_frames && frame>=next_unload)
 179                 {
 180                         unload_by_age();
 181
 182                         next_unload = frame;
 183                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 184                                 if(i->second.state==ManagedResource::LOADED)
 185                                         next_unload = min(next_unload, i->second.last_used);
 186                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
 187                 }
 188
 189                 if(size_limit)
 190                         unload_by_size();
 191         }
 192 }
 193
 194 void ResourceManager::dispatch_work()
 195 {
 196         queue.sort(age_order);
 197
 198         if(queue.front()->last_used+10<frame)
 199         {
 200                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
 201                         (*i)->state = ManagedResource::NOT_LOADED;
 202                 queue.clear();
 203                 return;
 204         }
 205
 206         while(thread.needs_work() && !queue.empty())
 207         {
 208                 ManagedResource *managed = queue.front();
 209                 queue.pop_front();
 210                 thread.add_resource(*managed);
 211         }
 212 }
 213
 214 void ResourceManager::unload_by_age()
 215 {
 216         unsigned unload_limit = frame-max_retain_frames;
 217
 218         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 219                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 220                         i->second.unload();
 221 }
 222
 223 void ResourceManager::unload_by_size()
 224 {
 225         unsigned unload_limit = frame-min_retain_frames;
 226
 227         while(get_total_data_size()>size_limit)
 228         {
 229                 ManagedResource *best = 0;
 230                 UInt64 best_impact = 0;
 231                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
 232                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
 233                         {
 234                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
 235                                 if(!best || impact>best_impact)
 236                                 {
 237                                         best = &i->second;
 238                                         best_impact = impact;
 239                                 }
 240                         }
 241
 242                 if(!best)
 243                         break;
 244
 245                 best->unload();
 246         }
 247 }
 248
 249 UInt64 ResourceManager::get_total_data_size() const
 250 {
 251         UInt64 total = 0;
 252         for(ResourceMap::const_iterator i=resources.begin(); i!=resources.end(); ++i)
 253                 if(i->second.state==ManagedResource::LOADED)
 254                         total += i->second.data_size;
 255         return total;
 256 }
 257
 258 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
 259 {
 260         return mr1->last_used>mr2->last_used;
 261 }
 262
 263
 264 ResourceManager::ResourceLocation::ResourceLocation():
 265         collection(0)
 266 { }
 267
 268 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
 269         collection(&c),
 270         name(n)
 271 { }
 272
 273
 274 ResourceManager::ManagedResource::ManagedResource(Resource &r):
 275         resource(&r),
 276         load_priority(r.get_load_priority()),
 277         io(0),
 278         loader(0),
 279         state(NOT_LOADED),
 280         last_used(0),
 281         data_size(0)
 282 { }
 283
 284 void ResourceManager::ManagedResource::start_loading()
 285 {
 286         io = location.collection->open_raw(location.name);
 287         if(!io)
 288                 throw resource_load_error(location.name, "open failed");
 289
 290         const Resources *res = dynamic_cast<Resources *>(location.collection);
 291         loader = resource->load(*io, res);
 292         if(!loader)
 293         {
 294                 delete io;
 295                 io = 0;
 296                 throw logic_error("no loader created");
 297         }
 298         state = LOADING;
 299 }
 300
 301 bool ResourceManager::ManagedResource::process(bool sync)
 302 {
 303         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
 304                 if(loader->process())
 305                         state = LOAD_FINISHED;
 306
 307         return state==LOAD_FINISHED;
 308 }
 309
 310 void ResourceManager::ManagedResource::finish_loading(bool successful)
 311 {
 312         delete loader;
 313         loader = 0;
 314         delete io;
 315         io = 0;
 316
 317         if(successful)
 318         {
 319                 state = LOADED;
 320                 data_size = resource->get_data_size();
 321
 322                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 323                         (*i)->resource_loaded(*resource);
 324         }
 325         else
 326         {
 327                 resource->unload();
 328                 state = NOT_LOADED;
 329         }
 330 }
 331
 332 void ResourceManager::ManagedResource::finish_loading()
 333 {
 334         finish_loading(state==LOAD_FINISHED);
 335 }
 336
 337 void ResourceManager::ManagedResource::unload()
 338 {
 339         resource->unload();
 340         state = NOT_LOADED;
 341
 342         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
 343                 (*i)->resource_unloaded(*resource);
 344 }
 345
 346 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
 347 {
 348         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
 349                 watchers.push_back(&w);
 350 }
 351
 352 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
 353 {
 354         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
 355         if(end!=watchers.end())
 356                 watchers.erase(end, watchers.end());
 357 }
 358
 359
 360 ResourceManager::LoadingThread::LoadingThread():
 361         sem(1),
 362         capacity(2),
 363         size(0),
 364         done(false)
 365 {
 366         launch();
 367 }
 368
 369 void ResourceManager::LoadingThread::main()
 370 {
 371         bool wait_for_work = false;
 372         while(!done)
 373         {
 374                 if(wait_for_work)
 375                         sem.wait();
 376
 377                 if(ManagedResource *managed = front(async_queue))
 378                 {
 379                         try
 380                         {
 381                                 managed->process(false);
 382                         }
 383                         catch(const exception &e)
 384                         {
 385                                 MutexLock lock(queue_mutex);
 386                                 error_queue.push_back(resource_load_error(managed->location.name, e));
 387                                 managed->state = ManagedResource::LOAD_ERROR;
 388                         }
 389
 390                         MutexLock lock(queue_mutex);
 391                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
 392                         wait_for_work = async_queue.empty();
 393                 }
 394                 else
 395                         wait_for_work = true;
 396         }
 397 }
 398
 399 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
 400 {
 401         MutexLock lock(queue_mutex);
 402         if(queue.empty())
 403                 return 0;
 404
 405         return queue.front();
 406 }
 407
 408 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
 409 {
 410         r.start_loading();
 411
 412         MutexLock lock(queue_mutex);
 413         if(r.loader->needs_sync())
 414                 sync_queue.push_back(&r);
 415         else
 416         {
 417                 bool was_empty = async_queue.empty();
 418                 async_queue.push_back(&r);
 419                 if(was_empty)
 420                         sem.signal();
 421         }
 422
 423         ++size;
 424 }
 425
 426 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
 427 {
 428         while(!try_remove_resource(r))
 429                 Time::sleep(Time::msec);
 430
 431         r.finish_loading();
 432 }
 433
 434 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
 435 {
 436         MutexLock lock(queue_mutex);
 437
 438         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
 439         if(i==async_queue.end())
 440         {
 441                 i = find(sync_queue.begin(), sync_queue.end(), &r);
 442                 if(i!=sync_queue.end())
 443                 {
 444                         sync_queue.erase(i);
 445                         --size;
 446                 }
 447         }
 448         else if(i==async_queue.begin())
 449                 return false;
 450         else
 451         {
 452                 async_queue.erase(i);
 453                 --size;
 454         }
 455
 456         return true;
 457 }
 458
 459 bool ResourceManager::LoadingThread::sync()
 460 {
 461         {
 462                 MutexLock lock(queue_mutex);
 463
 464                 if(!error_queue.empty())
 465                 {
 466                         resource_load_error err = error_queue.front();
 467                         error_queue.pop_front();
 468                         throw err;
 469                 }
 470
 471                 unsigned async_size = async_queue.size();
 472                 if(async_size==0 && size==capacity)
 473                         ++capacity;
 474                 else if(async_size>2 && capacity>2)
 475                         --capacity;
 476         }
 477
 478         bool any_finished = false;
 479         while(ManagedResource *managed = front(sync_queue))
 480         {
 481                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
 482                 {
 483                         managed->finish_loading();
 484                         any_finished = true;
 485                         --size;
 486
 487                         MutexLock lock(queue_mutex);
 488                         sync_queue.pop_front();
 489                 }
 490                 else
 491                 {
 492                         MutexLock lock(queue_mutex);
 493                         bool was_empty = async_queue.empty();
 494                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
 495                         if(was_empty)
 496                                 sem.signal();
 497                 }
 498         }
 499
 500         return any_finished;
 501 }
 502
 503 void ResourceManager::LoadingThread::terminate()
 504 {
 505         done = true;
 506         sem.signal();
 507         join();
 508         async_queue.clear();
 509         sync_queue.clear();
 510 }
 511
 512 } // namespace GL
 513 } // namespace Msp