]> git.tdb.fi Git - libs/gl.git/blob - source/resourcemanager.cpp
Give meshes a higher priority when loading
[libs/gl.git] / source / resourcemanager.cpp
1 #include <algorithm>
2 #include <typeinfo>
3 #include <msp/debug/demangle.h>
4 #include <msp/strings/format.h>
5 #include <msp/time/utils.h>
6 #include "resourcemanager.h"
7 #include "resources.h"
8 #include "resourcewatcher.h"
9
10 using namespace std;
11
12 namespace Msp {
13 namespace GL {
14
15 resource_load_error::resource_load_error(const string &name, const string &err):
16         runtime_error(format("%s: %s", name, err))
17 { }
18
19 resource_load_error::resource_load_error(const string &name, const exception &exc):
20         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
21 { }
22
23
24 ResourceManager::ResourceManager():
25         policy(LOAD_ON_DEMAND),
26         async_loads(true),
27         size_limit(0),
28         frame(0),
29         min_retain_frames(30),
30         max_retain_frames(0),
31         next_unload(0)
32 { }
33
34 ResourceManager::~ResourceManager()
35 {
36         thread.terminate();
37
38         while(!resources.empty())
39                 resources.begin()->second.resource->set_manager(0);
40 }
41
42 void ResourceManager::set_loading_policy(LoadingPolicy p)
43 {
44         policy = p;
45 }
46
47 void ResourceManager::set_async_loads(bool a)
48 {
49         async_loads = a;
50 }
51
52 void ResourceManager::set_size_limit(UInt64 s)
53 {
54         size_limit = s;
55 }
56
57 void ResourceManager::set_min_retain_frames(unsigned f)
58 {
59         min_retain_frames = f;
60 }
61
62 void ResourceManager::set_max_retain_frames(unsigned f)
63 {
64         max_retain_frames = f;
65 }
66
67 void ResourceManager::add_resource(Resource &r)
68 {
69         insert_unique(resources, &r, ManagedResource(r));
70 }
71
72 void *ResourceManager::get_data_for_resource(const Resource &r)
73 {
74         return &get_item(resources, &r);
75 }
76
77 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
78 {
79         set_resource_location(r, ResourceLocation(c, n));
80 }
81
82 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
83 {
84         ManagedResource &managed = get_item(resources, &r);
85         managed.location = l;
86
87         if(policy==LOAD_IMMEDIATELY)
88                 load_resource(r);
89 }
90
91 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
92 {
93         const ManagedResource &managed = get_item(resources, &r);
94         return managed.location.collection ? &managed.location : 0;
95 }
96
97 void ResourceManager::load_resource(Resource &r)
98 {
99         ManagedResource &managed = get_item(resources, &r);
100         if(!managed.location.collection)
101                 throw runtime_error("no location");
102
103         if(managed.state!=ManagedResource::NOT_LOADED)
104                 return;
105
106         if(async_loads)
107         {
108                 managed.state = ManagedResource::LOAD_QUEUED;
109                 LoadQueue::iterator i;
110                 for(i=queue.begin(); (i!=queue.end() && (*i)->load_priority>=managed.load_priority); ++i) ;
111                 queue.insert(i, &managed);
112         }
113         else
114         {
115                 managed.start_loading();
116                 while(!managed.loader->process()) ;
117                 managed.finish_loading(true);
118         }
119 }
120
121 bool ResourceManager::is_resource_loaded(const Resource &r) const
122 {
123         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
124         return managed ? managed->state==ManagedResource::LOADED : false;
125 }
126
127 void ResourceManager::resource_used(const Resource &r)
128 {
129         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
130         if(!managed)
131                 return;
132         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
133                 load_resource(*managed->resource);
134
135         managed->last_used = frame;
136         if(max_retain_frames && !next_unload)
137                 next_unload = frame+max_retain_frames+1;
138 }
139
140 void ResourceManager::remove_resource(Resource &r)
141 {
142         ManagedResource &managed = get_item(resources, &r);
143         ManagedResource::State state = managed.state;
144         if(state==ManagedResource::LOAD_QUEUED)
145         {
146                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
147                 if(i!=queue.end())
148                         queue.erase(i);
149         }
150         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
151                 thread.remove_resource(managed);
152         remove_existing(resources, &r);
153 }
154
155 void ResourceManager::watch_resource(const Resource &r, ResourceWatcher &w)
156 {
157         get_item(resources, &r).add_watcher(w);
158 }
159
160 void ResourceManager::unwatch_resource(const Resource &r, ResourceWatcher &w)
161 {
162         get_item(resources, &r).remove_watcher(w);
163 }
164
165 void ResourceManager::tick()
166 {
167         ++frame;
168
169         bool do_unload = (frame>=next_unload);
170         if(thread.sync())
171                 do_unload = true;
172
173         if(thread.needs_work() && !queue.empty())
174                 dispatch_work();
175
176         if(do_unload)
177         {
178                 if(max_retain_frames && frame>=next_unload)
179                 {
180                         unload_by_age();
181
182                         next_unload = frame;
183                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
184                                 if(i->second.state==ManagedResource::LOADED)
185                                         next_unload = min(next_unload, i->second.last_used);
186                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
187                 }
188
189                 if(size_limit)
190                         unload_by_size();
191         }
192 }
193
194 void ResourceManager::dispatch_work()
195 {
196         queue.sort(age_order);
197
198         if(queue.front()->last_used+10<frame)
199         {
200                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
201                         (*i)->state = ManagedResource::NOT_LOADED;
202                 queue.clear();
203                 return;
204         }
205
206         while(thread.needs_work() && !queue.empty())
207         {
208                 ManagedResource *managed = queue.front();
209                 queue.pop_front();
210                 thread.add_resource(*managed);
211         }
212 }
213
214 void ResourceManager::unload_by_age()
215 {
216         unsigned unload_limit = frame-max_retain_frames;
217
218         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
219                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
220                         i->second.unload();
221 }
222
223 void ResourceManager::unload_by_size()
224 {
225         unsigned unload_limit = frame-min_retain_frames;
226
227         while(get_total_data_size()>size_limit)
228         {
229                 ManagedResource *best = 0;
230                 UInt64 best_impact = 0;
231                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
232                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
233                         {
234                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
235                                 if(!best || impact>best_impact)
236                                 {
237                                         best = &i->second;
238                                         best_impact = impact;
239                                 }
240                         }
241
242                 if(!best)
243                         break;
244
245                 best->unload();
246         }
247 }
248
249 UInt64 ResourceManager::get_total_data_size() const
250 {
251         UInt64 total = 0;
252         for(ResourceMap::const_iterator i=resources.begin(); i!=resources.end(); ++i)
253                 if(i->second.state==ManagedResource::LOADED)
254                         total += i->second.data_size;
255         return total;
256 }
257
258 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
259 {
260         return mr1->last_used>mr2->last_used;
261 }
262
263
264 ResourceManager::ResourceLocation::ResourceLocation():
265         collection(0)
266 { }
267
268 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
269         collection(&c),
270         name(n)
271 { }
272
273
274 ResourceManager::ManagedResource::ManagedResource(Resource &r):
275         resource(&r),
276         load_priority(r.get_load_priority()),
277         io(0),
278         loader(0),
279         state(NOT_LOADED),
280         last_used(0),
281         data_size(0)
282 { }
283
284 void ResourceManager::ManagedResource::start_loading()
285 {
286         io = location.collection->open_raw(location.name);
287         if(!io)
288                 throw resource_load_error(location.name, "open failed");
289
290         const Resources *res = dynamic_cast<Resources *>(location.collection);
291         loader = resource->load(*io, res);
292         if(!loader)
293         {
294                 delete io;
295                 io = 0;
296                 throw logic_error("no loader created");
297         }
298         state = LOADING;
299 }
300
301 bool ResourceManager::ManagedResource::process(bool sync)
302 {
303         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
304                 if(loader->process())
305                         state = LOAD_FINISHED;
306
307         return state==LOAD_FINISHED;
308 }
309
310 void ResourceManager::ManagedResource::finish_loading(bool successful)
311 {
312         delete loader;
313         loader = 0;
314         delete io;
315         io = 0;
316
317         if(successful)
318         {
319                 state = LOADED;
320                 data_size = resource->get_data_size();
321
322                 for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
323                         (*i)->resource_loaded(*resource);
324         }
325         else
326         {
327                 resource->unload();
328                 state = NOT_LOADED;
329         }
330 }
331
332 void ResourceManager::ManagedResource::finish_loading()
333 {
334         finish_loading(state==LOAD_FINISHED);
335 }
336
337 void ResourceManager::ManagedResource::unload()
338 {
339         resource->unload();
340         state = NOT_LOADED;
341
342         for(vector<ResourceWatcher *>::const_iterator i=watchers.begin(); i!=watchers.end(); ++i)
343                 (*i)->resource_unloaded(*resource);
344 }
345
346 void ResourceManager::ManagedResource::add_watcher(ResourceWatcher &w)
347 {
348         if(find(watchers.begin(), watchers.end(), &w)==watchers.end())
349                 watchers.push_back(&w);
350 }
351
352 void ResourceManager::ManagedResource::remove_watcher(ResourceWatcher &w)
353 {
354         vector<ResourceWatcher *>::iterator end = remove(watchers.begin(), watchers.end(), &w);
355         if(end!=watchers.end())
356                 watchers.erase(end, watchers.end());
357 }
358
359
360 ResourceManager::LoadingThread::LoadingThread():
361         sem(1),
362         capacity(2),
363         size(0),
364         done(false)
365 {
366         launch();
367 }
368
369 void ResourceManager::LoadingThread::main()
370 {
371         bool wait_for_work = false;
372         while(!done)
373         {
374                 if(wait_for_work)
375                         sem.wait();
376
377                 if(ManagedResource *managed = front(async_queue))
378                 {
379                         try
380                         {
381                                 managed->process(false);
382                         }
383                         catch(const exception &e)
384                         {
385                                 MutexLock lock(queue_mutex);
386                                 error_queue.push_back(resource_load_error(managed->location.name, e));
387                                 managed->state = ManagedResource::LOAD_ERROR;
388                         }
389
390                         MutexLock lock(queue_mutex);
391                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
392                         wait_for_work = async_queue.empty();
393                 }
394                 else
395                         wait_for_work = true;
396         }
397 }
398
399 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &queue)
400 {
401         MutexLock lock(queue_mutex);
402         if(queue.empty())
403                 return 0;
404
405         return queue.front();
406 }
407
408 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
409 {
410         r.start_loading();
411
412         MutexLock lock(queue_mutex);
413         if(r.loader->needs_sync())
414                 sync_queue.push_back(&r);
415         else
416         {
417                 bool was_empty = async_queue.empty();
418                 async_queue.push_back(&r);
419                 if(was_empty)
420                         sem.signal();
421         }
422
423         ++size;
424 }
425
426 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
427 {
428         while(!try_remove_resource(r))
429                 Time::sleep(Time::msec);
430
431         r.finish_loading();
432 }
433
434 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
435 {
436         MutexLock lock(queue_mutex);
437
438         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
439         if(i==async_queue.end())
440         {
441                 i = find(sync_queue.begin(), sync_queue.end(), &r);
442                 if(i!=sync_queue.end())
443                         sync_queue.erase(i);
444         }
445         else if(i==async_queue.begin())
446                 return false;
447         else
448                 async_queue.erase(i);
449
450         return true;
451 }
452
453 bool ResourceManager::LoadingThread::sync()
454 {
455         {
456                 MutexLock lock(queue_mutex);
457
458                 if(!error_queue.empty())
459                 {
460                         resource_load_error err = error_queue.front();
461                         error_queue.pop_front();
462                         throw err;
463                 }
464
465                 unsigned async_size = async_queue.size();
466                 if(async_size==0 && size==capacity)
467                         ++capacity;
468                 else if(async_size>2 && capacity>2)
469                         --capacity;
470         }
471
472         bool any_finished = false;
473         while(ManagedResource *managed = front(sync_queue))
474         {
475                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
476                 {
477                         managed->finish_loading();
478                         any_finished = true;
479                         --size;
480
481                         MutexLock lock(queue_mutex);
482                         sync_queue.pop_front();
483                 }
484                 else
485                 {
486                         MutexLock lock(queue_mutex);
487                         bool was_empty = async_queue.empty();
488                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
489                         if(was_empty)
490                                 sem.signal();
491                 }
492         }
493
494         return any_finished;
495 }
496
497 void ResourceManager::LoadingThread::terminate()
498 {
499         done = true;
500         sem.signal();
501         join();
502         async_queue.clear();
503         sync_queue.clear();
504 }
505
506 } // namespace GL
507 } // namespace Msp