]> git.tdb.fi Git - libs/gl.git/blob - source/resources/resourcemanager.cpp
Add a scene type property to the Blender exporter
[libs/gl.git] / source / resources / resourcemanager.cpp
1 #include <algorithm>
2 #include <typeinfo>
3 #include <msp/debug/demangle.h>
4 #include <msp/strings/format.h>
5 #include <msp/time/utils.h>
6 #include "resourcemanager.h"
7 #include "resources.h"
8 #include "resourceobserver.h"
9
10 using namespace std;
11
12 namespace Msp {
13 namespace GL {
14
15 resource_load_error::resource_load_error(const string &name, const string &err):
16         runtime_error(format("%s: %s", name, err))
17 { }
18
19 resource_load_error::resource_load_error(const string &name, const exception &exc):
20         runtime_error(format("%s: %s: %s", name, Debug::demangle(typeid(exc).name()), exc.what()))
21 { }
22
23
24 ResourceManager::ResourceManager():
25         policy(LOAD_ON_DEMAND),
26         async_loads(true),
27         total_data_size(0),
28         size_limit(0),
29         frame(0),
30         min_retain_frames(30),
31         max_retain_frames(0),
32         next_unload(0)
33 { }
34
35 ResourceManager::~ResourceManager()
36 {
37         thread.terminate();
38
39         while(!resources.empty())
40                 resources.begin()->second.resource->set_manager(0);
41 }
42
43 void ResourceManager::set_loading_policy(LoadingPolicy p)
44 {
45         policy = p;
46 }
47
48 void ResourceManager::set_async_loads(bool a)
49 {
50         async_loads = a;
51 }
52
53 void ResourceManager::set_size_limit(UInt64 s)
54 {
55         size_limit = s;
56 }
57
58 void ResourceManager::set_min_retain_frames(unsigned f)
59 {
60         min_retain_frames = f;
61 }
62
63 void ResourceManager::set_max_retain_frames(unsigned f)
64 {
65         max_retain_frames = f;
66 }
67
68 void ResourceManager::add_resource(Resource &r)
69 {
70         MutexLock lock(map_mutex);
71         insert_unique(resources, &r, ManagedResource(r));
72 }
73
74 const ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r) const
75 {
76         MutexLock lock(map_mutex);
77         return get_item(resources, &r);
78 }
79
80 ResourceManager::ManagedResource &ResourceManager::get_managed_resource(const Resource &r)
81 {
82         MutexLock lock(map_mutex);
83         return get_item(resources, &r);
84 }
85
86 void ResourceManager::set_resource_location(Resource &r, DataFile::Collection &c, const string &n)
87 {
88         set_resource_location(r, ResourceLocation(c, n));
89 }
90
91 void ResourceManager::set_resource_location(Resource &r, const ResourceLocation &l)
92 {
93         {
94                 MutexLock lock(map_mutex);
95                 ManagedResource &managed = get_item(resources, &r);
96                 managed.location = l;
97         }
98
99         if(policy==LOAD_IMMEDIATELY)
100                 load_resource(r);
101 }
102
103 const ResourceManager::ResourceLocation *ResourceManager::get_resource_location(const Resource &r) const
104 {
105         const ManagedResource &managed = get_managed_resource(r);
106         return managed.location.collection ? &managed.location : 0;
107 }
108
109 void ResourceManager::load_resource(Resource &r)
110 {
111         ManagedResource &managed = get_managed_resource(r);
112         if(!managed.location.collection)
113                 throw runtime_error("no location");
114
115         if(managed.state!=ManagedResource::NOT_LOADED)
116                 return;
117
118         if(async_loads)
119         {
120                 managed.state = ManagedResource::LOAD_QUEUED;
121                 LoadQueue::iterator i;
122                 for(i=queue.begin(); (i!=queue.end() && (*i)->load_priority>=managed.load_priority); ++i) ;
123                 queue.insert(i, &managed);
124         }
125         else
126         {
127                 managed.start_loading();
128                 while(!managed.loader->process()) ;
129                 managed.finish_loading(true);
130                 total_data_size += managed.data_size;
131         }
132 }
133
134 bool ResourceManager::is_resource_loaded(const Resource &r) const
135 {
136         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
137         return managed ? managed->state==ManagedResource::LOADED : false;
138 }
139
140 void ResourceManager::resource_used(const Resource &r)
141 {
142         ManagedResource *managed = reinterpret_cast<ManagedResource *>(r.get_manager_data());
143         if(!managed)
144                 return;
145         if(managed->state==ManagedResource::NOT_LOADED && policy!=LOAD_MANUALLY)
146                 load_resource(*managed->resource);
147
148         managed->last_used = frame;
149         if(max_retain_frames && !next_unload)
150                 next_unload = frame+max_retain_frames+1;
151 }
152
153 void ResourceManager::remove_resource(Resource &r)
154 {
155         ManagedResource &managed = get_managed_resource(r);
156         ManagedResource::State state = managed.state;
157         if(state==ManagedResource::LOAD_QUEUED)
158         {
159                 LoadQueue::iterator i = find(queue.begin(), queue.end(), &managed);
160                 if(i!=queue.end())
161                         queue.erase(i);
162         }
163         else if(state>ManagedResource::LOAD_QUEUED && state<ManagedResource::LOADED)
164                 thread.remove_resource(managed);
165
166         for(vector<ResourceObserver *>::const_iterator i=managed.observers.begin(); i!=managed.observers.end(); ++i)
167                 (*i)->resource_removed(r);
168
169         MutexLock lock(map_mutex);
170         remove_existing(resources, &r);
171 }
172
173 void ResourceManager::observe_resource(const Resource &r, ResourceObserver &w)
174 {
175         get_managed_resource(r).add_observer(w);
176 }
177
178 void ResourceManager::unobserve_resource(const Resource &r, ResourceObserver &w)
179 {
180         get_managed_resource(r).remove_observer(w);
181 }
182
183 void ResourceManager::tick()
184 {
185         ++frame;
186
187         bool do_unload = (frame>=next_unload);
188         if(thread.sync())
189         {
190                 total_data_size += thread.get_and_reset_loaded_data_size();
191                 do_unload = true;
192         }
193
194         if(thread.needs_work() && !queue.empty())
195                 dispatch_work();
196
197         if(do_unload)
198         {
199                 MutexLock lock(map_mutex);
200                 if(max_retain_frames && frame>=next_unload)
201                 {
202                         unload_by_age();
203
204                         next_unload = frame;
205                         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
206                                 if(i->second.state==ManagedResource::LOADED)
207                                         next_unload = min(next_unload, i->second.last_used);
208                         next_unload = (next_unload<frame ? next_unload+max_retain_frames : 0);
209                 }
210
211                 if(size_limit)
212                         unload_by_size();
213         }
214 }
215
216 void ResourceManager::dispatch_work()
217 {
218         queue.sort(age_order);
219
220         if(queue.front()->last_used+min_retain_frames<frame)
221         {
222                 for(LoadQueue::iterator i=queue.begin(); i!=queue.end(); ++i)
223                         (*i)->state = ManagedResource::NOT_LOADED;
224                 queue.clear();
225                 return;
226         }
227
228         while(thread.needs_work() && !queue.empty())
229         {
230                 ManagedResource *managed = queue.front();
231                 queue.pop_front();
232                 thread.add_resource(*managed);
233         }
234 }
235
236 void ResourceManager::unload_by_age()
237 {
238         unsigned unload_limit = frame-max_retain_frames;
239
240         for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
241                 if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
242                 {
243                         i->second.unload();
244                         total_data_size -= i->second.data_size;
245                 }
246 }
247
248 void ResourceManager::unload_by_size()
249 {
250         unsigned unload_limit = frame-min_retain_frames;
251
252         while(total_data_size>size_limit)
253         {
254                 ManagedResource *best = 0;
255                 UInt64 best_impact = 0;
256                 for(ResourceMap::iterator i=resources.begin(); i!=resources.end(); ++i)
257                         if(i->second.state==ManagedResource::LOADED && i->second.last_used<unload_limit)
258                         {
259                                 UInt64 impact = i->second.data_size*(frame-i->second.last_used);
260                                 if(!best || impact>best_impact)
261                                 {
262                                         best = &i->second;
263                                         best_impact = impact;
264                                 }
265                         }
266
267                 if(!best)
268                         break;
269
270                 best->unload();
271                 total_data_size -= best->data_size;
272         }
273 }
274
275 bool ResourceManager::age_order(ManagedResource *mr1, ManagedResource *mr2)
276 {
277         return mr1->last_used>mr2->last_used;
278 }
279
280
281 ResourceManager::ResourceLocation::ResourceLocation():
282         collection(0)
283 { }
284
285 ResourceManager::ResourceLocation::ResourceLocation(DataFile::Collection &c, const string &n):
286         collection(&c),
287         name(n)
288 { }
289
290
291 ResourceManager::ManagedResource::ManagedResource(Resource &r):
292         resource(&r),
293         load_priority(r.get_load_priority()),
294         io(0),
295         loader(0),
296         state(NOT_LOADED),
297         last_used(0),
298         data_size(0)
299 { }
300
301 void ResourceManager::ManagedResource::start_loading()
302 {
303         io = location.collection->open_raw(location.name);
304         if(!io)
305                 throw resource_load_error(location.name, "open failed");
306
307         const Resources *res = dynamic_cast<Resources *>(location.collection);
308         loader = resource->load(*io, res);
309         if(!loader)
310         {
311                 delete io;
312                 io = 0;
313                 throw logic_error("no loader created");
314         }
315         state = LOADING;
316 }
317
318 bool ResourceManager::ManagedResource::process(bool sync)
319 {
320         while(state!=LOAD_FINISHED && loader->needs_sync()==sync)
321                 if(loader->process())
322                         state = LOAD_FINISHED;
323
324         return state==LOAD_FINISHED;
325 }
326
327 void ResourceManager::ManagedResource::finish_loading(bool successful)
328 {
329         delete loader;
330         loader = 0;
331         delete io;
332         io = 0;
333
334         if(successful)
335         {
336                 state = LOADED;
337                 data_size = resource->get_data_size();
338
339                 for(vector<ResourceObserver *>::const_iterator i=observers.begin(); i!=observers.end(); ++i)
340                         (*i)->resource_loaded(*resource);
341         }
342         else
343         {
344                 resource->unload();
345                 state = NOT_LOADED;
346         }
347 }
348
349 void ResourceManager::ManagedResource::finish_loading()
350 {
351         finish_loading(state==LOAD_FINISHED);
352 }
353
354 void ResourceManager::ManagedResource::unload()
355 {
356         resource->unload();
357         state = NOT_LOADED;
358
359         for(vector<ResourceObserver *>::const_iterator i=observers.begin(); i!=observers.end(); ++i)
360                 (*i)->resource_unloaded(*resource);
361 }
362
363 void ResourceManager::ManagedResource::add_observer(ResourceObserver &w)
364 {
365         if(find(observers.begin(), observers.end(), &w)==observers.end())
366                 observers.push_back(&w);
367 }
368
369 void ResourceManager::ManagedResource::remove_observer(ResourceObserver &w)
370 {
371         vector<ResourceObserver *>::iterator end = remove(observers.begin(), observers.end(), &w);
372         if(end!=observers.end())
373                 observers.erase(end, observers.end());
374 }
375
376
377 ResourceManager::LoadingThread::LoadingThread():
378         Thread("ResourceManager"),
379         sem(1),
380         capacity(2),
381         size(0),
382         loaded_data_size(0),
383         done(false)
384 {
385         launch();
386 }
387
388 void ResourceManager::LoadingThread::main()
389 {
390         bool wait_for_work = false;
391         while(!done)
392         {
393                 if(wait_for_work)
394                         sem.wait();
395
396                 if(ManagedResource *managed = front(async_queue))
397                 {
398                         try
399                         {
400                                 managed->process(false);
401                         }
402                         catch(const exception &e)
403                         {
404                                 MutexLock lock(queue_mutex);
405                                 error_queue.push_back(resource_load_error(managed->location.name, e));
406                                 managed->state = ManagedResource::LOAD_ERROR;
407                         }
408
409                         MutexLock lock(queue_mutex);
410                         sync_queue.splice(sync_queue.end(), async_queue, async_queue.begin());
411                         wait_for_work = async_queue.empty();
412                 }
413                 else
414                         wait_for_work = true;
415         }
416 }
417
418 ResourceManager::ManagedResource *ResourceManager::LoadingThread::front(LoadQueue &que)
419 {
420         MutexLock lock(queue_mutex);
421         if(que.empty())
422                 return 0;
423
424         return que.front();
425 }
426
427 void ResourceManager::LoadingThread::add_resource(ManagedResource &r)
428 {
429         r.start_loading();
430
431         MutexLock lock(queue_mutex);
432         if(r.loader->needs_sync())
433                 sync_queue.push_back(&r);
434         else
435         {
436                 bool was_empty = async_queue.empty();
437                 async_queue.push_back(&r);
438                 if(was_empty)
439                         sem.signal();
440         }
441
442         ++size;
443 }
444
445 void ResourceManager::LoadingThread::remove_resource(ManagedResource &r)
446 {
447         while(!try_remove_resource(r))
448                 Time::sleep(Time::msec);
449
450         r.finish_loading();
451         if(r.state==ManagedResource::LOADED)
452         {
453                 MutexLock lock(data_size_mutex);
454                 loaded_data_size += r.data_size;
455         }
456 }
457
458 bool ResourceManager::LoadingThread::try_remove_resource(ManagedResource &r)
459 {
460         MutexLock lock(queue_mutex);
461
462         LoadQueue::iterator i = find(async_queue.begin(), async_queue.end(), &r);
463         if(i==async_queue.end())
464         {
465                 i = find(sync_queue.begin(), sync_queue.end(), &r);
466                 if(i!=sync_queue.end())
467                 {
468                         sync_queue.erase(i);
469                         --size;
470                 }
471         }
472         else if(i==async_queue.begin())
473                 return false;
474         else
475         {
476                 async_queue.erase(i);
477                 --size;
478         }
479
480         return true;
481 }
482
483 bool ResourceManager::LoadingThread::sync()
484 {
485         {
486                 MutexLock lock(queue_mutex);
487
488                 if(!error_queue.empty())
489                 {
490                         resource_load_error err = error_queue.front();
491                         error_queue.pop_front();
492                         throw err;
493                 }
494
495                 unsigned async_size = async_queue.size();
496                 if(async_size==0 && size==capacity)
497                         ++capacity;
498                 else if(async_size>2 && capacity>2)
499                         --capacity;
500         }
501
502         bool any_finished = false;
503         while(ManagedResource *managed = front(sync_queue))
504         {
505                 if(managed->state==ManagedResource::LOAD_ERROR || managed->process(true))
506                 {
507                         managed->finish_loading();
508                         if(managed->state==ManagedResource::LOADED)
509                         {
510                                 MutexLock lock(data_size_mutex);
511                                 loaded_data_size += managed->data_size;
512                         }
513                         any_finished = true;
514                         --size;
515
516                         MutexLock lock(queue_mutex);
517                         sync_queue.pop_front();
518                 }
519                 else
520                 {
521                         MutexLock lock(queue_mutex);
522                         bool was_empty = async_queue.empty();
523                         async_queue.splice(async_queue.end(), sync_queue, sync_queue.begin());
524                         if(was_empty)
525                                 sem.signal();
526                 }
527         }
528
529         return any_finished;
530 }
531
532 UInt64 ResourceManager::LoadingThread::get_and_reset_loaded_data_size()
533 {
534         MutexLock lock(data_size_mutex);
535         UInt64 result = loaded_data_size;
536         loaded_data_size = 0;
537         return result;
538 }
539
540 void ResourceManager::LoadingThread::terminate()
541 {
542         done = true;
543         sem.signal();
544         join();
545         async_queue.clear();
546         sync_queue.clear();
547 }
548
549 } // namespace GL
550 } // namespace Msp