openscenegraph
DatabasePager
Go to the documentation of this file.
1/* -*-c++-*- OpenSceneGraph - Copyright (C) 1998-2006 Robert Osfield
2 *
3 * This library is open source and may be redistributed and/or modified under
4 * the terms of the OpenSceneGraph Public License (OSGPL) version 0.0 or
5 * (at your option) any later version. The full license is in LICENSE file
6 * included with this distribution, and on the openscenegraph.org website.
7 *
8 * This library is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * OpenSceneGraph Public License for more details.
12*/
13
14#ifndef OSGDB_DATABASEPAGER
15#define OSGDB_DATABASEPAGER 1
16
17#include <osg/NodeVisitor>
18#include <osg/Group>
19#include <osg/PagedLOD>
20#include <osg/Drawable>
21#include <osg/GraphicsThread>
22#include <osg/FrameStamp>
23#include <osg/ObserverNodePath>
24#include <osg/observer_ptr>
25
26#include <OpenThreads/Thread>
27#include <OpenThreads/Mutex>
28#include <OpenThreads/ScopedLock>
29#include <OpenThreads/Condition>
30
31#include <osgUtil/IncrementalCompileOperation>
32
33#include <osgDB/SharedStateManager>
34#include <osgDB/ReaderWriter>
35#include <osgDB/Options>
36
37
38#include <map>
39#include <list>
40#include <algorithm>
41#include <functional>
42
43namespace osgDB {
44
45
46
47/** Database paging class which manages the loading of files in a background thread,
48 * and synchronizing of loaded models with the main scene graph.*/
49class OSGDB_EXPORT DatabasePager : public osg::NodeVisitor::DatabaseRequestHandler
50{
51 public :
52
53 typedef OpenThreads::Thread::ThreadPriority ThreadPriority;
54
55 DatabasePager();
56
57 DatabasePager(const DatabasePager& rhs);
58
59 virtual const char* className() const { return "DatabasePager"; }
60
61 /** Create a shallow copy on the DatabasePager.*/
62 virtual DatabasePager* clone() const { return new DatabasePager(*this); }
63
64 /** get the prototype singleton used by DatabasePager::create().*/
65 static osg::ref_ptr<DatabasePager>& prototype();
66
67 /** create a DatabasePager by cloning DatabasePager::prototype().*/
68 static DatabasePager* create();
69
70
71
72 /** Add a request to load a node file to end the database request list.*/
73 virtual void requestNodeFile(const std::string& fileName, osg::NodePath& nodePath,
74 float priority, const osg::FrameStamp* framestamp,
75 osg::ref_ptr<osg::Referenced>& databaseRequest,
76 const osg::Referenced* options);
77
78 /** Set the priority of the database pager thread(s).*/
79 int setSchedulePriority(OpenThreads::Thread::ThreadPriority priority);
80
81 /** Cancel the database pager thread(s).*/
82 virtual int cancel();
83
84 virtual bool isRunning() const;
85
86 /** Clear all internally cached structures.*/
87 virtual void clear();
88
89 class OSGDB_EXPORT DatabaseThread : public osg::Referenced, public OpenThreads::Thread
90 {
91 public:
92
93 enum Mode
94 {
95 HANDLE_ALL_REQUESTS,
96 HANDLE_NON_HTTP,
97 HANDLE_ONLY_HTTP
98 };
99
100 DatabaseThread(DatabasePager* pager, Mode mode, const std::string& name);
101
102 DatabaseThread(const DatabaseThread& dt, DatabasePager* pager);
103
104 void setName(const std::string& name) { _name = name; }
105 const std::string& getName() const { return _name; }
106
107 void setDone(bool done) { _done.exchange(done?1:0); }
108 bool getDone() const { return _done!=0; }
109
110 void setActive(bool active) { _active = active; }
111 bool getActive() const { return _active; }
112
113 virtual int cancel();
114
115 virtual void run();
116
117 protected:
118
119 virtual ~DatabaseThread();
120
121 OpenThreads::Atomic _done;
122 volatile bool _active;
123 DatabasePager* _pager;
124 Mode _mode;
125 std::string _name;
126
127 };
128
129 virtual void setProcessorAffinity(const OpenThreads::Affinity& affinity);
130 OpenThreads::Affinity& getProcessorAffinity() { return _affinity; }
131 const OpenThreads::Affinity& getProcessorAffinity() const { return _affinity; }
132
133 void setUpThreads(unsigned int totalNumThreads=2, unsigned int numHttpThreads=1);
134
135 virtual unsigned int addDatabaseThread(DatabaseThread::Mode mode, const std::string& name);
136
137 DatabaseThread* getDatabaseThread(unsigned int i) { return _databaseThreads[i].get(); }
138
139 const DatabaseThread* getDatabaseThread(unsigned int i) const { return _databaseThreads[i].get(); }
140
141 unsigned int getNumDatabaseThreads() const { return static_cast<unsigned int>(_databaseThreads.size()); }
142
143 /** Set whether the database pager thread should be paused or not.*/
144 void setDatabasePagerThreadPause(bool pause);
145
146 /** Get whether the database pager thread should is paused or not.*/
147 bool getDatabasePagerThreadPause() const { return _databasePagerThreadPaused; }
148
149 /** Set whether new database request calls are accepted or ignored.*/
150 void setAcceptNewDatabaseRequests(bool acceptNewRequests) { _acceptNewRequests = acceptNewRequests; }
151
152 /** Get whether new database request calls are accepted or ignored.*/
153 bool getAcceptNewDatabaseRequests() const { return _acceptNewRequests; }
154
155 /** Get the number of frames that are currently active.*/
156 int getNumFramesActive() const { return _numFramesActive; }
157
158 /** Signal the database thread that the update, cull and draw has begun for a new frame.
159 * Note, this is called by the application so that the database pager can go to sleep while the CPU is busy on the main rendering threads. */
160 virtual void signalBeginFrame(const osg::FrameStamp* framestamp);
161
162 /** Signal the database thread that the update, cull and draw dispatch has completed.
163 * Note, this is called by the application so that the database pager can go to wake back up now the main rendering threads are iddle waiting for the next frame.*/
164 virtual void signalEndFrame();
165
166
167 /** Find all PagedLOD nodes in a subgraph and register them with
168 * the DatabasePager so it can keep track of expired nodes.
169 * note, should be only be called from the update thread. */
170 virtual void registerPagedLODs(osg::Node* subgraph, unsigned int frameNumber = 0);
171
172 /** Set the incremental compile operation.
173 * Used to manage the OpenGL object compilation and merging of subgraphs in a way that avoids overloading
174 * the rendering of frame with too many new objects in one frame. */
175 void setIncrementalCompileOperation(osgUtil::IncrementalCompileOperation* ico);
176
177 /** Get the incremental compile operation. */
178 osgUtil::IncrementalCompileOperation* getIncrementalCompileOperation() { return _incrementalCompileOperation.get(); }
179
180
181 /** Set whether the database pager should pre compile OpenGL objects before allowing
182 * them to be merged into the scene graph.
183 * Pre compilation helps reduce the chances of frame drops, but also slows the
184 * speed at which tiles are merged as they have to be compiled first.*/
185 void setDoPreCompile(bool flag) { _doPreCompile = flag; }
186
187 /** Get whether the database pager should pre compile OpenGL objects before allowing
188 * them to be merged into the scene graph.*/
189 bool getDoPreCompile() const { return _doPreCompile; }
190
191
192
193 /** Set the target maximum number of PagedLOD to maintain in memory.
194 * Note, if more than the target number are required for rendering of a frame then these active PagedLOD are excempt from being expiried.
195 * But once the number of active drops back below the target the inactive PagedLOD will be trimmed back to the target number.*/
196 void setTargetMaximumNumberOfPageLOD(unsigned int target) { _targetMaximumNumberOfPageLOD = target; }
197
198 /** Get the target maximum number of PagedLOD to maintain in memory.*/
199 unsigned int getTargetMaximumNumberOfPageLOD() const { return _targetMaximumNumberOfPageLOD; }
200
201
202 /** Set whether the removed subgraphs should be deleted in the database thread or not.*/
203 void setDeleteRemovedSubgraphsInDatabaseThread(bool flag) { _deleteRemovedSubgraphsInDatabaseThread = flag; }
204
205 /** Get whether the removed subgraphs should be deleted in the database thread or not.*/
206 bool getDeleteRemovedSubgraphsInDatabaseThread() const { return _deleteRemovedSubgraphsInDatabaseThread; }
207
208 enum DrawablePolicy
209 {
210 DO_NOT_MODIFY_DRAWABLE_SETTINGS,
211 USE_DISPLAY_LISTS,
212 USE_VERTEX_BUFFER_OBJECTS,
213 USE_VERTEX_ARRAYS
214 };
215
216 /** Set how loaded drawables should be handled w.r.t their display list/vertex buffer object/vertex array settings.*/
217 void setDrawablePolicy(DrawablePolicy policy) { _drawablePolicy = policy; }
218
219 /** Get how loaded drawables should be handled w.r.t their display list/vertex buffer object/vertex array settings.*/
220 DrawablePolicy getDrawablePolicy() const { return _drawablePolicy; }
221
222
223 /** Set whether newly loaded textures should have a PixelBufferObject assigned to them to aid download to the GPU.*/
224 void setApplyPBOToImages(bool assignPBOToImages) { _assignPBOToImages = assignPBOToImages; }
225
226 /** Get whether newly loaded textures should have a PixelBufferObject assigned to them.*/
227 bool getApplyPBOToImages() const { return _assignPBOToImages; }
228
229
230 /** Set whether newly loaded textures should have their UnrefImageDataAfterApply set to a specified value.*/
231 void setUnrefImageDataAfterApplyPolicy(bool changeAutoUnRef, bool valueAutoUnRef) { _changeAutoUnRef = changeAutoUnRef; _valueAutoUnRef = valueAutoUnRef; }
232
233 /** Get whether newly loaded textures should have their UnrefImageDataAfterApply set to a specified value.*/
234 void getUnrefImageDataAfterApplyPolicy(bool& changeAutoUnRef, bool& valueAutoUnRef) const { changeAutoUnRef = _changeAutoUnRef; valueAutoUnRef = _valueAutoUnRef; }
235
236
237 /** Set whether newly loaded textures should have their MaxAnisotopy set to a specified value.*/
238 void setMaxAnisotropyPolicy(bool changeAnisotropy, float valueAnisotropy) { _changeAnisotropy = changeAnisotropy; _valueAnisotropy = valueAnisotropy; }
239
240 /** Set whether newly loaded textures should have their MaxAnisotopy set to a specified value.*/
241 void getMaxAnisotropyPolicy(bool& changeAnisotropy, float& valueAnisotropy) const { changeAnisotropy = _changeAnisotropy; valueAnisotropy = _valueAnisotropy; }
242
243
244 /** Return true if there are pending updates to the scene graph that require a call to updateSceneGraph(double). */
245 bool requiresUpdateSceneGraph() const;
246
247 /** Merge the changes to the scene graph by calling calling removeExpiredSubgraphs then addLoadedDataToSceneGraph.
248 * Note, must only be called from single thread update phase. */
249 virtual void updateSceneGraph(const osg::FrameStamp& frameStamp);
250
251 /** Return true if there are GL objects that need to be compiled by a draw traversal. */
252 bool requiresRedraw() const;
253
254 /** Report how many items are in the _fileRequestList queue */
255 unsigned int getFileRequestListSize() const { return static_cast<unsigned int>(_fileRequestQueue->size() + _httpRequestQueue->size()); }
256
257 /** Report how many items are in the _dataToCompileList queue */
258 unsigned int getDataToCompileListSize() const { return static_cast<unsigned int>(_dataToCompileList->size()); }
259
260 /** Report how many items are in the _dataToMergeList queue */
261 unsigned int getDataToMergeListSize() const { return static_cast<unsigned int>(_dataToMergeList->size()); }
262
263 /** Report whether any requests are in the pager.*/
264 bool getRequestsInProgress() const;
265
266 /** Get the minimum time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
267 double getMinimumTimeToMergeTile() const { return _minimumTimeToMergeTile; }
268
269 /** Get the maximum time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
270 double getMaximumTimeToMergeTile() const { return _maximumTimeToMergeTile; }
271
272 /** Get the average time between the first request for a tile to be loaded and the time of its merge into the main scene graph.*/
273 double getAverageTimeToMergeTiles() const { return (_numTilesMerges > 0) ? _totalTimeToMergeTiles/static_cast<double>(_numTilesMerges) : 0; }
274
275 /** Reset the Stats variables.*/
276 void resetStats();
277
278 typedef std::set< osg::ref_ptr<osg::StateSet> > StateSetList;
279 typedef std::vector< osg::ref_ptr<osg::Drawable> > DrawableList;
280
281 class ExpirePagedLODsVisitor;
282
283 typedef std::list< osg::ref_ptr<osg::Object> > ObjectList;
284
285 struct PagedLODList : public osg::Referenced
286 {
287 virtual PagedLODList* clone() = 0;
288 virtual void clear() = 0;
289 virtual unsigned int size() = 0;
290 virtual void removeExpiredChildren(int numberChildrenToRemove, double expiryTime, unsigned int expiryFrame, ObjectList& childrenRemoved, bool visitActive) = 0;
291 virtual void removeNodes(osg::NodeList& nodesToRemove) = 0;
292 virtual void insertPagedLOD(const osg::observer_ptr<osg::PagedLOD>& plod) = 0;
293 virtual bool containsPagedLOD(const osg::observer_ptr<osg::PagedLOD>& plod) const = 0;
294 };
295
296 void setMarkerObject(osg::Object* mo) { _markerObject = mo; }
297 osg::Object* getMarkerObject() { return _markerObject.get(); }
298 const osg::Object* getMarkerObject() const { return _markerObject.get(); }
299
300 protected:
301
302 virtual ~DatabasePager();
303
304 friend class DatabaseThread;
305 friend struct DatabaseRequest;
306
307 struct RequestQueue;
308
309 struct OSGDB_EXPORT DatabaseRequest : public osg::Referenced
310 {
311 DatabaseRequest():
312 osg::Referenced(true),
313 _valid(false),
314 _frameNumberFirstRequest(0),
315 _timestampFirstRequest(0.0),
316 _priorityFirstRequest(0.f),
317 _frameNumberLastRequest(0),
318 _timestampLastRequest(0.0),
319 _priorityLastRequest(0.0f),
320 _numOfRequests(0),
321 _groupExpired(false)
322 {}
323
324 void invalidate();
325
326 bool valid() const { return _valid; }
327
328 bool isRequestCurrent (int frameNumber) const
329 {
330 return _valid && (frameNumber - _frameNumberLastRequest <= 1);
331 }
332
333 bool _valid;
334 std::string _fileName;
335 unsigned int _frameNumberFirstRequest;
336 double _timestampFirstRequest;
337 float _priorityFirstRequest;
338 unsigned int _frameNumberLastRequest;
339 double _timestampLastRequest;
340 float _priorityLastRequest;
341 unsigned int _numOfRequests;
342
343 osg::observer_ptr<osg::Node> _terrain;
344 osg::observer_ptr<osg::Group> _group;
345
346 osg::ref_ptr<osg::Node> _loadedModel;
347 osg::ref_ptr<Options> _loadOptions;
348 osg::ref_ptr<ObjectCache> _objectCache;
349
350 osg::observer_ptr<osgUtil::IncrementalCompileOperation::CompileSet> _compileSet;
351 bool _groupExpired; // flag used only in update thread
352 };
353
354
355 struct OSGDB_EXPORT RequestQueue : public osg::Referenced
356 {
357 public:
358
359 RequestQueue(DatabasePager* pager);
360
361 void add(DatabaseRequest* databaseRequest);
362 void remove(DatabaseRequest* databaseRequest);
363
364 void addNoLock(DatabaseRequest* databaseRequest);
365
366 void takeFirst(osg::ref_ptr<DatabaseRequest>& databaseRequest);
367
368 /// prune all the old requests and then return true if requestList left empty
369 bool pruneOldRequestsAndCheckIfEmpty();
370
371 virtual void updateBlock() {}
372
373 void invalidate(DatabaseRequest* dr);
374
375 bool empty();
376
377 unsigned int size();
378
379 void clear();
380
381
382 typedef std::list< osg::ref_ptr<DatabaseRequest> > RequestList;
383 void swap(RequestList& requestList);
384
385 DatabasePager* _pager;
386 RequestList _requestList;
387 OpenThreads::Mutex _requestMutex;
388 unsigned int _frameNumberLastPruned;
389
390 protected:
391 virtual ~RequestQueue();
392 };
393
394
395 typedef std::vector< osg::ref_ptr<DatabaseThread> > DatabaseThreadList;
396
397 struct OSGDB_EXPORT ReadQueue : public RequestQueue
398 {
399 ReadQueue(DatabasePager* pager, const std::string& name);
400
401 void block() { _block->block(); }
402
403 void release() { _block->release(); }
404
405 virtual void updateBlock();
406
407
408 osg::ref_ptr<osg::RefBlock> _block;
409
410 std::string _name;
411
412 OpenThreads::Mutex _childrenToDeleteListMutex;
413 ObjectList _childrenToDeleteList;
414 };
415
416 // forward declare inner helper classes
417 class FindCompileableGLObjectsVisitor;
418 friend class FindCompileableGLObjectsVisitor;
419
420 struct DatabasePagerCompileCompletedCallback;
421 friend struct DatabasePagerCompileCompletedCallback;
422
423 class FindPagedLODsVisitor;
424 friend class FindPagedLODsVisitor;
425
426 struct SortFileRequestFunctor;
427 friend struct SortFileRequestFunctor;
428
429
430 OpenThreads::Mutex _run_mutex;
431 OpenThreads::Mutex _dr_mutex;
432 bool _startThreadCalled;
433
434 void compileCompleted(DatabaseRequest* databaseRequest);
435
436 /** Iterate through the active PagedLOD nodes children removing
437 * children which haven't been visited since specified expiryTime.
438 * note, should be only be called from the update thread. */
439 virtual void removeExpiredSubgraphs(const osg::FrameStamp &frameStamp);
440
441 /** Add the loaded data to the scene graph.*/
442 void addLoadedDataToSceneGraph(const osg::FrameStamp &frameStamp);
443
444
445 OpenThreads::Affinity _affinity;
446
447 bool _done;
448 bool _acceptNewRequests;
449 bool _databasePagerThreadPaused;
450
451 DatabaseThreadList _databaseThreads;
452
453 int _numFramesActive;
454 mutable OpenThreads::Mutex _numFramesActiveMutex;
455 OpenThreads::Atomic _frameNumber;
456
457 osg::ref_ptr<ReadQueue> _fileRequestQueue;
458 osg::ref_ptr<ReadQueue> _httpRequestQueue;
459 osg::ref_ptr<RequestQueue> _dataToCompileList;
460 osg::ref_ptr<RequestQueue> _dataToMergeList;
461
462 DrawablePolicy _drawablePolicy;
463
464 bool _assignPBOToImages;
465 bool _changeAutoUnRef;
466 bool _valueAutoUnRef;
467 bool _changeAnisotropy;
468 float _valueAnisotropy;
469
470 bool _deleteRemovedSubgraphsInDatabaseThread;
471
472
473 osg::ref_ptr<PagedLODList> _activePagedLODList;
474
475 unsigned int _targetMaximumNumberOfPageLOD;
476
477 bool _doPreCompile;
478 osg::ref_ptr<osgUtil::IncrementalCompileOperation> _incrementalCompileOperation;
479
480
481 double _minimumTimeToMergeTile;
482 double _maximumTimeToMergeTile;
483 double _totalTimeToMergeTiles;
484 unsigned int _numTilesMerges;
485
486 osg::ref_ptr<osg::Object> _markerObject;
487};
488
489}
490
491#endif