Ptex

PtexCache.cpp
Go to the documentation of this file.
1/*
2 PTEX SOFTWARE
3 Copyright 2014 Disney Enterprises, Inc. All rights reserved
4
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are
7 met:
8
9 * Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
11
12 * Redistributions in binary form must reproduce the above copyright
13 notice, this list of conditions and the following disclaimer in
14 the documentation and/or other materials provided with the
15 distribution.
16
17 * The names "Disney", "Walt Disney Pictures", "Walt Disney Animation
18 Studios" or the names of its contributors may NOT be used to
19 endorse or promote products derived from this software without
20 specific prior written permission from Walt Disney Pictures.
21
22 Disclaimer: THIS SOFTWARE IS PROVIDED BY WALT DISNEY PICTURES AND
23 CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
24 BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
25 FOR A PARTICULAR PURPOSE, NONINFRINGEMENT AND TITLE ARE DISCLAIMED.
26 IN NO EVENT SHALL WALT DISNEY PICTURES, THE COPYRIGHT HOLDER OR
27 CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND BASED ON ANY
31 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
34*/
35
69#include "PtexPlatform.h"
70#include <algorithm>
71#include <sys/types.h>
72#include <sys/stat.h>
73#include <stdlib.h>
74#include <iostream>
75#include <ctype.h>
76#include "Ptexture.h"
77#include "PtexReader.h"
78#include "PtexCache.h"
79
80
81 PTEX_NAMESPACE_BEGIN
82
83 void PtexCachedReader::release()
84{
85 if (0 == unref()) {
86 _cache->logRecentlyUsed(this);
87 }
88}
89
90
91 bool PtexReaderCache::findFile(const char*& filename, std::string& buffer, Ptex::String& error)
92{
93 bool isAbsolute = (filename[0] == '/'
94#ifdef PTEX_PLATFORM_WINDOWS
95 || filename[0] == '\\'
96 || (isalpha(filename[0]) && filename[1] == ':')
97#endif
98 );
99 if (isAbsolute || _searchdirs.empty()) return true; // no need to search
100
101 // file is relative, search in searchpath
102 buffer.reserve(256); // minimize reallocs (will grow automatically)
103 struct stat statbuf;
104 for (size_t i = 0, size = _searchdirs.size(); i < size; i++) {
105 buffer = _searchdirs[i];
106 buffer += "/";
107 buffer += filename;
108 if (stat(buffer.c_str(), &statbuf) == 0) {
109 filename = buffer.c_str();
110 return true;
111 }
112 }
113 // not found
114 std::string errstr = "Can't find ptex file: ";
115 errstr += filename;
116 error = errstr.c_str();
117 return false;
118}
119
120
121 PtexTexture* PtexReaderCache::get(const char* filename, Ptex::String& error)
122{
123 // lookup reader in map
124 StringKey key(filename);
125 PtexCachedReader* reader = _files.get(key);
126 bool isNew = false;
127
128 if (reader) {
129 if (!reader->ok()) return 0;
130 if (reader->pendingPurge()) {
131 // a previous purge attempt was made and file was busy. Try again now.
132 purge(reader);
133 }
134 reader->ref();
135 } else {
136 reader = new PtexCachedReader(_premultiply, _io, _err, this);
137 isNew = true;
138 }
139
140 bool needOpen = reader->needToOpen();
141 if (needOpen) {
142 std::string buffer;
143 const char* pathToOpen = filename;
144 // search for the file (unless we have an I/O handler)
145 if (_io || findFile(pathToOpen, buffer, error)) {
146 reader->open(pathToOpen, error);
147 } else {
148 // flag reader as invalid so we don't try to open it again on next lookup
149 reader->invalidate();
150 }
151 }
152
153 if (isNew) {
154 size_t newMemUsed = 0;
155 PtexCachedReader* newreader = reader;
156 reader = _files.tryInsert(key, reader, newMemUsed);
157 adjustMemUsed(newMemUsed);
158 if (reader != newreader) {
159 // another thread got here first
160 reader->ref();
161 delete newreader;
162 }
163 }
164
165 if (!reader->ok()) {
166 reader->unref();
167 return 0;
168 }
169
170 if (needOpen) {
171 reader->logOpen();
172 }
173
174 return reader;
175}
176
177 PtexCache* PtexCache::create(int maxFiles, size_t maxMem, bool premultiply,
178 PtexInputHandler* inputHandler,
179 PtexErrorHandler* errorHandler)
180{
181 // set default files to 100
182 if (maxFiles <= 0) maxFiles = 100;
183
184 return new PtexReaderCache(maxFiles, maxMem, premultiply, inputHandler, errorHandler);
185}
186
187
188 void PtexReaderCache::logRecentlyUsed(PtexCachedReader* reader)
189{
190 while (1) {
191 MruList* mruList = _mruList;
192 int slot = AtomicIncrement(&mruList->next)-1;
193 if (slot < numMruFiles) {
194 mruList->files[slot] = reader;
195 return;
196 }
197 // no mru slot available, process mru list and try again
198 do processMru();
199 while (_mruList->next >= numMruFiles);
200 }
201}
202
203 void PtexReaderCache::processMru()
204{
205 // use a non-blocking lock so we can proceed as soon as space has been freed in the mru list
206 // (which happens almost immediately in the processMru thread that has the lock)
207 if (!_mruLock.trylock()) return;
208 if (_mruList->next < numMruFiles) {
209 _mruLock.unlock();
210 return;
211 }
212
213 // switch mru buffers and reset slot counter so other threads can proceed immediately
214 MruList* mruList = _mruList;
215 AtomicStore(&_mruList, _prevMruList);
216 _prevMruList = mruList;
217
218 // extract relevant stats and add to open/active list
219 size_t memUsedChange = 0, filesOpenChange = 0;
220 for (int i = 0; i < numMruFiles; ++i) {
221 PtexCachedReader* reader;
222 do { reader = mruList->files[i]; } while (!reader); // loop on (unlikely) race condition
223 mruList->files[i] = 0;
224 memUsedChange += reader->getMemUsedChange();
225 size_t opens = reader->getOpensChange();
226 size_t blockReads = reader->getBlockReadsChange();
227 filesOpenChange += opens;
228 if (opens || blockReads) {
229 _fileOpens += opens;
230 _blockReads += blockReads;
231 _openFiles.push(reader);
232 }
233 if (_maxMem) {
234 _activeFiles.push(reader);
235 }
236 }
237 AtomicStore(&mruList->next, 0);
238 adjustMemUsed(memUsedChange);
239 adjustFilesOpen(filesOpenChange);
240
241 bool shouldPruneFiles = _filesOpen > _maxFiles;
242 bool shouldPruneData = _maxMem && _memUsed > _maxMem;
243
244 if (shouldPruneFiles) {
245 pruneFiles();
246 }
247 if (shouldPruneData) {
248 pruneData();
249 }
250 _mruLock.unlock();
251}
252
253
254 void PtexReaderCache::pruneFiles()
255{
256 size_t numToClose = _filesOpen - _maxFiles;
257 if (numToClose > 0) {
258 while (numToClose) {
259 PtexCachedReader* reader = _openFiles.pop();
260 if (!reader) { _filesOpen = 0; break; }
261 if (reader->tryClose()) {
262 --numToClose;
263 --_filesOpen;
264 }
265 }
266 }
267}
268
269
270 void PtexReaderCache::pruneData()
271{
272 size_t memUsedChangeTotal = 0;
273 size_t memUsed = _memUsed;
274 while (memUsed + memUsedChangeTotal > _maxMem) {
275 PtexCachedReader* reader = _activeFiles.pop();
276 if (!reader) break;
277 size_t memUsedChange;
278 if (reader->tryPrune(memUsedChange)) {
279 // Note: after clearing, memUsedChange is negative
280 memUsedChangeTotal += memUsedChange;
281 }
282 }
283 adjustMemUsed(memUsedChangeTotal);
284}
285
286
287 void PtexReaderCache::purge(PtexTexture* texture)
288{
289 PtexCachedReader* reader = static_cast<PtexCachedReader*>(texture);
290 reader->unref();
291 purge(reader);
292 reader->ref();
293}
294
295
296 void PtexReaderCache::purge(const char* filename)
297{
298 StringKey key(filename);
299 PtexCachedReader* reader = _files.get(key);
300 if (reader) purge(reader);
301}
302
303 void PtexReaderCache::purge(PtexCachedReader* reader)
304{
305 size_t memUsedChange;
306 if (reader->tryPurge(memUsedChange)) {
307 adjustMemUsed(memUsedChange);
308 }
309}
310
311 void PtexReaderCache::Purger::operator()(PtexCachedReader* reader)
312{
313 size_t memUsedChange;
314 if (reader->tryPurge(memUsedChange)) {
315 memUsedChangeTotal += memUsedChange;
316 }
317}
318
319 void PtexReaderCache::purgeAll()
320{
321 Purger purger;
322 _files.foreach(purger);
323 adjustMemUsed(purger.memUsedChangeTotal);
324}
325
326 void PtexReaderCache::getStats(Stats& stats)
327{
328 stats.memUsed = _memUsed;
329 stats.peakMemUsed = _peakMemUsed;
330 stats.filesOpen = _filesOpen;
331 stats.peakFilesOpen = _peakFilesOpen;
332 stats.filesAccessed = _files.size();
333 stats.fileReopens = _fileOpens < stats.filesAccessed ? 0 : _fileOpens - stats.filesAccessed;
334 stats.blockReads = _blockReads;
335}
336
337 PTEX_NAMESPACE_END
PtexCache.h
PtexPlatform.h
Platform-specific classes, functions, and includes.
AtomicStore
PTEX_INLINE void AtomicStore(T volatile *target, T value)
Definition PtexPlatform.h:284
AtomicIncrement
PTEX_INLINE T AtomicIncrement(volatile T *target)
Definition PtexPlatform.h:225
PtexReader.h
PTEX_NAMESPACE_END
#define PTEX_NAMESPACE_END
Definition PtexVersion.h:62
Ptexture.h
Public API classes for reading, writing, caching, and filtering Ptex files.
Mutex::trylock
bool trylock()
Definition PtexPlatform.h:142
Mutex::unlock
void unlock()
Definition PtexPlatform.h:143
PtexCache
File-handle and memory cache for reading ptex files.
Definition Ptexture.h:684
PtexCache::create
static PtexCache * create(int maxFiles, size_t maxMem, bool premultiply=false, PtexInputHandler *inputHandler=0, PtexErrorHandler *errorHandler=0)
Create a cache with the specified limits.
Definition PtexCache.cpp:177
PtexCachedReader
Definition PtexCache.h:109
PtexCachedReader::_cache
PtexReaderCache * _cache
Definition PtexCache.h:110
PtexCachedReader::unref
int32_t unref()
Definition PtexCache.h:146
PtexCachedReader::tryPurge
bool tryPurge(size_t &memUsedChange)
Definition PtexCache.h:162
PtexCachedReader::getMemUsedChange
size_t getMemUsedChange()
Definition PtexCache.h:173
PtexCachedReader::tryPrune
bool tryPrune(size_t &memUsedChange)
Definition PtexCache.h:152
PtexCachedReader::release
virtual void release()
Release resources held by this pointer (pointer becomes invalid).
Definition PtexCache.cpp:83
PtexCachedReader::ref
void ref()
Definition PtexCache.h:138
PtexCachedReader::getBlockReadsChange
size_t getBlockReadsChange()
Definition PtexCache.h:187
PtexCachedReader::getOpensChange
size_t getOpensChange()
Definition PtexCache.h:180
PtexErrorHandler
Custom handler interface redirecting Ptex error messages.
Definition Ptexture.h:658
PtexHashMap::foreach
void foreach(Fn &fn)
Definition PtexHashMap.h:252
PtexHashMap::size
uint32_t size() const
Definition PtexHashMap.h:201
PtexHashMap::get
Value get(Key &key)
Definition PtexHashMap.h:203
PtexHashMap::tryInsert
Value tryInsert(Key &key, Value value, size_t &newMemUsed)
Definition PtexHashMap.h:224
PtexInputHandler
Custom handler interface for intercepting and redirecting Ptex input stream calls.
Definition Ptexture.h:619
PtexLruList::pop
T * pop()
Definition PtexCache.h:95
PtexLruList::push
void push(T *node)
Definition PtexCache.h:88
PtexReaderCache
Cache for reading Ptex texture files.
Definition PtexCache.h:198
PtexReaderCache::_activeFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_activeFilesItem > _activeFiles
Definition PtexCache.h:301
PtexReaderCache::_io
PtexInputHandler * _io
Definition PtexCache.h:280
PtexReaderCache::_maxMem
size_t _maxMem
Definition PtexCache.h:279
PtexReaderCache::adjustFilesOpen
void adjustFilesOpen(size_t amount)
Definition PtexCache.h:259
PtexReaderCache::_fileOpens
size_t _fileOpens
Definition PtexCache.h:305
PtexReaderCache::_memUsed
volatile size_t _memUsed
Definition PtexCache.h:287
PtexReaderCache::_mruLock
Mutex _mruLock
Definition PtexCache.h:289
PtexReaderCache::logRecentlyUsed
void logRecentlyUsed(PtexCachedReader *reader)
Definition PtexCache.cpp:188
PtexReaderCache::pruneFiles
void pruneFiles()
Definition PtexCache.cpp:254
PtexReaderCache::_prevMruList
MruList *volatile _prevMruList
Definition PtexCache.h:298
PtexReaderCache::_files
FileMap _files
Definition PtexCache.h:285
PtexReaderCache::adjustMemUsed
void adjustMemUsed(size_t amount)
Definition PtexCache.h:253
PtexReaderCache::_premultiply
bool _premultiply
Definition PtexCache.h:286
PtexReaderCache::_peakMemUsed
size_t _peakMemUsed
Definition PtexCache.h:303
PtexReaderCache::get
virtual PtexTexture * get(const char *path, Ptex::String &error)
Access a texture.
Definition PtexCache.cpp:121
PtexReaderCache::findFile
bool findFile(const char *&filename, std::string &buffer, Ptex::String &error)
Definition PtexCache.cpp:91
PtexReaderCache::getStats
virtual void getStats(Stats &stats)
Get stats.
Definition PtexCache.cpp:326
PtexReaderCache::pruneData
void pruneData()
Definition PtexCache.cpp:270
PtexReaderCache::numMruFiles
static const int numMruFiles
Definition PtexCache.h:291
PtexReaderCache::_maxFiles
size_t _maxFiles
Definition PtexCache.h:278
PtexReaderCache::_peakFilesOpen
size_t _peakFilesOpen
Definition PtexCache.h:304
PtexReaderCache::_err
PtexErrorHandler * _err
Definition PtexCache.h:281
PtexReaderCache::_openFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_openFilesItem > _openFiles
Definition PtexCache.h:300
PtexReaderCache::processMru
void processMru()
Definition PtexCache.cpp:203
PtexReaderCache::purgeAll
virtual void purgeAll()
Remove all texture files from the cache.
Definition PtexCache.cpp:319
PtexReaderCache::_blockReads
size_t _blockReads
Definition PtexCache.h:306
PtexReaderCache::_filesOpen
volatile size_t _filesOpen
Definition PtexCache.h:288
PtexReaderCache::_mruList
MruList *volatile _mruList
Definition PtexCache.h:297
PtexReaderCache::purge
virtual void purge(PtexTexture *)
Remove a texture file from the cache.
Definition PtexCache.cpp:287
PtexReaderCache::_searchdirs
std::vector< std::string > _searchdirs
Definition PtexCache.h:283
PtexReader::ok
bool ok() const
Definition PtexReader.h:64
PtexReader::needToOpen
bool needToOpen() const
Definition PtexReader.h:57
PtexReader::pendingPurge
bool pendingPurge() const
Definition PtexReader.h:62
PtexReader::invalidate
void invalidate()
Definition PtexReader.h:66
PtexReader::logOpen
void logOpen()
Definition PtexReader.h:72
PtexReader::open
bool open(const char *path, Ptex::String &error)
Definition PtexReader.cpp:137
PtexReader::tryClose
bool tryClose()
Definition PtexReader.cpp:219
PtexTexture
Interface for reading data from a ptex file.
Definition Ptexture.h:457
Ptex::String
Memory-managed string.
Definition Ptexture.h:296
Ptex::String::c_str
const char * c_str() const
Definition Ptexture.h:304
StringKey
Definition PtexHashMap.h:83
PTEX_NAMESPACE_BEGIN
Definition PtexSeparableKernel.cpp:42
PtexCache::Stats
Definition Ptexture.h:779
PtexCache::Stats::filesAccessed
uint64_t filesAccessed
Definition Ptexture.h:784
PtexCache::Stats::filesOpen
uint64_t filesOpen
Definition Ptexture.h:782
PtexCache::Stats::peakFilesOpen
uint64_t peakFilesOpen
Definition Ptexture.h:783
PtexCache::Stats::fileReopens
uint64_t fileReopens
Definition Ptexture.h:785
PtexCache::Stats::memUsed
uint64_t memUsed
Definition Ptexture.h:780
PtexCache::Stats::blockReads
uint64_t blockReads
Definition Ptexture.h:786
PtexCache::Stats::peakMemUsed
uint64_t peakMemUsed
Definition Ptexture.h:781
PtexReaderCache::MruList
Definition PtexCache.h:292
PtexReaderCache::MruList::files
PtexCachedReader *volatile files[numMruFiles]
Definition PtexCache.h:294
PtexReaderCache::MruList::next
volatile int next
Definition PtexCache.h:293
PtexReaderCache::Purger
Definition PtexCache.h:268
PtexReaderCache::Purger::operator()
void operator()(PtexCachedReader *reader)
Definition PtexCache.cpp:311
PtexReaderCache::Purger::memUsedChangeTotal
size_t memUsedChangeTotal
Definition PtexCache.h:269