PtexCache.cpp
Go to the documentation of this file.
/*
PTEX SOFTWARE
Copyright 2014 Disney Enterprises, Inc. All rights reserved
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
* The names "Disney", "Walt Disney Pictures", "Walt Disney Animation
Studios" or the names of its contributors may NOT be used to
endorse or promote products derived from this software without
specific prior written permission from Walt Disney Pictures.
Disclaimer: THIS SOFTWARE IS PROVIDED BY WALT DISNEY PICTURES AND
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE, NONINFRINGEMENT AND TITLE ARE DISCLAIMED.
IN NO EVENT SHALL WALT DISNEY PICTURES, THE COPYRIGHT HOLDER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND BASED ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
*/
#include "PtexPlatform.h"
#include <algorithm>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <iostream>
#include <ctype.h>
#include "Ptexture.h"
#include "PtexReader.h"
#include "PtexCache.h"
PTEX_NAMESPACE_BEGIN
void PtexCachedReader::release()
{
if (0 == unref()) {
_cache->logRecentlyUsed(this);
}
}
bool PtexReaderCache::findFile(const char*& filename, std::string& buffer, Ptex::String& error)
{
bool isAbsolute = (filename[0] == '/'
#ifdef PTEX_PLATFORM_WINDOWS
|| filename[0] == '\\'
|| (isalpha(filename[0]) && filename[1] == ':')
#endif
);
if (isAbsolute || _searchdirs.empty()) return true; // no need to search
// file is relative, search in searchpath
buffer.reserve(256); // minimize reallocs (will grow automatically)
struct stat statbuf;
for (size_t i = 0, size = _searchdirs.size(); i < size; i++) {
buffer = _searchdirs[i];
buffer += "/";
buffer += filename;
if (stat(buffer.c_str(), &statbuf) == 0) {
filename = buffer.c_str();
return true;
}
}
// not found
std::string errstr = "Can't find ptex file: ";
errstr += filename;
error = errstr.c_str();
return false;
}
PtexTexture* PtexReaderCache::get(const char* filename, Ptex::String& error)
{
// lookup reader in map
StringKey key(filename);
PtexCachedReader* reader = _files.get(key);
bool isNew = false;
if (reader) {
if (!reader->ok()) return 0;
if (reader->pendingPurge()) {
// a previous purge attempt was made and file was busy. Try again now.
purge(reader);
}
reader->ref();
} else {
reader = new PtexCachedReader(_premultiply, _io, _err, this);
isNew = true;
}
bool needOpen = reader->needToOpen();
if (needOpen) {
std::string buffer;
const char* pathToOpen = filename;
// search for the file (unless we have an I/O handler)
if (_io || findFile(pathToOpen, buffer, error)) {
reader->open(pathToOpen, error);
} else {
// flag reader as invalid so we don't try to open it again on next lookup
reader->invalidate();
}
}
if (isNew) {
size_t newMemUsed = 0;
PtexCachedReader* newreader = reader;
reader = _files.tryInsert(key, reader, newMemUsed);
adjustMemUsed(newMemUsed);
if (reader != newreader) {
// another thread got here first
reader->ref();
delete newreader;
}
}
if (!reader->ok()) {
reader->unref();
return 0;
}
if (needOpen) {
reader->logOpen();
}
return reader;
}
PtexCache* PtexCache::create(int maxFiles, size_t maxMem, bool premultiply,
PtexInputHandler* inputHandler,
PtexErrorHandler* errorHandler)
{
// set default files to 100
if (maxFiles <= 0) maxFiles = 100;
return new PtexReaderCache(maxFiles, maxMem, premultiply, inputHandler, errorHandler);
}
void PtexReaderCache::logRecentlyUsed(PtexCachedReader* reader)
{
while (1) {
MruList* mruList = _mruList;
int slot = AtomicIncrement(&mruList->next)-1;
if (slot < numMruFiles) {
mruList->files[slot] = reader;
return;
}
// no mru slot available, process mru list and try again
do processMru();
while (_mruList->next >= numMruFiles);
}
}
void PtexReaderCache::processMru()
{
// use a non-blocking lock so we can proceed as soon as space has been freed in the mru list
// (which happens almost immediately in the processMru thread that has the lock)
if (!_mruLock.trylock()) return;
if (_mruList->next < numMruFiles) {
_mruLock.unlock();
return;
}
// switch mru buffers and reset slot counter so other threads can proceed immediately
MruList* mruList = _mruList;
AtomicStore(&_mruList, _prevMruList);
_prevMruList = mruList;
// extract relevant stats and add to open/active list
size_t memUsedChange = 0, filesOpenChange = 0;
for (int i = 0; i < numMruFiles; ++i) {
PtexCachedReader* reader;
do { reader = mruList->files[i]; } while (!reader); // loop on (unlikely) race condition
mruList->files[i] = 0;
memUsedChange += reader->getMemUsedChange();
size_t opens = reader->getOpensChange();
size_t blockReads = reader->getBlockReadsChange();
filesOpenChange += opens;
if (opens || blockReads) {
_fileOpens += opens;
_blockReads += blockReads;
_openFiles.push(reader);
}
if (_maxMem) {
_activeFiles.push(reader);
}
}
AtomicStore(&mruList->next, 0);
adjustMemUsed(memUsedChange);
adjustFilesOpen(filesOpenChange);
bool shouldPruneFiles = _filesOpen > _maxFiles;
bool shouldPruneData = _maxMem && _memUsed > _maxMem;
if (shouldPruneFiles) {
pruneFiles();
}
if (shouldPruneData) {
pruneData();
}
_mruLock.unlock();
}
void PtexReaderCache::pruneFiles()
{
size_t numToClose = _filesOpen - _maxFiles;
if (numToClose > 0) {
while (numToClose) {
PtexCachedReader* reader = _openFiles.pop();
if (!reader) { _filesOpen = 0; break; }
if (reader->tryClose()) {
--numToClose;
--_filesOpen;
}
}
}
}
void PtexReaderCache::pruneData()
{
size_t memUsedChangeTotal = 0;
size_t memUsed = _memUsed;
while (memUsed + memUsedChangeTotal > _maxMem) {
PtexCachedReader* reader = _activeFiles.pop();
if (!reader) break;
size_t memUsedChange;
if (reader->tryPrune(memUsedChange)) {
// Note: after clearing, memUsedChange is negative
memUsedChangeTotal += memUsedChange;
}
}
adjustMemUsed(memUsedChangeTotal);
}
void PtexReaderCache::purge(PtexTexture* texture)
{
PtexCachedReader* reader = static_cast<PtexCachedReader*>(texture);
reader->unref();
purge(reader);
reader->ref();
}
void PtexReaderCache::purge(const char* filename)
{
StringKey key(filename);
PtexCachedReader* reader = _files.get(key);
if (reader) purge(reader);
}
void PtexReaderCache::purge(PtexCachedReader* reader)
{
size_t memUsedChange;
if (reader->tryPurge(memUsedChange)) {
adjustMemUsed(memUsedChange);
}
}
void PtexReaderCache::Purger::operator()(PtexCachedReader* reader)
{
size_t memUsedChange;
if (reader->tryPurge(memUsedChange)) {
memUsedChangeTotal += memUsedChange;
}
}
void PtexReaderCache::purgeAll()
{
Purger purger;
_files.foreach(purger);
adjustMemUsed(purger.memUsedChangeTotal);
}
void PtexReaderCache::getStats(Stats& stats)
{
stats.memUsed = _memUsed;
stats.peakMemUsed = _peakMemUsed;
stats.filesOpen = _filesOpen;
stats.peakFilesOpen = _peakFilesOpen;
stats.filesAccessed = _files.size();
stats.fileReopens = _fileOpens < stats.filesAccessed ? 0 : _fileOpens - stats.filesAccessed;
stats.blockReads = _blockReads;
}
PTEX_NAMESPACE_END
PtexCache.h
PtexPlatform.h
Platform-specific classes, functions, and includes.
AtomicStore
PTEX_INLINE void AtomicStore(T volatile *target, T value)
Definition PtexPlatform.h:284
AtomicIncrement
PTEX_INLINE T AtomicIncrement(volatile T *target)
Definition PtexPlatform.h:225
PtexReader.h
PTEX_NAMESPACE_END
#define PTEX_NAMESPACE_END
Definition PtexVersion.h:62
Ptexture.h
Public API classes for reading, writing, caching, and filtering Ptex files.
Mutex::trylock
bool trylock()
Definition PtexPlatform.h:142
Mutex::unlock
void unlock()
Definition PtexPlatform.h:143
PtexCache
File-handle and memory cache for reading ptex files.
Definition Ptexture.h:684
PtexCache::create
static PtexCache * create(int maxFiles, size_t maxMem, bool premultiply=false, PtexInputHandler *inputHandler=0, PtexErrorHandler *errorHandler=0)
Create a cache with the specified limits.
Definition PtexCache.cpp:177
PtexCachedReader
Definition PtexCache.h:109
PtexCachedReader::_cache
PtexReaderCache * _cache
Definition PtexCache.h:110
PtexCachedReader::unref
int32_t unref()
Definition PtexCache.h:146
PtexCachedReader::tryPurge
bool tryPurge(size_t &memUsedChange)
Definition PtexCache.h:162
PtexCachedReader::getMemUsedChange
size_t getMemUsedChange()
Definition PtexCache.h:173
PtexCachedReader::tryPrune
bool tryPrune(size_t &memUsedChange)
Definition PtexCache.h:152
PtexCachedReader::release
virtual void release()
Release resources held by this pointer (pointer becomes invalid).
Definition PtexCache.cpp:83
PtexCachedReader::ref
void ref()
Definition PtexCache.h:138
PtexCachedReader::getBlockReadsChange
size_t getBlockReadsChange()
Definition PtexCache.h:187
PtexCachedReader::getOpensChange
size_t getOpensChange()
Definition PtexCache.h:180
PtexErrorHandler
Custom handler interface redirecting Ptex error messages.
Definition Ptexture.h:658
PtexHashMap::foreach
void foreach(Fn &fn)
Definition PtexHashMap.h:252
PtexHashMap::size
uint32_t size() const
Definition PtexHashMap.h:201
PtexHashMap::get
Value get(Key &key)
Definition PtexHashMap.h:203
PtexHashMap::tryInsert
Value tryInsert(Key &key, Value value, size_t &newMemUsed)
Definition PtexHashMap.h:224
PtexInputHandler
Custom handler interface for intercepting and redirecting Ptex input stream calls.
Definition Ptexture.h:619
PtexLruList::pop
T * pop()
Definition PtexCache.h:95
PtexLruList::push
void push(T *node)
Definition PtexCache.h:88
PtexReaderCache
Cache for reading Ptex texture files.
Definition PtexCache.h:198
PtexReaderCache::_activeFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_activeFilesItem > _activeFiles
Definition PtexCache.h:301
PtexReaderCache::_io
PtexInputHandler * _io
Definition PtexCache.h:280
PtexReaderCache::_maxMem
size_t _maxMem
Definition PtexCache.h:279
PtexReaderCache::adjustFilesOpen
void adjustFilesOpen(size_t amount)
Definition PtexCache.h:259
PtexReaderCache::_fileOpens
size_t _fileOpens
Definition PtexCache.h:305
PtexReaderCache::_memUsed
volatile size_t _memUsed
Definition PtexCache.h:287
PtexReaderCache::_mruLock
Mutex _mruLock
Definition PtexCache.h:289
PtexReaderCache::logRecentlyUsed
void logRecentlyUsed(PtexCachedReader *reader)
Definition PtexCache.cpp:188
PtexReaderCache::pruneFiles
void pruneFiles()
Definition PtexCache.cpp:254
PtexReaderCache::_prevMruList
MruList *volatile _prevMruList
Definition PtexCache.h:298
PtexReaderCache::_files
FileMap _files
Definition PtexCache.h:285
PtexReaderCache::adjustMemUsed
void adjustMemUsed(size_t amount)
Definition PtexCache.h:253
PtexReaderCache::_premultiply
bool _premultiply
Definition PtexCache.h:286
PtexReaderCache::_peakMemUsed
size_t _peakMemUsed
Definition PtexCache.h:303
PtexReaderCache::get
virtual PtexTexture * get(const char *path, Ptex::String &error)
Access a texture.
Definition PtexCache.cpp:121
PtexReaderCache::findFile
bool findFile(const char *&filename, std::string &buffer, Ptex::String &error)
Definition PtexCache.cpp:91
PtexReaderCache::getStats
virtual void getStats(Stats &stats)
Get stats.
Definition PtexCache.cpp:326
PtexReaderCache::pruneData
void pruneData()
Definition PtexCache.cpp:270
PtexReaderCache::numMruFiles
static const int numMruFiles
Definition PtexCache.h:291
PtexReaderCache::_maxFiles
size_t _maxFiles
Definition PtexCache.h:278
PtexReaderCache::_peakFilesOpen
size_t _peakFilesOpen
Definition PtexCache.h:304
PtexReaderCache::_err
PtexErrorHandler * _err
Definition PtexCache.h:281
PtexReaderCache::_openFiles
PtexLruList< PtexCachedReader, &PtexCachedReader::_openFilesItem > _openFiles
Definition PtexCache.h:300
PtexReaderCache::processMru
void processMru()
Definition PtexCache.cpp:203
PtexReaderCache::purgeAll
virtual void purgeAll()
Remove all texture files from the cache.
Definition PtexCache.cpp:319
PtexReaderCache::_blockReads
size_t _blockReads
Definition PtexCache.h:306
PtexReaderCache::_filesOpen
volatile size_t _filesOpen
Definition PtexCache.h:288
PtexReaderCache::_mruList
MruList *volatile _mruList
Definition PtexCache.h:297
PtexReaderCache::purge
virtual void purge(PtexTexture *)
Remove a texture file from the cache.
Definition PtexCache.cpp:287
PtexReaderCache::_searchdirs
std::vector< std::string > _searchdirs
Definition PtexCache.h:283
PtexReader::ok
bool ok() const
Definition PtexReader.h:64
PtexReader::needToOpen
bool needToOpen() const
Definition PtexReader.h:57
PtexReader::pendingPurge
bool pendingPurge() const
Definition PtexReader.h:62
PtexReader::invalidate
void invalidate()
Definition PtexReader.h:66
PtexReader::logOpen
void logOpen()
Definition PtexReader.h:72
PtexReader::open
bool open(const char *path, Ptex::String &error)
Definition PtexReader.cpp:137
PtexReader::tryClose
bool tryClose()
Definition PtexReader.cpp:219
PtexTexture
Interface for reading data from a ptex file.
Definition Ptexture.h:457
Ptex::String
Memory-managed string.
Definition Ptexture.h:296
Ptex::String::c_str
const char * c_str() const
Definition Ptexture.h:304
StringKey
Definition PtexHashMap.h:83
PTEX_NAMESPACE_BEGIN
Definition PtexSeparableKernel.cpp:42
PtexCache::Stats
Definition Ptexture.h:779
PtexCache::Stats::filesAccessed
uint64_t filesAccessed
Definition Ptexture.h:784
PtexCache::Stats::filesOpen
uint64_t filesOpen
Definition Ptexture.h:782
PtexCache::Stats::peakFilesOpen
uint64_t peakFilesOpen
Definition Ptexture.h:783
PtexCache::Stats::fileReopens
uint64_t fileReopens
Definition Ptexture.h:785
PtexCache::Stats::memUsed
uint64_t memUsed
Definition Ptexture.h:780
PtexCache::Stats::blockReads
uint64_t blockReads
Definition Ptexture.h:786
PtexCache::Stats::peakMemUsed
uint64_t peakMemUsed
Definition Ptexture.h:781
PtexReaderCache::MruList
Definition PtexCache.h:292
PtexReaderCache::MruList::files
PtexCachedReader *volatile files[numMruFiles]
Definition PtexCache.h:294
PtexReaderCache::MruList::next
volatile int next
Definition PtexCache.h:293
PtexReaderCache::Purger
Definition PtexCache.h:268
PtexReaderCache::Purger::operator()
void operator()(PtexCachedReader *reader)
Definition PtexCache.cpp:311
PtexReaderCache::Purger::memUsedChangeTotal
size_t memUsedChangeTotal
Definition PtexCache.h:269