Field3D
SparseFile.h
Go to the documentation of this file.
00001 //----------------------------------------------------------------------------//
00002 
00003 /*
00004  * Copyright (c) 2009 Sony Pictures Imageworks Inc
00005  *
00006  * All rights reserved.
00007  *
00008  * Redistribution and use in source and binary forms, with or without
00009  * modification, are permitted provided that the following conditions
00010  * are met:
00011  *
00012  * Redistributions of source code must retain the above copyright
00013  * notice, this list of conditions and the following disclaimer.
00014  * Redistributions in binary form must reproduce the above copyright
00015  * notice, this list of conditions and the following disclaimer in the
00016  * documentation and/or other materials provided with the
00017  * distribution.  Neither the name of Sony Pictures Imageworks nor the
00018  * names of its contributors may be used to endorse or promote
00019  * products derived from this software without specific prior written
00020  * permission.
00021  *
00022  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
00023  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
00024  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
00025  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
00026  * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
00027  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
00028  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
00029  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
00030  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
00031  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
00032  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
00033  * OF THE POSSIBILITY OF SUCH DAMAGE.
00034  */
00035 
00036 //----------------------------------------------------------------------------//
00037 
00042 //----------------------------------------------------------------------------//
00043 
00044 #ifndef _INCLUDED_Field3D_SparseFile_H_
00045 #define _INCLUDED_Field3D_SparseFile_H_
00046 
00047 //----------------------------------------------------------------------------//
00048 
00049 #include <vector>
00050 #include <list>
00051 
00052 #include <hdf5.h>
00053 
00054 #include "Exception.h"
00055 #include "Hdf5Util.h"
00056 #include "SparseDataReader.h"
00057 #include "Traits.h"
00058 
00059 //----------------------------------------------------------------------------//
00060 
00061 #include "ns.h"
00062 
00063 FIELD3D_NAMESPACE_OPEN
00064 
00065 //----------------------------------------------------------------------------//
00066 // Forward declarations
00067 //----------------------------------------------------------------------------//
00068 
00069 namespace Sparse {
00070 
00071   template <typename Data_T>
00072   struct SparseBlock;
00073 
00074 }
00075 
00076 template <typename Data_T>
00077 class SparseField;
00078 
00079 //----------------------------------------------------------------------------//
00080 
00081 namespace SparseFile {
00082 
00083 //----------------------------------------------------------------------------//
00084 // Reference
00085 //----------------------------------------------------------------------------//
00086 
00092 //----------------------------------------------------------------------------//
00093 
00094 template <class Data_T>
00095 class Reference 
00096 {
00097 public:
00098 
00099   // Typedefs ------------------------------------------------------------------
00100 
00101   typedef std::vector<Sparse::SparseBlock<Data_T>*> BlockPtrs;
00102 
00103   // Public data members -------------------------------------------------------
00104 
00105   std::string filename;
00106   std::string layerPath;
00107   int valuesPerBlock;
00108   int occupiedBlocks;
00109  
00111   std::vector<int> fileBlockIndices;
00114   std::vector<int> blockLoaded;
00117   BlockPtrs blocks;
00121   std::vector<bool> blockUsed;
00124   std::vector<int> loadCounts;
00128   std::vector<int> refCounts;
00132   boost::mutex *blockMutex;
00133 
00134   // Ctors, dtor ---------------------------------------------------------------
00135 
00137   Reference(const std::string filename, const std::string layerPath);
00138   ~Reference();
00139 
00141   Reference(const Reference &o);
00142 
00144   Reference & operator=(const Reference &o);
00145 
00146   // Main methods --------------------------------------------------------------
00147 
00149   bool fileIsOpen();
00151   void setNumBlocks(int numBlocks);
00155   void openFile();
00159   void loadBlock(int blockIdx);
00161   void unloadBlock(int blockIdx);
00164   void incBlockRef(int blockIdx);
00166   void decBlockRef(int blockIdx);
00168   int blockSize(int blockIdx) const;
00171   int totalLoads() const;
00174   int numLoadedBlocks() const;
00177   int totalLoadedBlocks() const;
00180   float averageLoads() const;
00182   void resetCacheStatistics();
00183 
00184 private:
00185 
00187   hid_t m_fileHandle;
00188   
00191   Hdf5Util::H5ScopedGopen m_layerGroup;
00192 
00195   SparseDataReader<Data_T> *m_reader;
00196 
00198   boost::mutex m_mutex;
00199 
00200 };
00201 
00202 //----------------------------------------------------------------------------//
00203 // References
00204 //----------------------------------------------------------------------------//
00205 
00206 class FileReferences
00207 {
00208 public:
00209 
00210   // Main methods --------------------------------------------------------------
00211 
00214   template <class Data_T>
00215   Reference<Data_T>& ref(int idx);
00216 
00219   template <class Data_T>
00220   int append(const Reference<Data_T>& ref);
00221 
00223   template <class Data_T>
00224     int numRefs() const;
00225 
00226 private:
00227 
00228   // Data members --------------------------------------------------------------
00229 
00230   std::vector<Reference<half> > m_hRefs;
00231   std::vector<Reference<V3h> > m_vhRefs;
00232   std::vector<Reference<float> > m_fRefs;
00233   std::vector<Reference<V3f> > m_vfRefs;
00234   std::vector<Reference<double> > m_dRefs;
00235   std::vector<Reference<V3d> > m_vdRefs;
00236 
00237 };
00238 
00239 //----------------------------------------------------------------------------//
00240 
00241 class CacheBlock {
00242 public:
00243   DataTypeEnum blockType;
00244   int refIdx;
00245   int blockIdx;
00246   CacheBlock(DataTypeEnum blockTypeIn, int refIdxIn,  int blockIdxIn) :
00247     blockType(blockTypeIn), refIdx(refIdxIn), blockIdx(blockIdxIn) 
00248   { }
00249 };
00250 
00251 //----------------------------------------------------------------------------//
00252 
00253 } // namespace SparseFile
00254 
00255 //----------------------------------------------------------------------------//
00256 // SparseFileManager
00257 //----------------------------------------------------------------------------//
00258 
00314 //----------------------------------------------------------------------------//
00315 
00316 class SparseFileManager
00317 {
00318 
00319 public:
00320 
00321   template <class Data_T>
00322   friend class SparseField;
00323 
00324   // typedefs ------------------------------------------------------------------
00325 
00326   typedef std::list<SparseFile::CacheBlock> CacheList;
00327 
00328   // Main methods --------------------------------------------------------------
00329 
00331   static SparseFileManager &singleton();
00332 
00335   void setLimitMemUse(bool enabled);
00336 
00339   bool doLimitMemUse() const;
00340 
00342   void setMaxMemUse(float maxMemUse);
00343 
00346   void flushCache();
00347 
00349   long long totalLoads();
00350 
00352   long long numLoadedBlocks();
00353 
00355   long long totalLoadedBlocks();
00356 
00359   float cacheFractionLoaded();
00360 
00362   float cacheLoadsPerBlock();
00363 
00366   float cacheEfficiency();
00367 
00369   void resetCacheStatistics();
00370 
00371   //--------------------------------------------------------------------------//
00372   // Utility functions
00373 
00378   template <class Data_T>
00379   void incBlockRef(int fileId, int blockIdx);
00380 
00385   template <class Data_T>
00386   void decBlockRef(int fileId, int blockIdx);
00387 
00391   template <class Data_T>
00392   void activateBlock(int fileId, int blockIdx);
00393 
00394 protected:
00395 
00397   template <class Data_T>
00398   SparseFile::Reference<Data_T> &reference(int index);
00399 
00402   template <class Data_T>
00403   int getNextId(const std::string filename, const std::string layerPath);
00404 
00405   template <class Data_T>
00406   void removeFieldFromCache(int refIdx);
00407 
00408 private:
00409 
00411   SparseFileManager();
00412 
00414   static SparseFileManager *ms_singleton;
00415 
00417   void addBlockToCache(DataTypeEnum blockType, int fileId, int blockIdx);
00418 
00421   void deallocateBlocks(int bytesNeeded);
00422 
00425   template <class Data_T>
00426   int deallocateBlock(const SparseFile::CacheBlock &cb);
00427 
00429   template <class Data_T>
00430   void deallocateBlock(CacheList::iterator &it);
00431 
00433   float m_maxMemUse;
00434 
00436   int m_maxMemUseInBytes;
00437 
00439   int m_memUse;
00440 
00443   bool m_limitMemUse;
00444 
00447   SparseFile::FileReferences m_fileData;
00448 
00454   CacheList m_blockCacheList;
00455 
00458   CacheList::iterator m_nextBlock;
00459 
00462   boost::mutex m_mutex;
00463 
00464 };
00465 
00466 //----------------------------------------------------------------------------//
00467 // Reference implementations
00468 //----------------------------------------------------------------------------//
00469 
00470 namespace SparseFile {
00471 
00472 //----------------------------------------------------------------------------//
00473 
00474 template <class Data_T>
00475 Reference<Data_T>::Reference(const std::string a_filename, 
00476                              const std::string a_layerPath)
00477   : filename(a_filename), layerPath(a_layerPath),
00478     valuesPerBlock(-1), occupiedBlocks(-1),
00479     blockMutex(NULL), m_fileHandle(-1), m_reader(NULL) { 
00480   /* Empty */ 
00481 }
00482 
00483 //----------------------------------------------------------------------------//
00484 
00485 template <class Data_T>
00486 Reference<Data_T>::~Reference()
00487 {
00488   if (m_reader)
00489     delete m_reader;
00490 
00491   if (blockMutex)
00492     delete [] blockMutex;
00493 }
00494 
00495 //----------------------------------------------------------------------------//
00496 
00497 template <class Data_T>
00498 Reference<Data_T>::Reference(const Reference<Data_T> &o)
00499 {
00500   m_reader = NULL;
00501   blockMutex = NULL;
00502   *this = o;
00503 }
00504 
00505 //----------------------------------------------------------------------------//
00506 
00507 template <class Data_T>
00508 Reference<Data_T> &
00509 Reference<Data_T>::operator=(const Reference<Data_T> &o)
00510 {
00511   // Copy public member variables (where appropriate)
00512   filename = o.filename;
00513   layerPath = o.layerPath;
00514   valuesPerBlock = o.valuesPerBlock;
00515   occupiedBlocks = o.occupiedBlocks;
00516   fileBlockIndices = o.fileBlockIndices;
00517   blockLoaded = o.blockLoaded;
00518   blocks = o.blocks;
00519   blockUsed = o.blockUsed;
00520   loadCounts = o.loadCounts;
00521   refCounts = o.refCounts;
00522   if (blockMutex)
00523     delete[] blockMutex;
00524   blockMutex = new boost::mutex[blocks.size()];
00525 
00526   // Copy private member variables (where appropriate)
00527   m_fileHandle = o.m_fileHandle;
00528   // Don't copy id, let hdf5 generate a new one.
00529   if (m_fileHandle >= 0) {
00530     m_layerGroup.open(m_fileHandle, layerPath.c_str());
00531   }
00532 
00533   if (m_reader)
00534     delete m_reader;
00535   m_reader = NULL;
00536 
00537   return *this;
00538 }
00539 
00540 //----------------------------------------------------------------------------//
00541 
00542 template <class Data_T>
00543 bool Reference<Data_T>::fileIsOpen()
00544 {
00545   return m_fileHandle >= 0;
00546 }
00547 
00548 //----------------------------------------------------------------------------//
00549 
00550 template <class Data_T>
00551 void Reference<Data_T>::setNumBlocks(int numBlocks)
00552 {
00553   boost::mutex::scoped_lock lock(m_mutex);
00554 
00555   fileBlockIndices.resize(numBlocks);
00556   blockLoaded.resize(numBlocks, 0);
00557   blocks.resize(numBlocks, 0);
00558   blockUsed.resize(numBlocks, false);
00559   loadCounts.resize(numBlocks, 0);
00560   refCounts.resize(numBlocks, 0);
00561   if (blockMutex)
00562     delete[] blockMutex;
00563   blockMutex = new boost::mutex[numBlocks];
00564 }
00565 
00566 //----------------------------------------------------------------------------//
00567 
00568 template <class Data_T>
00569 void Reference<Data_T>::openFile()
00570 {
00571   using namespace Exc;
00572   using namespace Hdf5Util;
00573 
00574   boost::mutex::scoped_lock lock_A(m_mutex);
00575 
00576   // check that the file wasn't already opened before obtaining the lock
00577   if (fileIsOpen()) {
00578     return;
00579   }
00580 
00581   m_fileHandle = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
00582   if (m_fileHandle < 0)
00583     throw NoSuchFileException(filename);
00584 
00585   m_layerGroup.open(m_fileHandle, layerPath.c_str());
00586   if (m_layerGroup.id() < 0) {
00587     Msg::print(Msg::SevWarning, "In SparseFile::Reference::openFile: "
00588               "Couldn't find layer group " + layerPath + 
00589               " in .f3d file ");
00590     throw FileIntegrityException(filename);
00591   }
00592 
00593   m_reader = new SparseDataReader<Data_T>(m_layerGroup.id(), valuesPerBlock, 
00594                                           occupiedBlocks);
00595 }
00596 
00597 //----------------------------------------------------------------------------//
00598 
00599 template <class Data_T>
00600 void Reference<Data_T>::loadBlock(int blockIdx)
00601 {
00602   boost::mutex::scoped_lock lock(m_mutex);
00603 
00604   // Allocate the block  
00605   blocks[blockIdx]->resize(valuesPerBlock);
00606   assert(blocks[blockIdx]->data.size() > 0);
00607   // Read the data
00608   assert(m_reader);
00609   m_reader->readBlock(fileBlockIndices[blockIdx], blocks[blockIdx]->dataRef());
00610   // Mark block as loaded
00611   blockLoaded[blockIdx] = 1;
00612 }
00613 
00614 //----------------------------------------------------------------------------//
00615 
00616 template <class Data_T>
00617 void Reference<Data_T>::unloadBlock(int blockIdx)
00618 {
00619   // Deallocate the block
00620   blocks[blockIdx]->clear();
00621 
00622   // Mark block as unloaded
00623   blockLoaded[blockIdx] = 0;
00624 }
00625 
00626 //----------------------------------------------------------------------------//
00627 
00628 template <class Data_T>
00629 void Reference<Data_T>::incBlockRef(int blockIdx)
00630 {
00631   boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
00632   ++refCounts[blockIdx];
00633 }
00634 
00635 //----------------------------------------------------------------------------//
00636 
00637 template <class Data_T>
00638 void Reference<Data_T>::decBlockRef(int blockIdx)
00639 {
00640   boost::mutex::scoped_lock lock(blockMutex[blockIdx]);
00641   --refCounts[blockIdx];
00642 }
00643 
00644 //----------------------------------------------------------------------------//
00645 
00646 template <class Data_T>
00647 int Reference<Data_T>::blockSize(int /* blockIdx */) const
00648 {
00649   return valuesPerBlock * sizeof(Data_T);
00650 }
00651 
00652 //----------------------------------------------------------------------------//
00653 
00654 template <class Data_T>
00655 int Reference<Data_T>::totalLoads() const
00656 {
00657   std::vector<int>::const_iterator i = loadCounts.begin();
00658   std::vector<int>::const_iterator end = loadCounts.end();
00659   int numLoads = 0;
00660   for (; i != end; ++i)
00661     numLoads += *i;
00662 
00663   return numLoads;
00664 }
00665 
00666 //----------------------------------------------------------------------------//
00667 
00668 template <class Data_T>
00669 int Reference<Data_T>::numLoadedBlocks() const
00670 {
00671   std::vector<int>::const_iterator i = blockLoaded.begin();
00672   std::vector<int>::const_iterator end = blockLoaded.end();
00673   int numBlocks = 0;
00674   for (; i != end; ++i)
00675     if (*i)
00676       numBlocks++;
00677 
00678   return numBlocks;
00679 }
00680 
00681 //----------------------------------------------------------------------------//
00682 
00683 template <class Data_T>
00684 int Reference<Data_T>::totalLoadedBlocks() const
00685 {
00686   std::vector<int>::const_iterator i = loadCounts.begin();
00687   std::vector<int>::const_iterator li = blockLoaded.begin();
00688   std::vector<int>::const_iterator end = loadCounts.end();
00689   int numBlocks = 0;
00690   for (; i != end; ++i, ++li)
00691     if (*i || *li)
00692       numBlocks++;
00693 
00694   return numBlocks;
00695 }
00696 
00697 //----------------------------------------------------------------------------//
00698 
00699 template <class Data_T>
00700 float Reference<Data_T>::averageLoads() const
00701 {
00702   std::vector<int>::const_iterator i = loadCounts.begin();
00703   std::vector<int>::const_iterator end = loadCounts.end();
00704   int numLoads = 0, numBlocks = 0;
00705   for (; i != end; ++i) {
00706     if (*i) {
00707       numLoads += *i;
00708       numBlocks++;
00709     }
00710   }
00711 
00712   return (float)numLoads / std::max(1, numBlocks);
00713 }
00714 
00715 //----------------------------------------------------------------------------//
00716 
00717 template <class Data_T>
00718 void Reference<Data_T>::resetCacheStatistics()
00719 {
00720   std::vector<int>::iterator li = loadCounts.begin();
00721   std::vector<int>::iterator lend = loadCounts.end();
00722   for (; li != lend; ++li)
00723     *li = 0;
00724 }
00725 
00726 //----------------------------------------------------------------------------//
00727 
00728 } // namespace SparseFile
00729 
00730 //----------------------------------------------------------------------------//
00731 // Specializations for FileReferences
00732 //----------------------------------------------------------------------------//
00733 
00734 namespace SparseFile {
00735 
00736 //----------------------------------------------------------------------------//
00737 
00738 template <>
00739 inline Reference<half>& 
00740 FileReferences::ref(int idx)
00741 {
00742   return m_hRefs[idx];
00743 }
00744 
00745 //----------------------------------------------------------------------------//
00746 
00747 template <>
00748 inline Reference<V3h>& 
00749 FileReferences::ref(int idx)
00750 {
00751   return m_vhRefs[idx];
00752 }
00753 
00754 //----------------------------------------------------------------------------//
00755 
00756 template <>
00757 inline Reference<float>& 
00758 FileReferences::ref(int idx)
00759 {
00760   return m_fRefs[idx];
00761 }
00762 
00763 //----------------------------------------------------------------------------//
00764 
00765 template <>
00766 inline Reference<V3f>& 
00767 FileReferences::ref(int idx)
00768 {
00769   return m_vfRefs[idx];
00770 }
00771 
00772 //----------------------------------------------------------------------------//
00773 
00774 template <>
00775 inline Reference<double>& 
00776 FileReferences::ref(int idx)
00777 {
00778   return m_dRefs[idx];
00779 }
00780 
00781 //----------------------------------------------------------------------------//
00782 
00783 template <>
00784 inline Reference<V3d>& 
00785 FileReferences::ref(int idx)
00786 {
00787   return m_vdRefs[idx];
00788 }
00789 
00790 //----------------------------------------------------------------------------//
00791 
00792 template <>
00793 inline int FileReferences::append(const Reference<half>& ref)
00794 {
00795   m_hRefs.push_back(ref);
00796   return m_hRefs.size() - 1;
00797 }
00798 
00799 //----------------------------------------------------------------------------//
00800 
00801 template <>
00802 inline int FileReferences::append(const Reference<V3h>& ref)
00803 {
00804   m_vhRefs.push_back(ref);
00805   return m_vhRefs.size() - 1;
00806 }
00807 
00808 //----------------------------------------------------------------------------//
00809 
00810 template <>
00811 inline int FileReferences::append(const Reference<float>& ref)
00812 {
00813   m_fRefs.push_back(ref);
00814   return m_fRefs.size() - 1;
00815 }
00816 
00817 //----------------------------------------------------------------------------//
00818 
00819 template <>
00820 inline int FileReferences::append(const Reference<V3f>& ref)
00821 {
00822   m_vfRefs.push_back(ref);
00823   return m_vfRefs.size() - 1;
00824 }
00825 
00826 //----------------------------------------------------------------------------//
00827 
00828 template <>
00829 inline int FileReferences::append(const Reference<double>& ref)
00830 {
00831   m_dRefs.push_back(ref);
00832   return m_dRefs.size() - 1;
00833 }
00834 
00835 //----------------------------------------------------------------------------//
00836 
00837 template <>
00838 inline int FileReferences::append(const Reference<V3d>& ref)
00839 {
00840   m_vdRefs.push_back(ref);
00841   return m_vdRefs.size() - 1;
00842 }
00843 
00844 //----------------------------------------------------------------------------//
00845 
00846 template <>
00847 inline int FileReferences::numRefs<half>() const
00848 {
00849   return m_hRefs.size();
00850 }
00851 
00852 //----------------------------------------------------------------------------//
00853 
00854 template <>
00855 inline int FileReferences::numRefs<V3h>() const
00856 {
00857   return m_vhRefs.size();
00858 }
00859 
00860 //----------------------------------------------------------------------------//
00861 
00862 template <>
00863 inline int FileReferences::numRefs<float>() const
00864 {
00865   return m_fRefs.size();
00866 }
00867 
00868 //----------------------------------------------------------------------------//
00869 
00870 template <>
00871 inline int FileReferences::numRefs<V3f>() const
00872 {
00873   return m_vfRefs.size();
00874 }
00875 
00876 //----------------------------------------------------------------------------//
00877 
00878 template <>
00879 inline int FileReferences::numRefs<double>() const
00880 {
00881   return m_dRefs.size();
00882 }
00883 
00884 //----------------------------------------------------------------------------//
00885 
00886 template <>
00887 inline int FileReferences::numRefs<V3d>() const
00888 {
00889   return m_vdRefs.size();
00890 }
00891 
00892 //----------------------------------------------------------------------------//
00893 // Implementations for FileReferences
00894 //----------------------------------------------------------------------------//
00895 
00896 template <class Data_T>
00897 Reference<Data_T>& FileReferences::ref(int idx)
00898 {
00899   assert(false && "Do not use memory limiting on sparse fields that aren't "
00900          "simple scalars or vectors!");
00901   Msg::print(Msg::SevWarning, 
00902              "FileReferences::ref(): Do not use memory limiting on sparse "
00903              "fields that aren't simple scalars or vectors!");
00904   static Reference<Data_T> dummy("", "");
00905   return dummy;
00906 }
00907 
00908 //----------------------------------------------------------------------------//
00909 
00910 template <class Data_T>
00911 int FileReferences::append(const Reference<Data_T>& ref)
00912 {
00913   assert(false && "Do not use memory limiting on sparse fields that aren't "
00914          "simple scalars or vectors!");
00915   Msg::print(Msg::SevWarning,
00916              "FileReferences::append(): Do not use memory limiting on sparse "
00917              "fields that aren't simple scalars or vectors!");
00918   return -1;    
00919 }
00920 
00921 //----------------------------------------------------------------------------//
00922 
00923 template <class Data_T>
00924 int FileReferences::numRefs() const
00925 {
00926   assert(false && "Do not use memory limiting on sparse fields that aren't "
00927          "simple scalars or vectors!");
00928   Msg::print(Msg::SevWarning,
00929              "FileReferences::numRefs(): "
00930              "Do not use memory limiting on sparse "
00931              "fields that aren't "
00932              "simple scalars or vectors!");
00933   return -1;
00934 }
00935 
00936 //----------------------------------------------------------------------------//
00937 
00938 } // namespace SparseFile
00939 
00940 //----------------------------------------------------------------------------//
00941 // SparseFileManager implementations
00942 //----------------------------------------------------------------------------//
00943 
00944 template <class Data_T>
00945 int 
00946 SparseFileManager::getNextId(const std::string filename, 
00947                              const std::string layerPath)
00948 {
00949   using namespace SparseFile;
00950 
00951   int id = m_fileData.append(Reference<Data_T>(filename, layerPath));
00952   return id;
00953 }
00954 
00955 //----------------------------------------------------------------------------//
00956 
00957 template <class Data_T>
00958 void
00959 SparseFileManager::removeFieldFromCache(int refIdx)
00960 {
00961   boost::mutex::scoped_lock lock(m_mutex);
00962 
00963   DataTypeEnum blockType = DataTypeTraits<Data_T>::typeEnum();
00964   SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(refIdx);
00965 
00966   CacheList::iterator it = m_blockCacheList.begin();
00967   CacheList::iterator end = m_blockCacheList.end();
00968   CacheList::iterator next;
00969 
00970   int bytesFreed = 0;
00971 
00972   while (it != end) {
00973     if (it->blockType == blockType && it->refIdx == refIdx) {
00974       if (it == m_nextBlock) {
00975         ++m_nextBlock;
00976       }
00977       next = it;
00978       ++next;
00979       bytesFreed += reference.blockSize(it->blockIdx);
00980       m_blockCacheList.erase(it);
00981       it = next;
00982     } else {
00983       ++it;
00984     }
00985   }
00986   m_memUse -= bytesFreed;
00987 
00988   // reset the block indices to -1, to ensure that the cache manager
00989   // won't try to activate a block
00990   reference.fileBlockIndices.clear();
00991   reference.fileBlockIndices.resize(reference.blocks.size(), -1);
00992   // clear the reference's pointers into the field, and relevant 
00993   reference.blocks.clear();
00994   reference.blockLoaded.clear();
00995   reference.blockUsed.clear();
00996 }
00997 
00998 //----------------------------------------------------------------------------//
00999 
01000 template <class Data_T>
01001 SparseFile::Reference<Data_T> &
01002 SparseFileManager::reference(int index)
01003 { 
01004   return m_fileData.ref<Data_T>(index); 
01005 }
01006 
01007 //----------------------------------------------------------------------------//
01008 
01009 template <class Data_T>
01010 void 
01011 SparseFileManager::activateBlock(int fileId, int blockIdx)
01012 {
01013   SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(fileId);
01014 
01015   if (reference.fileBlockIndices[blockIdx] >= 0) {
01016     if (!reference.blockLoaded[blockIdx]) {
01017       int blockSize = reference.blockSize(blockIdx);
01018       if (m_limitMemUse) {
01019         // if we already have enough free memory, deallocateBlocks()
01020         // will just return
01021         deallocateBlocks(blockSize);
01022       }
01023 
01024       if (!reference.fileIsOpen()) {
01025         reference.openFile();
01026       }
01027 
01028       boost::mutex::scoped_lock lock_A(m_mutex);
01029       boost::mutex::scoped_lock lock_B(reference.blockMutex[blockIdx]);
01030       // check to see if it was loaded between when the function
01031       // started and we got the lock on the block
01032       if (!reference.blockLoaded[blockIdx]) {
01033         reference.loadBlock(blockIdx);
01034         reference.loadCounts[blockIdx]++;
01035         addBlockToCache(DataTypeTraits<Data_T>::typeEnum(), fileId, blockIdx);
01036         m_memUse += blockSize;
01037       }
01038     }
01039   }
01040   reference.blockUsed[blockIdx] = true;
01041 }
01042 
01043 //----------------------------------------------------------------------------//
01044 
01045 template <class Data_T>
01046 void 
01047 SparseFileManager::incBlockRef(int fileId, int blockIdx)
01048 {
01049   SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(fileId);
01050 
01051   if (reference.fileBlockIndices[blockIdx] >= 0) {
01052     reference.incBlockRef(blockIdx);
01053   }
01054 }
01055 
01056 //----------------------------------------------------------------------------//
01057 
01058 template <class Data_T>
01059 void 
01060 SparseFileManager::decBlockRef(int fileId, int blockIdx)
01061 {
01062   SparseFile::Reference<Data_T> &reference = m_fileData.ref<Data_T>(fileId);
01063 
01064   if (reference.fileBlockIndices[blockIdx] >= 0) {
01065     reference.decBlockRef(blockIdx);
01066   }
01067 }
01068 
01069 //----------------------------------------------------------------------------//
01070 
01071 FIELD3D_NAMESPACE_HEADER_CLOSE
01072 
01073 //----------------------------------------------------------------------------//
01074 
01075 #endif