DbiTableProxy.cxx

Go to the documentation of this file.

// $Id: DbiTableProxy.cxx,v 1.28 2006/08/08 10:51:32 west Exp $

#include "Conventions/Detector.h"
#include "Conventions/SimFlag.h"
#include "DatabaseInterface/DbiBinaryFile.h"
#include "DatabaseInterface/DbiCache.h"
#include "DatabaseInterface/DbiConnectionMaintainer.h"
#include "DatabaseInterface/DbiResultAgg.h"
#include "DatabaseInterface/DbiResultNonAgg.h"
#include "DatabaseInterface/DbiResultSet.h"
#include "DatabaseInterface/DbiTableProxy.h"
#include "DatabaseInterface/DbiTableRow.h"
#include "DatabaseInterface/DbiTimerManager.h"
#include "DatabaseInterface/DbiValidityRec.h"
#include "DatabaseInterface/DbiValidityRecBuilder.h"
#include "LeakChecker/Lea.h"
#include "MessageService/MsgService.h"

ClassImp(DbiTableProxy)

//   Definition of static data members
//   *********************************

CVSID("$Id: DbiTableProxy.cxx,v 1.28 2006/08/08 10:51:32 west Exp $");

//    Definition of all member functions (static or otherwise)
//    *******************************************************
//
//    -  ordered: ctors, dtor, operators then in alphabetical order.

//.....................................................................

DbiTableProxy::DbiTableProxy(DbiCascader* cascader, 
                             const string& tableName,
                             const string& vldSuffix,
                             const DbiTableRow* tableRow) :
  fCascader(cascader),
fMetaData(tableName),
fMetaValid(tableName+vldSuffix),
fCanL2Cache(kFALSE),
fCache(0),
fDBProxy(*cascader,tableName,&fMetaData,&fMetaValid,this),
fExists(0),
fTableName(tableName),
fTableRow(tableRow->CreateTableRow())
{
//
//
//  Purpose:  Constructor
//
//  Arguments:   
//             in  cascader   Reference to one and only cascader
//             in  tableName  Table name.
//             in  vldSuffix  Suffix to tableName for validity table
//             in  tableRow   Example table row object.
//
//  Return:    n/a
//
//  Contact:   N. West
//
//  Specification:-
//  =============
//
//  o Create table proxy for supplied table name.


//  Program Notes:-
//  =============

//  None.

  LEA_CTOR    //Leak Checker

  fCache = new DbiCache(*this,fTableName);
  this->RefreshMetaData();
  fExists = fDBProxy.TableExists();
  fCanL2Cache = tableRow->CanL2Cache();
  if ( fCanL2Cache ) 
    MSG("Dbi", Msg::kInfo) << "DbiTableProxy: Can L2 cache " << this->GetRowName() << endl;
  MSG("Dbi", Msg::kVerbose) << "Creating DbiTableProxy " 
                          << fTableName.c_str() << " at " << this
                          << ( fExists ? " (table exists)" 
                                       : " (table missing)" )
                          << endl;
}

//.....................................................................

DbiTableProxy::~DbiTableProxy() {
//
//
//  Purpose: Destructor
//
//  Arguments: 
//    None.
//
//  Return:    n/a
//
//  Contact:   N. West
//
//  Specification:-
//  =============
//
//  o  Destroy object.


//  Program Notes:-
//  =============

//  None.

  LEA_DTOR    //Leak Checker

    MSG("Dbi", Msg::kVerbose) << "Destroying DbiTableProxy " 
                            << fTableName << " at " << this
                            << endl;
  delete fCache;
  delete fTableRow;

}
//.....................................................................

Bool_t DbiTableProxy::CanReadL2Cache() const {
//

  return fCanL2Cache && DbiBinaryFile::CanReadL2Cache();

}
Bool_t DbiTableProxy::CanWriteL2Cache() const {
//

  return fCanL2Cache && DbiBinaryFile::CanWriteL2Cache();

}

//.....................................................................

const DbiResult* DbiTableProxy::Query(const VldContext& vc, 
                                      const Dbi::Task& task,
                                      Bool_t findFullTimeWindow) {
//
//
//  Purpose:  Apply context specific query to database table and return result.
//
//  Arguments: 
//    vc           in    The Validity Context for the query.
//    task         in    The task of the query.
//    findFullTimeWindow
//                 in    Attempt to find full validity of query 
//                        i.e. beyond Dbi::GetTimeGate
//
//  Return:    Query result (never zero even if query fails).
//
//  Contact:   N. West
//
//  Specification:-
//  =============
//
//  o Apply query to database table and return result.

//  Program Notes:-
//  =============

//  None.

//  See if there is one already in the cache for universal aggregate no.

  if ( const DbiResult* result = fCache->Search(vc,task) 
     ) return result;

  DbiConnectionMaintainer cm(fCascader);  //Stack object to hold connections

// Make Global Exception Log bookmark
  UInt_t startGEL = DbiExceptionLog::GetGELog().Size()+1;

// Build a complete set of effective validity record from the database.
  DbiValidityRecBuilder builder(fDBProxy,vc,task,-1,findFullTimeWindow);

// Deal with non-aggregated data.

  if ( builder.NonAggregated() ) {

    DbiValidityRec effVRec = builder.GetValidityRec(0);
//  Force off const - we haven't finished with DbiResult yet!
    DbiResult* result = const_cast<DbiResult*>(Query(effVRec));
//  Record latest entries from Global Exception Log.
    result->CaptureExceptionLog(startGEL);
    return result;
  }

// Deal with aggregated data.

// Don't look in the level 2 cache if more than half of the 
// component aggregates are already in the cache;
// for in this case, the chances are that we have just 
// crossed a validity boundary in only a few aggregates and 
// we don't want to waste time loading in a full set only to throw
// it away again.

  if ( this->CanReadL2Cache() ) {
    UInt_t numPresent  = 0;
    UInt_t numRequired = 0;
    Int_t maxRow = builder.GetNumValidityRec() - 1;
    for ( Int_t rowNo = 1; rowNo <= maxRow; ++rowNo ) {
      const DbiValidityRec& vrec = builder.GetValidityRec(rowNo);
      if ( fCache->Search(vrec) ) ++numPresent;
      else if ( ! vrec.IsGap() ) ++numRequired;
    }
    if ( numRequired < numPresent ) MSG("Dbi",Msg::kInfo) 
      << "Skipping search of L2 cache; already have "
      << numPresent << " aggregates, and only require a further "
      << numRequired << endl;
    else this->RestoreFromL2Cache(builder);
  }

  DbiResult* result = new DbiResultAgg(fTableName,
                                       fTableRow,
                                       fCache,
                                       &builder,
                                       &fDBProxy);
// Record latest entries from Global Exception Log.
  result->CaptureExceptionLog(startGEL);

  fCache->Adopt(result);
  this->SaveToL2Cache(builder.GetL2CacheName(),*result);
  return result;

}
//.....................................................................

const DbiResult* DbiTableProxy::Query(const string& context, 
                                      const Dbi::Task& task,
                                      const string& data,
                                      const string&fillOpts) {
//
//
//  Purpose:  Apply extended context query to database table and return result.
//
//  Arguments: 
//    context      in    The Validity Context (see DbiSqlContext)
//    task         in    The task of the query.
//    data         in    Optional SQL extension to secondary query.
//    fillOpts     in    Optional fill options (available to DbiTableRow)
//
//  Return:    Query result (never zero even if query fails).
//
//  Contact:   N. West
//
//  Specification:-
//  =============
//
//  o Apply extended context query to database table and return result.
//
//  o Don't save/restore to L2 cache: encoding the query name as a file name
//    would be cumbersome and in any case extended queries are abnormal
//    so optimisation is unwarranted.


//  Construct the query's "SQL Qualifiers" by forming the 3 strings 
//  (which task encoded into the context) into a single semi-colon 
//  separated string.

  std::ostringstream os;
  os << context;
  if ( task != Dbi::kAnyTask
       ) os << " and  Task = " << task;
  os <<  ';' << data << ';' << fillOpts;
  string sqlQualifiers = os.str();

  MSG("Dbi", Msg::kVerbose) 
    << "Extended query: sqlQualifiers: " << sqlQualifiers << endl;

//  See if there is one already in the cache.

  if ( const DbiResult* result = fCache->Search(sqlQualifiers) 
     ) return result;

  DbiConnectionMaintainer cm(fCascader);  //Stack object to hold connections

// Make Global Exception Log bookmark
  UInt_t startGEL = DbiExceptionLog::GetGELog().Size()+1;

// Build a complete set of effective validity records from the database.
  DbiValidityRecBuilder builder(fDBProxy,context,task);

// For extended context queries, DbiValidityRecBuilder will always 
// assemble a result that has to be represented by a DbiResultAgg

  DbiResult* result = new DbiResultAgg(fTableName,
                                       fTableRow,
                                       fCache,
                                       &builder,
                                       &fDBProxy,
                                       sqlQualifiers);
// Record latest entries from Global Exception Log.
  result->CaptureExceptionLog(startGEL);

  fCache->Adopt(result);
  return result;

}
//.....................................................................

const DbiResult* DbiTableProxy::Query(UInt_t seqNo,UInt_t dbNo) {
//
//
//  Purpose:  Apply non-agregate query to database table and return result.
//
//  Arguments: 
//    seqNo        in    The sequence number of validity record that satisfies the query.
//    dbNo         in    Database number in the cascade.
//
//  Return:    Query result (never zero even if query fails).

  DbiConnectionMaintainer cm(fCascader);  //Stack object to hold connections

// Make Global Exception Log bookmark
  UInt_t startGEL = DbiExceptionLog::GetGELog().Size()+1;

  // Apply SEQNO query to cascade member.
  DbiResultSet* rs = fDBProxy.QueryValidity(seqNo,dbNo);
  DbiValidityRec tr;
  DbiResultNonAgg result(rs,&tr,0,kFALSE);
  delete rs;

  // If query failed, return an empty result.
  if ( result.GetNumRows() == 0 ) {
    DbiResultNonAgg* empty = new DbiResultNonAgg();
//  Record latest entries from Global Exception Log.
    empty->CaptureExceptionLog(startGEL);
    fCache->Adopt(empty);
    return empty;
  }

// Otherwise perform a validity rec query, but don't
// allow result to be used; it's validity has not been trimmed
// by neighbouring records.

  const DbiValidityRec* vrec 
       = dynamic_cast<const DbiValidityRec*>(result.GetTableRow(0));
//  Force off const - we haven't finished with DbiResult yet!
  DbiResult* res = const_cast<DbiResult*>(Query(*vrec,kFALSE));
// Record latest entries from Global Exception Log.
  res->CaptureExceptionLog(startGEL);
  return res;

}
//.....................................................................

const DbiResult* DbiTableProxy::Query(const DbiValidityRec& vrec, 
                                      Bool_t canReuse /* = kTRUE */) {
//
//
//  Purpose:  Apply non-agregate query to database table and return result.
//
//  Arguments: 
//    vrec         in    The validity record that satisfies the query.
//    canReuse     in    True if result is to be cached.
//
//  Return:    Query result (never zero even if query fails).
//
//  Contact:   N. West
//
//  Specification:-
//  =============
//
//  o Apply non-aggregated query to main database table. Cache if required,
//    and return result.


// See if it can be recovered from the level 2 disk cache.

  DbiConnectionMaintainer cm(fCascader);  //Stack object to hold connections

// Make Global Exception Log bookmark
  UInt_t startGEL = DbiExceptionLog::GetGELog().Size()+1;

  if ( canReuse ) {
    DbiValidityRecBuilder builder(vrec,this->GetTableName());
    if ( this->RestoreFromL2Cache(builder) ) {
      const DbiResult* res = fCache->Search(vrec);
      if ( res ) return res;
    }
  }

  unsigned int seqNo = vrec.GetSeqNo();
  DbiResult* result = 0;

//  If no records, create an empty DbiResult.
  if ( ! seqNo ) {
    result = new DbiResultNonAgg(0,0,&vrec);
  }

//  If query does not apply to this table, report error and
//  produce an empty DbiResult.

  else if (vrec.GetTableProxy()->GetTableName() != GetTableName() ) {
    MAXMSG("Dbi",Msg::kError,20) 
       << "Unable to satisfy DbiValidityRec keyed query:" << endl
       << vrec
       << " was filled by " << vrec.GetTableProxy()->GetTableName() 
       << " not by this DbiTableProxy (" 
       << GetTableName() << ")" << endl;
    result = new DbiResultNonAgg(0,0,&vrec);
  }

  else {


// Apply query, and build DiResult from its DbiResultSet.

    DbiResultSet* rs = fDBProxy.QuerySeqNo(seqNo,vrec.GetDbNo());
    result = new DbiResultNonAgg(rs,fTableRow,&vrec);
    delete rs;
  }

// Record latest entries from Global Exception Log.
  result->CaptureExceptionLog(startGEL);

//  Cache in memory and on disk if required and return the results.

  fCache->Adopt(result);
  if ( canReuse ) this->SaveToL2Cache(vrec.GetL2CacheName(),*result);
  else result->SetCanReuse(kFALSE);

  return result;

}

//.....................................................................

void DbiTableProxy::RefreshMetaData() {
//
//
//  Purpose:  Refresh meta data for table.
//

  fDBProxy.StoreMetaData(fMetaData);
  fDBProxy.StoreMetaData(fMetaValid);

}
//.....................................................................

VldTimeStamp DbiTableProxy::QueryOverlayCreationDate(const DbiValidityRec& vrec,
                                                     UInt_t dbNo)
{
//
//  Purpose:  Determine a suitable Creation Date so that this validity
//            record, if written to the selected DB, will overlay
//            correctly.
//
//  Specification:-
//  =============
//
//  o Determine optimal Creation Date to overlay new data.  See Program Notes.

//  Program Notes:-
//  =============


// It is normal practice, particularly for calibration data, to have
// overlapping the validity records.  Each time a new set of runs are
// processed the start time of the validity is set to the start time of
// the first run and the end time is set beyond the start time by an
// interval that characterises the stability of the constants.  So long
// as a new set of constants is created before the end time is reached
// there will be no gap.  Where there is an overlap the Creation Date is
// used to select the later constants on the basis that later is better.
// However, if reprocessing old data it is also normal practice to
// process recent data first and in this case the constants for earlier
// data get later creation dates and overlay works the wrong way.  To
// solve this, the creation date is faked as follows:-
//
//
//   1.  For new data i.e. data that does not overlay any existing data,
//       the creation date is set to the validity start time.
//
//   2.  For replacement data i.e. data that does overlay existing data,
//       the creation date is set to be one minute greater than the Creation
//       Date on the current best data.
//
// This scheme ensures that new data will overlay existing data at the 
// start of its validity but will be itself overlaid by data that has 
// a later start time (assuming validity record start times are more 
// than a few minutes apart)


  //  Create a context that corresponds to the start time of the validity
  //  range.  Note that it is O.K. to use SimFlag and Detector masks 
  //  even though this could make the context ambiguous because the 
  //  context is only to be used to query the database and the SimFlag and 
  //  Detector values will be ORed against existing data so will match
  //  all possible data that this validity range could overlay which is
  //  just what we want.

  const VldRange& vr(vrec.GetVldRange());
  VldContext vc((Detector::Detector_t) vr.GetDetectorMask(),
                  (SimFlag::SimFlag_t) vr.GetSimMask(),
                                       vr.GetTimeStart());

  DbiConnectionMaintainer cm(fCascader);  //Stack object to hold connections

  // Build a complete set of effective validity records from the 
  // selected database.
  DbiValidityRecBuilder builder(fDBProxy,vc,vrec.GetTask(),dbNo);

  // Pick up the validity record for the current aggregate.
  const DbiValidityRec& vrecOvlay(builder.GetValidityRecFromAggNo(vrec.GetAggregateNo()));

  // If its a gap i.e. nothing is overlayed, return the start time, otherwise
  // return its Creation Date plus one minute.
  VldTimeStamp ovlayTS(vr.GetTimeStart());
  if ( ! vrecOvlay.IsGap() ) {
    time_t overlaySecs = vrecOvlay.GetCreationDate().GetSec();
    ovlayTS = VldTimeStamp(overlaySecs + 60,0);
  }

  MSG("Dbi",Msg::kDebug) << "Looking for overlay creation date for: " 
                   << vrec << "found it would overlap: "
                   << vrecOvlay << " so overlay creation date set to "
                   << ovlayTS.AsString("s") << endl;
  return ovlayTS;                                          

}
//.....................................................................

Bool_t DbiTableProxy::RestoreFromL2Cache(const DbiValidityRecBuilder& builder) {
//
//
//  Purpose: Restore results from named level 2 disk cache into memory cache. 
//  Returns true if anything restored

//  Specification:-
//  =============
//
//  o Restore to cache but only if enabled and exists.

  const string name(builder.GetL2CacheName());
  MSG("Dbi",Msg::kDebug) << "Request to restore query result  " << name 
                         << endl;
  if ( ! this->CanReadL2Cache() ) return kFALSE;
  string cacheFileName;
  if (  name != "" 
   ) cacheFileName =  this->GetTableName() + "_" 
                    + this->GetRowName() + "_" 
                    +  name + ".dbi_cache";
  DbiBinaryFile bf(cacheFileName.c_str());
  if ( ! bf.IsOK() ) {
    MSG("Dbi",Msg::kDebug) << "Caching disabled or cannot open " 
                           << bf.GetFileName() << endl;
    return kFALSE;
  }

  static bool warnOnce = true;
  if ( warnOnce ) {
    MSG("Dbi",Msg::kWarning) << "\n\n\n"
     << " WARNING:  Reading from the Level 2 cache has been activated.\n"
     << " *******   This should only be used for development and never for production !!!\n\n\n";
    warnOnce = false;
  }

  MSG("Dbi",Msg::kInfo) << "Restoring query result from " << bf.GetFileName() << endl;
  DbiTimerManager::gTimerManager.RecMainQuery();

  DbiResult* result    = 0;
  unsigned numRowsRest = 0;
  unsigned numRowsIgn  = 0;
  UInt_t numNonAgg     = 0;
  bf >> numNonAgg;  

  while ( numNonAgg-- ) {
    if ( ! bf.IsOK() ) break;
    if ( ! result ) result = new DbiResultNonAgg;
    bf >> *result;

//  The original query may have had a validity range truncated by
//  the time window, so replace its DbiValidityRec with the one
//  just obtained from the database.
    const DbiValidityRec& vrec = result->GetValidityRec();
    UInt_t seqNo = vrec.GetSeqNo();
    MSG("Dbi",Msg::kDebug) << "Fix up L2 cache DbiValidityRec, by replacing: " << vrec 
                           << "    with: " << builder.GetValidityRecFromSeqNo(seqNo) << endl;
//  Sneaky end-run round const to fix-up DbiValidityRec.
    (const_cast<DbiValidityRec&>(vrec)) = builder.GetValidityRecFromSeqNo(seqNo);

//  Adopt only if not already in memory cache.
    if ( ! fCache->Search(vrec) ) {
      numRowsRest += result->GetNumRows();
      fCache->Adopt(result);
      result = 0;
    }
    else numRowsIgn += result->GetNumRows();
  }
  MSG("Dbi",Msg::kInfo) << "   a total of " << numRowsRest << " were restored ("
                        << numRowsIgn << " ignored - already in memory)" << endl;

  delete result;
  result = 0;

  return numRowsRest > 0;

}
//.....................................................................

Bool_t DbiTableProxy::SaveToL2Cache(const string& name, DbiResult& res) {
//
//
//  Purpose: Save result to named level 2 cache. Returns true if saved.

//  Specification:-
//  =============
//
//  o Save to cache but only if enabled and suitable.

  MSG("Dbi",Msg::kDebug) << "Request to save query result as " << name 
                         << " data from DB? " << res.ResultsFromDb()
                         << " can be saved? " << res.CanSave() << endl;
  if ( ! this->CanWriteL2Cache() || ! res.ResultsFromDb() || ! res.CanSave() ) return kFALSE;

  string cacheFileName;
  if (  name != "" 
   ) cacheFileName =  this->GetTableName() + "_" 
                    + this->GetRowName() + "_" 
                    +  name + ".dbi_cache";
  DbiBinaryFile bf(cacheFileName.c_str(),kFALSE);
  if ( bf.IsOK() ) {
    MSG("Dbi",Msg::kInfo) << "Saving query result (" << res.GetNumRows()
                          << " rows) to " << bf.GetFileName() << endl;
     DbiTimerManager::gTimerManager.RecMainQuery();

    // if writing a DbiResultNonAgg, add leading count of 1. (if writing
    // a DbiResultAgg it will writes its one leading count.
    if ( dynamic_cast<DbiResultNonAgg*>(&res) ) {
      UInt_t numNonAgg = 1;
      bf << numNonAgg;
    }
    bf << res;
    return kTRUE;
  }
  MSG("Dbi",Msg::kDebug) << "Caching disabled or cannot open " 
                         << bf.GetFileName() << endl;
  return kFALSE;

}
//.....................................................................

void DbiTableProxy::SetSqlCondition(const string& sql) {
//
//
//  Purpose:  Apply Sql condition to its DbiDBProxy.
//
//  Arguments: 
//   sql           in    SQL condition string (excluding where).
//
//  Return:  n/a  
//
//  Contact:   N. West
//
//  Specification:-
//  =============
//
//  o Apply Sql condition to its DbiDBProxy.

//  Program Notes:-
//  =============

//  None.

  fDBProxy.SetSqlCondition(sql);

}

---