DDSFileHandler.cxx

Go to the documentation of this file.

//                                                                           //
// DDSFileHandler                                                            //
//                                                                           //
// Package: DDS (Data Dispatcher System).                                    //
//                                                                           //
// S. Kasahara 5/2001                                                        //
//                                                                           //
// Purpose: Interacts with system to determine next available file for a     //
//          given DDS::EDataSource.  Will construct and return the full file //
//          path name of the next file when requested.                       //
//                                                                           //
// The fullfilepathname of the current file for a given data source is       //
// determined from the following:                                            //
// The enumerated data source given to the DDSFileHandler at construction    //
// (DDS::kDaq or DDS::kDcs) is used to read the externally set               //
// environment variable DDS_DAQ_DATA_DIR or DDS_DCS_DATA_DIR which contains  //
// the path to the directory of the source data files.                       //
// If the user has subscribed to the Dispatcher with keepup mode             //
// DDS::kFileKeepUp, then it is expected that the user maintains a symbolic  //
// link file "currentfile" within this source directory that points to the   //
// most recent data file generated by that datasource.                       //
// Thus the full path of the current file generated by the data source is:   //
// $DDS_DAQ_DATA_DIR/currentfile or $DDS_DCS_DATA_DIR/currentfile            //
// If the user has subscribed to the Dispatcher with keepup mode             //
// DDS::kAll, then all files in the directory ending with the extension      //
// ".root" will be served starting with the currentfile if defined or with   //
// the first file in the directory if not defined.                           //
//                                                                           //
// If the user wishes to use the rootd server to serve *open* files from a   //
// disk on a machine other than the one on which the DDS runs (this should   //
// only be of concern for the detector sites), the path to the source        //
// directory should be expanded to include the "root:" preface and the host  //
// name on which the rootd server actually runs.  An example:                //
// setenv DDS_DAQ_DATA_DIR root://urheim.hep.umn.edu/data/minos/daq         //
// or                                                                        //
// setenv DDS_DAQ_DATA_DIR root://urheim.hep.umn.edu/~minos/daq              //
// where the format is:                                                      //
//                         root://<host name>/<path name>                    //
// The username&password of the remote machine on which the rootd server runs//
// must be specified in a .netrc file located in the user's home directory,  //
// unless rootd has been set up for anonymous usage.                         //
// The format for a line in the .netrc file is:                              //
//            machine <host name> login <username> password <password>       //
//            e.g.                                                           //
//            machine urheim.hep.umn.edu login myusername password mypassword//
// If the username&password is not found in the .netrc file, then anonymous  //
// login is attempted with username:anonymous password:username@hostname     //
//                                                                           //
// Note that even when rootd is being used to serve the files to the         //
// ddschildserver, the ddschildserver must still be able to "see" the        //
// directory containing the data using a file sharing system so that it can  //
// the symbolic link "currentfile" and directory contents with unix system   //
// functions.   If the local directory path differs from the <path name>     //
// specified in the env DDS_DAQ_DATA_DIR as root://<host name>/<path name>   //
// then the local directory path should be specified in a separate env       //
// variable DAQ_DATA_DIR.  Confused yet?                                     //
//                                                                           //

#include <algorithm>  // vector sort
#include <sys/stat.h>  
#include <unistd.h>    
#include <pwd.h>

#include "TAuthenticate.h"
#include "TString.h"
#include "TSystem.h"
#include "TUrl.h"

#include "MessageService/MsgService.h"
#include "Dispatcher/DDSFileHandler.h"

std::ostream& operator << (std::ostream& ms, DDSFileHandler* dfh)
                                 { return dfh->Print(ms); } 

ClassImp(DDSFileHandler)

// Definition of static data members
// *********************************

CVSID("$Id: DDSFileHandler.cxx,v 1.26 2008/11/18 21:16:14 rhatcher Exp $");
 
// Definition of methods (alphabetical order)
// ******************************************

Int_t DDSFileHandler::BuildFileIndex() {
  //
  // Purpose: Build file index of files in source directory.  Files are
  //          assumed to have subscript of form ".root".  
  //
  // Argument: none.
  // 
  // Return: Int_t  number of files in index
  //
  // Contact: S. Kasahara
  //


  fFileIndex.clear(); fFileIndexItr = fFileIndex.end(); // clear previous index

  if ( !IsValid() ) {
    MSG("DDS",Msg::kWarning) 
    << "BuildFileIndex called on InValid FileHandler." << endl;
    return 0;
  }

  // Open source directory
  void *dir = gSystem -> OpenDirectory(fLocalDataSourceDir.c_str());
  if (!dir) {
    MSG("DDS",Msg::kWarning) << "Error in open of file source directory " 
     << fLocalDataSourceDir << ".\nCannot build file index." << endl;
    return 0;
  }
  
  // update last modification time of source directory
  struct stat keystat;
  if ( stat(fLocalDataSourceDir.c_str(),&keystat) >= 0 ) {
    fSourceDirModTime = keystat.st_ctime;
  }

  const char* file;
  string filename;
  while ((file = gSystem -> GetDirEntry(dir)) ) {
    filename = file;
    // Determine if file has required extension
    if ( filename.length() > 5 
      && filename.find(".root") == filename.length()-5 ) {
      fFileIndex.push_back(fDataSourceDir+"/"+filename);
    }
  }
  // sort the directory contents in alphanumeric order
  std::sort(fFileIndex.begin(),fFileIndex.end());

  gSystem -> FreeDirectory(dir);

  fFileIndexItr = fFileIndex.begin();

  MsgStream& ms = MSGSTREAM("DDS",Msg::kVerbose);
  ms << "CS_" << gSystem->GetPid() << ": "<< "Built new FileIndex: " << endl;
  for (FileIndexConstItr citr  = fFileIndex.begin(); 
                         citr != fFileIndex.end(); citr++) {
    ms << "    " << *citr << endl; 
  }
  return fFileIndex.size();

}

std::string DDSFileHandler::BuildFullFilePathName(std::string filename) {
  //
  // Purpose: Build full file path name from file name.
  //
  // Argument: string    filename of interest.
  //
  // Return: fullfilepathname of file.
  //
  // Contact: S. Kasahara
  //
  // Notes: If the filename contains an absolute
  //        path specification beginning with "/", then nothing is added
  //        to the filename to create the fullfilepathname.  If the filename
  //        indicates a relative path, this method will prepend the data source
  //        directory path to the input filename if not already present. 
  //  

  std::string fullfilepathname = filename;
  unsigned int namelen = filename.length();
  if ( filename.find(fDataSourceDir) >= namelen ) {
    // Check if filename is a relative path and prepend data source directory
    if( filename.find("/") != 0 ) {
      fullfilepathname = fDataSourceDir + "/" + fullfilepathname;
    }
  }  

  // Check to see if fullfilepathname is symbolic link, and if so
  // expand to link target name
  const Int_t bufsize = 512;
  char buffer[bufsize];

  Int_t len = readlink(fullfilepathname.c_str(),buffer,bufsize);
  if ( len >= 0 ) {
    buffer[len] = '\0';  // null terminate string
    fullfilepathname = buffer;
    fullfilepathname = this -> BuildFullFilePathName(fullfilepathname);
  }
  return fullfilepathname;

}


DDSFileHandler::DDSFileHandler(DDS::EDataSource datasource,bool offLine) : 
fDataSource(datasource),fDataSourceEnv(""),fDataSourceDir(""),
fLocalDataSourceDir(""),fSymLink(""),fSymLinkTarget(""),fSymLinkModTime(0),
fSourceDirModTime(0),fLastServedFile(""),fValid(true),fFileIndexItr(0),
fOffLine(offLine) {
  // 
  // Purpose: Default constructor for DDSFileHandler class.  
  //  
  // Arguments: datasource   enumerated type specifies source of data. 
  //            offline      true => use symbolic link 'offlinefile'
  //                         instead of 'currentfile'. default = false.
  // Return: n/a.
  //
  // Contact: S. Kasahara
  //
  // Notes: Use IsValid() to determine if DDSFileHandler was initialized
  //        properly.
  //


  // Get name of directory containing data for requested datasource
  string dataSourceString = DDS::AsString(datasource);
  fDataSourceEnv = "DDS_"+dataSourceString+"_DATA_DIR";
  const char* sourcedir = getenv(fDataSourceEnv.c_str());
  if ( sourcedir == NULL ) {
    MSG("DDS",Msg::kWarning) << "Environment variable " 
                             << fDataSourceEnv << " undefined." << endl;
    fValid = false;
    return;
  } 
  else {
    fDataSourceDir = sourcedir;
  }

  // Determine if fDataSourceDir points to a remote host with rootd server
  if ( fDataSourceDir.find("root:")==0 || fDataSourceDir.find("roots:")==0 ) {
    TUrl url(fDataSourceDir.c_str());

    // check for existence of DAQ_DATA_DIR or DCS_DATA_DIR
    // indicating a local directory path different than that contained
    // in DDS_DAQ_DATA_DIR or DDS_DCS_DATA_DIR.
    string localDataSourceEnv = dataSourceString+"_DATA_DIR";
    const char* localSourceDir = getenv(localDataSourceEnv.c_str());
    if (localSourceDir != NULL) {
      fLocalDataSourceDir = localSourceDir;
    }
    else { 
      // Parse the source directory to determine the "local path name", used
      // to find the symbolic link
      string symlink = url.GetFile();
      fLocalDataSourceDir = symlink.substr(1,symlink.length()-1);
    }
    // Check to make sure that user has defined username&password for remote
    // host in .netrc file, if not will assume anonymous access
    TAuthenticate auth(0,url.GetHost(),url.GetProtocol(),0);

    TString tuser;
    TString tpasswd;
    if ( !auth.CheckNetrc(tuser,tpasswd) ) {
      MSG("DDS",Msg::kInfo) << "CS_" << gSystem->GetPid() << ": "
        << "No remote login data in ~.netrc. Assuming anonymous rootd usage."
        << endl;
      TAuthenticate::SetGlobalUser("anonymous");
      passwd* pwstruct = getpwuid(getuid());
      const char* username = 0;
      if (pwstruct) username = pwstruct -> pw_name;
      string password;
      if (username != NULL) 
        password = string(username) +"@" + string(gSystem->HostName());
      else
        password = "minos@" + string(gSystem->HostName());
      TAuthenticate::SetGlobalPasswd(password.c_str());
    }

  }
  else {
    fLocalDataSourceDir = fDataSourceDir; 
  }
  if ( fOffLine ) 
    fSymLink = fLocalDataSourceDir + "/offlinefile";
  else
    fSymLink = fLocalDataSourceDir + "/currentfile";
  
}

DDSFileHandler::~DDSFileHandler() {
  // Purpose: Destructor.
  //
  // Argument: n/a.
  //
  // Return: n/a.
  //
  // Contact: S. Kasahara
  //

}

std::string DDSFileHandler::GetCurrentFileName() const {
  //
  // Purpose: Retrieves full path name of file pointed to by fFileIndexItr.  
  //
  // Argument: none.
  // 
  // Return: std::string  full file path name of current file.
  //
  // Contact: S. Kasahara
  //

  std::string currentFileName;
  if (fFileIndexItr != fFileIndex.end() ) currentFileName = *fFileIndexItr;
  return currentFileName;

}

std::string DDSFileHandler::GetSymLinkTargetName() {
  //
  // Purpose: Retrieves full path name of file pointed to by fSymLink.
  //          (either "currentfile" or "offlinefile" depending on fOffLine
  //           bool).  
  //
  // Argument: none.
  // 
  // Return: std::string  full file path name of target of fSymLink.
  //
  // Contact: S. Kasahara
  //

  if ( this -> IsModifiedSymLink() ) {
    fSymLinkTarget = "";
    // update last modification time of key file
    struct stat keystat;
    if ( lstat(fSymLink.c_str(),&keystat) >= 0 ) {
      fSymLinkModTime = keystat.st_ctime;
      const Int_t bufsize = 512;
      char buffer[bufsize];
      Int_t len = readlink(fSymLink.c_str(),buffer,bufsize);
      if ( len >= 0 ) {
        buffer[len] = '\0';  // null terminate string
        if (buffer[0] == '/')   // absolute path
            fSymLinkTarget = buffer;
        else
            fSymLinkTarget = fDataSourceDir + "/" + buffer;
      }
      else {
        MSG("DDS",Msg::kWarning) << "CS_" << gSystem -> GetPid()
                                 << ": Error in reading symbolic link file:\n"
                                 << fSymLink << "." << endl;
      }
    }
    else {
      MSG("DDS",Msg::kWarning) << "CS_" << gSystem->GetPid()
                               << ": Error in reading symbolic link file:\n"
                               << fSymLink << " modification time." << endl;
    }
  }  
  
  return fSymLinkTarget;
  
}

std::string DDSFileHandler::GoToFile(std::string filename) {
  //
  // Purpose: Advance file iterator to point to the file indicated by
  //          fullFilePathName.  
  //
  // Argument: std::string  file name to advance to.  If file name does
  //                        not include the directory source path, this
  //                        information will be prepended to the file name.
  //                        If filename is null, advances to first file in 
  //                        directory. 
  //
  // Return: name of file corresponding to position index iterator was left at.
  //
  // Contact: S. Kasahara
  //
  // Notes: If specified file is not found, index iterator is left to
  //        point at 1 just beyond the missing file in the file index.
  //        If specified filename is null, index iterator is left to
  //        point at beginning of file index.
  //

  if ( this -> IsModifiedDirectory() ) this -> BuildFileIndex();

  std::string fullFilePathName = this -> BuildFullFilePathName(filename);

  if ( filename.empty() ) {
    fFileIndexItr = fFileIndex.begin();
  }
  else {
    for ( fFileIndexItr=fFileIndex.begin(); fFileIndexItr!=fFileIndex.end(); 
          fFileIndexItr++ ) {
      if ( (*fFileIndexItr) >= fullFilePathName ) {
        break;
      } 
    }
  }

  std::string nextFile = this -> GetCurrentFileName();
  if ( !nextFile.empty() ) {
    fLastServedFile = nextFile;
  }

  return nextFile;

}

std::string DDSFileHandler::GoToNextFile() {
  //
  // Purpose: Move current file iterator to point to the next file
  //          in source directory and return fullfilepathname of this file.
  //
  // Argument: none.
  // 
  // Return: std::string  full file path name of next file.
  //
  // Contact: S. Kasahara
  //
  // Notes:  Files are presumed to have names ending in  ".root".
  //         Files are in alphanumeric order.  If first call,
  //         the start of the file index is positioned at file pointed
  //         to by the symbolic link "currentfile", or if that symbolic
  //         link is not present, to the first file in the directory. 
  //

  std::string nextFileName = "";
  std::string currentFileName = this -> GetCurrentFileName();

  bool isNewBuild = false;
  if ( this -> IsModifiedDirectory() ) {
    this -> GetSymLinkTargetName();
    this -> BuildFileIndex();
    isNewBuild = true;
  }
 
  if ( fFileIndex.empty() ) {
    fFileIndexItr = fFileIndex.end();
    return "";
  }

  if ( fLastServedFile.empty() ) {
    // Start from currentfile symbolic link if available
    // Else first file in directory
    nextFileName = fSymLinkTarget;
    nextFileName = this -> GoToFile(nextFileName); 
  }
  else {
    // Move to next file in file index
    if ( !isNewBuild ) {
      if ( fFileIndexItr != fFileIndex.end() ) fFileIndexItr++;
      nextFileName = this -> GetCurrentFileName();
    }
    else {
      // if index has changed, must find position of currentfile before advance
      nextFileName =  this -> GoToFile(fLastServedFile);
      if ( strcmp(nextFileName.c_str(),fLastServedFile.c_str()) ) {
        MSG("DDS",Msg::kWarning) << "CS_" <<  gSystem -> GetPid()
        << ": The data source directory\n" 
        << "    no longer contains the last served file " << currentFileName
        << "\n    indicating a potential gap in the files served." 
        << " Will set index to next file beyond missing file in directory." 
        << endl;
      }
      else if ( fFileIndexItr != fFileIndex.end() ) {
        fFileIndexItr++;
        nextFileName = this->GetCurrentFileName();
      }
    }
  }            

  if ( !nextFileName.empty() ) {
    fLastServedFile = nextFileName;
  }

  return nextFileName;
  
}

std::string DDSFileHandler::GoToSymLinkFile() {
  //
  // Purpose: Move current file iterator to point to the file pointed
  //          to by the symbolic link 'currentfile' or 'offlinefile'.
  //
  // Argument: none.
  // 
  // Return: std::string  fullfilepathname of file pointed to by
  //                      symbolic link.
  //
  // Contact: S. Kasahara
  //

  std::string symLinkTarget = this -> GetSymLinkTargetName();
  if ( !symLinkTarget.empty() ) {
    // Attempt to move to it
    this -> GoToFile(symLinkTarget);
  }

  std::string nextFile = this -> GetCurrentFileName();
  if (!nextFile.empty()) {
    fLastServedFile = nextFile;
  }

  return nextFile;
  
}

bool DDSFileHandler::IsModifiedDirectory() const {
  //
  // Purpose: Determine if directory containing files has been modified.
  //
  // Argument: none.
  // 
  // Return: none.
  //
  // Contact: S. Kasahara
  //

  if ( IsValid() ) { 
  // Check last mod time of source directory and compare it to stored mod time
    struct stat keystat;
    if (stat(fLocalDataSourceDir.c_str(),&keystat) >= 0) {
      if (fSourceDirModTime != keystat.st_ctime) {
        return true;
      }
    }
  }

  return false;

}

bool DDSFileHandler::IsModifiedSymLink() const {
  //
  // Purpose: Determine if symbolic link has been modified since last
  //          call to GetSymLinkFileName().
  //
  // Argument: none.
  // 
  // Return: none.
  //
  // Contact: S. Kasahara
  //

  if ( IsValid() ) { 
  // Check last mod time of key file and compare it to stored mod time
    struct stat keystat;
    if (lstat(fSymLink.c_str(),&keystat) >= 0) {
      if (fSymLinkModTime != keystat.st_ctime) {
        return true;
      }
    }
  }

  return false;

}

bool DDSFileHandler::NewFileAvailable() {
  //
  // Purpose: Compare GetSymLinkFileName() to GetCurrentFileName() and
  //          return true if different.
  //
  // Argument: none.
  // 
  // Return: true if current file is different than symbolic link file.
  //
  // Contact: S. Kasahara
  //

  if ( IsValid() ) { 
    this -> GetSymLinkTargetName();
    if ( !fSymLinkTarget.empty() ) {
      if ( strcmp( (this->GetSymLinkTargetName()).c_str(),
                   (this->GetCurrentFileName()).c_str() ) ) {
        return true;
      }
    }
  }

  return false;

}

std::ostream& DDSFileHandler::Print(std::ostream& ms) const {
  //
  // Purpose: Print DDSFileHandler status on std::ostream.
  //
  // Argument: ms  std::ostream to print on.
  //
  // Return: std::ostream reference.
  //
  // Contact: S. Kasahara
  //

  if ( IsValid() ) {
    ms << "DDSFileHandler is serving files for data source " 
       << DDS::AsString(fDataSource) << " from directory:\n  "
       << fDataSourceDir << "." << endl;
    ms << "The symbolic link:\n  " << fSymLink << "\nshould point to "
       << "the most recent " << DDS::AsString(fDataSource)        
       << " file." << endl;
    if ( !fSymLinkTarget.empty() ) {
      ms << "This symbolic link file pointed to " 
         << fSymLinkTarget << " on last check." << endl;
    }
    else {
      ms << "This symbolic link file did not point to a file when last checked." 
         << endl;
    }
    ms << "There are " << fFileIndex.size() << " entries in the file index."
       << endl;
    for (unsigned int i = 0; i < fFileIndex.size(); i++) {
      ms << "  " << i+1 << ")" << fFileIndex[i] << endl;
    } 
  }
  else {
    ms << "DDSFileHandler was unable to access the environment variable "
       << fDataSourceEnv << "." << endl;
  }

  return ms;

}

---