xrootd
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
XrdClZipArchive.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN)
3 // Author: Michal Simon <michal.simon@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #ifndef SRC_XRDZIP_XRDZIPARCHIVE_HH_
26 #define SRC_XRDZIP_XRDZIPARCHIVE_HH_
27 
28 #include "XrdCl/XrdClFile.hh"
30 #include "XrdCl/XrdClJobManager.hh"
31 #include "XrdCl/XrdClDefaultEnv.hh"
32 #include "XrdCl/XrdClPostMaster.hh"
33 #include "XrdZip/XrdZipEOCD.hh"
34 #include "XrdZip/XrdZipCDFH.hh"
36 #include "XrdZip/XrdZipLFH.hh"
37 #include "XrdCl/XrdClZipCache.hh"
38 
39 #include <memory>
40 #include <unordered_map>
41 
42 //-----------------------------------------------------------------------------
43 // Forward declaration needed for friendship
44 //-----------------------------------------------------------------------------
45 namespace XrdEc{ class StrmWriter; class Reader; template<bool> class OpenOnlyImpl; };
46 class MicroTest;
47 
48 namespace XrdCl
49 {
50  using namespace XrdZip;
51 
52  //---------------------------------------------------------------------------
53  // ZipArchive provides following functionalities:
54  // - parsing of existing ZIP archive
55  // - reading data from existing ZIP archive
56  // - appending data to existing ZIP archive
57  // - querying stat info and checksum for given file in ZIP archive
58  //---------------------------------------------------------------------------
59  class ZipArchive
60  {
61  friend class XrdEc::StrmWriter;
62  friend class XrdEc::Reader;
63  template<bool>
64  friend class XrdEc::OpenOnlyImpl;
65  friend class ::MicroTest;
66 
67  template<typename RSP>
68  friend XRootDStatus ReadFromImpl( ZipArchive&, const std::string&, uint64_t, uint32_t, void*, ResponseHandler*, uint16_t );
69 
70  public:
71  //-----------------------------------------------------------------------
73  //-----------------------------------------------------------------------
74  ZipArchive( bool enablePlugIns = true );
75 
76  //-----------------------------------------------------------------------
78  //-----------------------------------------------------------------------
79  virtual ~ZipArchive();
80 
81  //-----------------------------------------------------------------------
89  //-----------------------------------------------------------------------
90  XRootDStatus OpenArchive( const std::string &url,
91  OpenFlags::Flags flags,
92  ResponseHandler *handler,
93  uint16_t timeout = 0 );
94 
95  //-----------------------------------------------------------------------
103  //-----------------------------------------------------------------------
104  XRootDStatus OpenFile( const std::string &fn,
106  uint64_t size = 0,
107  uint32_t crc32 = 0 );
108 
109  //-----------------------------------------------------------------------
118  //-----------------------------------------------------------------------
119  inline
120  XRootDStatus Read( uint64_t offset,
121  uint32_t size,
122  void *buffer,
123  ResponseHandler *handler,
124  uint16_t timeout = 0 )
125  {
126  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
127  return ReadFrom( openfn, offset, size, buffer, handler, timeout );
128  }
129 
130  //-----------------------------------------------------------------------
139  //-----------------------------------------------------------------------
140  inline
141  XRootDStatus PgRead( uint64_t offset,
142  uint32_t size,
143  void *buffer,
144  ResponseHandler *handler,
145  uint16_t timeout = 0 )
146  {
147  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
148  return PgReadFrom( openfn, offset, size, buffer, handler, timeout );
149  }
150 
151  //-----------------------------------------------------------------------
161  //-----------------------------------------------------------------------
162  XRootDStatus ReadFrom( const std::string &fn,
163  uint64_t offset,
164  uint32_t size,
165  void *buffer,
166  ResponseHandler *handler,
167  uint16_t timeout = 0 );
168 
169  //-----------------------------------------------------------------------
179  //-----------------------------------------------------------------------
180  XRootDStatus PgReadFrom( const std::string &fn,
181  uint64_t offset,
182  uint32_t size,
183  void *buffer,
184  ResponseHandler *handler,
185  uint16_t timeout = 0 );
186 
187  //-----------------------------------------------------------------------
195  //-----------------------------------------------------------------------
196  inline XRootDStatus Write( uint32_t size,
197  const void *buffer,
198  ResponseHandler *handler,
199  uint16_t timeout = 0 )
200  {
201  if( openstage != Done || openfn.empty() )
202  return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
203 
204  return WriteImpl( size, buffer, handler, timeout );
205  }
206 
207  //-----------------------------------------------------------------------
212  //-----------------------------------------------------------------------
213  XRootDStatus UpdateMetadata( uint32_t crc32 );
214 
215  //-----------------------------------------------------------------------
225  //-----------------------------------------------------------------------
226  XRootDStatus AppendFile( const std::string &fn,
227  uint32_t crc32,
228  uint32_t size,
229  const void *buffer,
230  ResponseHandler *handler,
231  uint16_t timeout = 0 );
232 
233  //-----------------------------------------------------------------------
239  //-----------------------------------------------------------------------
240  inline XRootDStatus Stat( const std::string &fn, StatInfo *&info )
241  { // make sure archive has been opened and CD has been parsed
242  if( openstage != Done )
243  return XRootDStatus( stError, errInvalidOp );
244  // make sure the file is part of the archive
245  auto cditr = cdmap.find( fn );
246  if( cditr == cdmap.end() )
247  return XRootDStatus( stError, errNotFound );
248  // create the result
249  info = make_stat( fn );
250  if (info)
251  return XRootDStatus();
252  else // have difficult to access the openned archive.
253  return XRootDStatus( stError, errNotFound );
254  }
255 
256  //-----------------------------------------------------------------------
261  //-----------------------------------------------------------------------
262  inline XRootDStatus Stat( StatInfo *&info )
263  {
264  if( openfn.empty() )
265  return XRootDStatus( stError, errInvalidOp );
266  return Stat( openfn, info );
267  }
268 
269  //-----------------------------------------------------------------------
275  //-----------------------------------------------------------------------
276  inline XRootDStatus GetCRC32( const std::string &fn, uint32_t &cksum )
277  { // make sure archive has been opened and CD has been parsed
278  if( openstage != Done )
279  return XRootDStatus( stError, errInvalidOp );
280  // make sure the file is part of the archive
281  auto cditr = cdmap.find( fn );
282  if( cditr == cdmap.end() )
283  return XRootDStatus( stError, errNotFound );
284  cksum = cdvec[cditr->second]->ZCRC32;
285  return XRootDStatus();
286  }
287 
288  //-----------------------------------------------------------------------
290  //
294  //-----------------------------------------------------------------------
296  uint16_t timeout = 0 );
297 
298  //-----------------------------------------------------------------------
301  //-----------------------------------------------------------------------
303  {
304  if( openstage != Done || openfn.empty() )
306  0, "Archive not opened." );
307  openfn.clear();
308  lfh.reset();
309  return XRootDStatus();
310  }
311 
312  //-----------------------------------------------------------------------
315  //-----------------------------------------------------------------------
316  XRootDStatus List( DirectoryList *&list );
317 
318  //-----------------------------------------------------------------------
320  //-----------------------------------------------------------------------
321  inline bool IsOpen()
322  {
323  return openstage == Done;
324  }
325 
326  //------------------------------------------------------------------------
328  //------------------------------------------------------------------------
329  inline bool IsSecure()
330  {
331  return archive.IsSecure();
332  }
333 
334  //-----------------------------------------------------------------------
336  //-----------------------------------------------------------------------
337  inline bool SetProperty( const std::string &name, const std::string &value )
338  {
339  return archive.SetProperty( name, value );
340  }
341 
342  //-----------------------------------------------------------------------
344  //-----------------------------------------------------------------------
345  inline bool GetProperty( const std::string &name, std::string &value )
346  {
347  return archive.GetProperty( name, value );
348  }
349 
350  //-----------------------------------------------------------------------
352  //-----------------------------------------------------------------------
353  inline File& GetFile()
354  {
355  return archive;
356  }
357 
358  private:
359 
360  //-----------------------------------------------------------------------
369  //-----------------------------------------------------------------------
370  XRootDStatus WriteImpl( uint32_t size,
371  const void *buffer,
372  ResponseHandler *handler,
373  uint16_t timeout );
374 
375  //-----------------------------------------------------------------------
383  //-----------------------------------------------------------------------
384  XRootDStatus OpenOnly( const std::string &url,
385  bool update,
386  ResponseHandler *handler,
387  uint16_t timeout = 0 );
388 
389  //-----------------------------------------------------------------------
393  //-----------------------------------------------------------------------
394  buffer_t GetCD();
395 
396  //-----------------------------------------------------------------------
400  //-----------------------------------------------------------------------
401  void SetCD( const buffer_t &buffer );
402 
403  //-----------------------------------------------------------------------
408  //-----------------------------------------------------------------------
409  template<typename Response>
410  inline static AnyObject* PkgRsp( Response *rsp )
411  {
412  if( !rsp ) return nullptr;
413  AnyObject *pkg = new AnyObject();
414  pkg->Set( rsp );
415  return pkg;
416  }
417 
418  //-----------------------------------------------------------------------
420  //-----------------------------------------------------------------------
421  template<typename Response>
422  inline static void Free( XRootDStatus *st, Response *rsp )
423  {
424  delete st;
425  delete rsp;
426  }
427 
428  //-----------------------------------------------------------------------
435  //-----------------------------------------------------------------------
436  template<typename Response>
437  inline static void Schedule( ResponseHandler *handler, XRootDStatus *st, Response *rsp = nullptr )
438  {
439  if( !handler ) return Free( st, rsp );
440  ResponseJob *job = new ResponseJob( handler, st, PkgRsp( rsp ), 0 );
442  }
443 
444  //-----------------------------------------------------------------------
450  //-----------------------------------------------------------------------
451  inline static StatInfo* make_stat( const StatInfo &starch, uint64_t size )
452  {
453  StatInfo *info = new StatInfo( starch );
454  uint32_t flags = info->GetFlags();
455  info->SetFlags( flags & ( ~StatInfo::IsWritable ) ); // make sure it is not listed as writable
456  info->SetSize( size );
457  return info;
458  }
459 
460  //-----------------------------------------------------------------------
465  //-----------------------------------------------------------------------
466  inline StatInfo* make_stat( const std::string &fn )
467  {
468  StatInfo *infoptr = 0;
469  XRootDStatus st = archive.Stat( false, infoptr );
470  if (!st.IsOK()) return nullptr;
471  std::unique_ptr<StatInfo> stinfo( infoptr );
472  auto itr = cdmap.find( fn );
473  if( itr == cdmap.end() ) return nullptr;
474  size_t index = itr->second;
475  return make_stat( *stinfo, cdvec[index]->uncompressedSize );
476  }
477 
478  //-----------------------------------------------------------------------
480  //-----------------------------------------------------------------------
481  inline static XRootDStatus* make_status( const XRootDStatus &status = XRootDStatus() )
482  {
483  return new XRootDStatus( status );
484  }
485 
486  //-----------------------------------------------------------------------
488  //-----------------------------------------------------------------------
489  inline void Clear()
490  {
491  buffer.reset();
492  eocd.reset();
493  cdvec.clear();
494  cdmap.clear();
495  zip64eocd.reset();
496  openstage = None;
497  }
498 
499  //-----------------------------------------------------------------------
501  //-----------------------------------------------------------------------
503  {
504  None = 0, //< opening/parsing not started
505  HaveEocdBlk, //< we have the End of Central Directory record
506  HaveZip64EocdlBlk, //< we have the ZIP64 End of Central Directory locator record
507  HaveZip64EocdBlk, //< we have the ZIP64 End of Central Directory record
508  HaveCdRecords, //< we have Central Directory records
509  Done, //< we are done parsing the Central Directory
510  Error, //< opening/parsing failed
511  NotParsed //< the ZIP archive has been opened but Central Directory is not parsed
512  };
513 
514  //-----------------------------------------------------------------------
516  //-----------------------------------------------------------------------
517  struct NewFile
518  {
519  NewFile( uint64_t offset, std::unique_ptr<LFH> lfh ) : offset( offset ),
520  lfh( std::move( lfh ) ),
521  overwrt( false )
522  {
523  }
524 
525  NewFile( NewFile && nf ) : offset( nf.offset ),
526  lfh( std::move( nf.lfh ) ),
527  overwrt( nf.overwrt )
528  {
529  }
530 
531  uint64_t offset; // the offset of the LFH of the file
532  std::unique_ptr<LFH> lfh; // LFH of the file
533  bool overwrt; // if true the LFH needs to be overwritten on close
534  };
535 
536  //-----------------------------------------------------------------------
538  //-----------------------------------------------------------------------
539  typedef std::unordered_map<std::string, ZipCache> zipcache_t;
540  typedef std::unordered_map<std::string, NewFile> new_files_t;
541 
542  File archive; //> File object for handling the ZIP archive
543  uint64_t archsize; //> size of the ZIP archive
544  bool cdexists; //> true if Central Directory exists, false otherwise
545  bool updated; //> true if the ZIP archive has been updated, false otherwise
546  std::unique_ptr<char[]> buffer; //> buffer for keeping the data to be parsed or raw data
547  std::unique_ptr<EOCD> eocd; //> End of Central Directory record
548  cdvec_t cdvec; //> vector of Central Directory File Headers
549  cdmap_t cdmap; //> mapping of file name to CDFH index
550  uint64_t cdoff; //> Central Directory offset
551  uint32_t orgcdsz; //> original CD size
552  uint32_t orgcdcnt; //> original number CDFH records
553  buffer_t orgcdbuf; //> buffer with the original CDFH records
554  std::unique_ptr<ZIP64_EOCD> zip64eocd; //> ZIP64 End of Central Directory record
555  OpenStages openstage; //> stage of opening / parsing a ZIP archive
556  std::string openfn; //> file name of opened file
557  zipcache_t zipcache; //> cache for inflating compressed data
558  std::unique_ptr<LFH> lfh; //> Local File Header record for the newly appended file
559  bool ckpinit; //> a flag indicating whether a checkpoint has been initialized
560  new_files_t newfiles; //> all newly appended files
561  };
562 
563 } /* namespace XrdZip */
564 
565 #endif /* SRC_XRDZIP_XRDZIPARCHIVE_HH_ */
Write access is allowed.
Definition: XrdClXRootDResponses.hh:414
ZipReadFromImpl< false > ReadFrom(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void * > buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:302
Definition: XrdClAnyObject.hh:32
static void Free(XRootDStatus *st, Response *rsp)
Free status and response.
Definition: XrdClZipArchive.hh:422
buffer_t orgcdbuf
Definition: XrdClZipArchive.hh:553
std::vector< char > buffer_t
Definition: XrdZipUtils.hh:54
Definition: XrdClZipArchive.hh:505
cdvec_t cdvec
Definition: XrdClZipArchive.hh:548
NewFile(uint64_t offset, std::unique_ptr< LFH > lfh)
Definition: XrdClZipArchive.hh:519
Directory list.
Definition: XrdClXRootDResponses.hh:649
Object stat info.
Definition: XrdClXRootDResponses.hh:399
Call the user callback.
Definition: XrdClResponseJob.hh:30
bool ckpinit
Definition: XrdClZipArchive.hh:559
void Clear()
Clear internal ZipArchive objects.
Definition: XrdClZipArchive.hh:489
Definition: XrdClZipArchive.hh:507
LFH of a newly appended file (in case it needs to be overwritten)
Definition: XrdClZipArchive.hh:517
StatInfo * make_stat(const std::string &fn)
Definition: XrdClZipArchive.hh:466
uint64_t offset
Definition: XrdClZipArchive.hh:531
Definition: XrdClZipArchive.hh:506
bool IsSecure()
Check if the underlying file is using an encrypted connection.
Definition: XrdClZipArchive.hh:329
File archive
Definition: XrdClZipArchive.hh:542
XRootDStatus Stat(const std::string &fn, StatInfo *&info)
Definition: XrdClZipArchive.hh:240
std::unordered_map< std::string, ZipCache > zipcache_t
Type that maps file name to its cache.
Definition: XrdClZipArchive.hh:539
Definition: XrdEcStrmWriter.hh:52
XRootDStatus CloseFile()
Definition: XrdClZipArchive.hh:302
static XRootDStatus * make_status(const XRootDStatus &status=XRootDStatus())
Allocate new XRootDStatus object.
Definition: XrdClZipArchive.hh:481
void QueueJob(Job *job, void *arg=0)
Add a job to be run.
Definition: XrdClJobManager.hh:92
std::unordered_map< std::string, NewFile > new_files_t
Definition: XrdClZipArchive.hh:540
bool overwrt
Definition: XrdClZipArchive.hh:533
XRootDStatus PgRead(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:141
uint32_t GetFlags() const
Get flags.
cdmap_t cdmap
Definition: XrdClZipArchive.hh:549
A file.
Definition: XrdClFile.hh:45
XRootDStatus Write(uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:196
std::unique_ptr< ZIP64_EOCD > zip64eocd
Definition: XrdClZipArchive.hh:554
CloseArchiveImpl< false > CloseArchive(Ctx< ZipArchive > zip, uint16_t timeout=0)
Factory for creating CloseFileImpl objects.
Definition: XrdClZipOperations.hh:621
Definition: XrdClZipArchive.hh:510
zipcache_t zipcache
Definition: XrdClZipArchive.hh:557
none object for initializing empty Optional
Definition: XrdClOptional.hh:35
void Set(Type object, bool own=true)
Definition: XrdClAnyObject.hh:59
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errNotFound
Definition: XrdClStatus.hh:100
bool GetProperty(const std::string &name, std::string &value)
Get property on the underlying File object.
Definition: XrdClZipArchive.hh:345
Request status.
Definition: XrdClXRootDResponses.hh:218
StatImpl< false > Stat(Ctx< File > file, Arg< bool > force, uint16_t timeout=0)
Definition: XrdClFileOperations.hh:535
static StatInfo * make_stat(const StatInfo &starch, uint64_t size)
Definition: XrdClZipArchive.hh:451
OpenArchiveImpl< false > OpenArchive(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< OpenFlags::Flags > flags, uint16_t timeout=0)
Factory for creating OpenArchiveImpl objects.
Definition: XrdClZipOperations.hh:123
static void Schedule(ResponseHandler *handler, XRootDStatus *st, Response *rsp=nullptr)
Definition: XrdClZipArchive.hh:437
std::unique_ptr< EOCD > eocd
Definition: XrdClZipArchive.hh:547
std::unordered_map< std::string, size_t > cdmap_t
Definition: XrdZipCDFH.hh:54
uint32_t orgcdsz
Definition: XrdClZipArchive.hh:551
bool SetProperty(const std::string &name, const std::string &value)
Set property on the underlying File object.
Definition: XrdClZipArchive.hh:337
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:532
OpenStages
Stages of opening and parsing a ZIP archive.
Definition: XrdClZipArchive.hh:502
Handle an async response.
Definition: XrdClXRootDResponses.hh:1116
bool IsOpen()
Definition: XrdClZipArchive.hh:321
bool cdexists
Definition: XrdClZipArchive.hh:544
Definition: XrdEcReader.hh:57
uint32_t orgcdcnt
Definition: XrdClZipArchive.hh:552
void SetFlags(uint32_t flags)
Set flags.
Definition: XrdClZipArchive.hh:508
JobManager * GetJobManager()
Get the job manager object user by the post master.
XRootDStatus GetCRC32(const std::string &fn, uint32_t &cksum)
Definition: XrdClZipArchive.hh:276
File & GetFile()
Get the underlying File object.
Definition: XrdClZipArchive.hh:353
XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:120
Definition: XrdClZipArchive.hh:45
Definition: XrdClZipArchive.hh:59
OpenFileImpl< false > OpenFile(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< OpenFlags::Flags > flags=OpenFlags::None, Arg< uint64_t > size=0, Arg< uint32_t > crc32=0, uint16_t timeout=0)
Factory for creating OpenFileImpl objects.
Definition: XrdClZipOperations.hh:184
Nothing.
Definition: XrdClFileSystem.hh:77
NewFile(NewFile &&nf)
Definition: XrdClZipArchive.hh:525
std::vector< std::unique_ptr< CDFH > > cdvec_t
Definition: XrdZipCDFH.hh:44
std::string openfn
Definition: XrdClZipArchive.hh:556
OpenStages openstage
Definition: XrdClZipArchive.hh:555
new_files_t newfiles
Definition: XrdClZipArchive.hh:560
static PostMaster * GetPostMaster()
Get default post master.
Flags
Open flags, may be or&#39;d when appropriate.
Definition: XrdClFileSystem.hh:75
void SetSize(uint64_t size)
Set size.
bool IsOK() const
We&#39;re fine.
Definition: XrdClStatus.hh:123
XRootDStatus Stat(StatInfo *&info)
Definition: XrdClZipArchive.hh:262
Write operation (.
Definition: XrdClFileOperations.hh:544
Definition: XrdClZipArchive.hh:509
uint64_t cdoff
Definition: XrdClZipArchive.hh:550
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:558
const uint16_t errInvalidOp
Definition: XrdClStatus.hh:51
std::unique_ptr< char[]> buffer
Definition: XrdClZipArchive.hh:546
uint64_t archsize
Definition: XrdClZipArchive.hh:543
AppendFileImpl< false > AppendFile(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint32_t > crc32, Arg< uint32_t > size, Arg< const void * > buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:421
bool updated
Definition: XrdClZipArchive.hh:545
ZipListImpl< false > List(Ctx< ZipArchive > zip)
Factory for creating ZipStatImpl objects.
Definition: XrdClZipOperations.hh:574
static AnyObject * PkgRsp(Response *rsp)
Definition: XrdClZipArchive.hh:410