GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include <curl/curl.h>
43 
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48 
50 
51 // 7.18.1
52 #if LIBCURL_VERSION_NUM >= 0x071201
53 #define HAVE_CURLINFO_REDIRECT_URL
54 #endif
55 
56 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
57 
58 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
59  const char * const* papszOptions);
60 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
61  struct curl_slist* poSrcToDestroy );
62 
63 namespace cpl {
64 
65 typedef enum
66 {
67  EXIST_UNKNOWN = -1,
68  EXIST_NO,
69  EXIST_YES,
70 } ExistStatus;
71 
72 class FileProp
73 {
74  public:
75  unsigned int nGenerationAuthParameters = 0;
76  ExistStatus eExists = EXIST_UNKNOWN;
77  vsi_l_offset fileSize = 0;
78  time_t mTime = 0;
79  time_t nExpireTimestampLocal = 0;
80  CPLString osRedirectURL{};
81  bool bHasComputedFileSize = false;
82  bool bIsDirectory = false;
83  bool bS3LikeRedirect = false;
84  CPLString ETag{};
85 };
86 
87 struct CachedDirList
88 {
89  bool bGotFileList = false;
90  unsigned int nGenerationAuthParameters = 0;
91  CPLStringList oFileList{}; /* only file name without path */
92 };
93 
94 struct WriteFuncStruct
95 {
96  char* pBuffer = nullptr;
97  size_t nSize = 0;
98  bool bIsHTTP = false;
99  bool bIsInHeader = false;
100  bool bMultiRange = false;
101  vsi_l_offset nStartOffset = 0;
102  vsi_l_offset nEndOffset = 0;
103  int nHTTPCode = 0;
104  vsi_l_offset nContentLength = 0;
105  bool bFoundContentRange = false;
106  bool bError = false;
107  bool bDownloadHeaderOnly = false;
108  bool bDetectRangeDownloadingError = false;
109  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
110 
111  VSILFILE *fp = nullptr;
112  VSICurlReadCbkFunc pfnReadCbk = nullptr;
113  void *pReadCbkUserData = nullptr;
114  bool bInterrupted = false;
115 
116 #if LIBCURL_VERSION_NUM < 0x073600
117  // Workaround to ignore extra HTTP response headers from
118  // proxies in older versions of curl.
119  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
120  bool bIsProxyConnectHeader = false;
121 #endif
122 };
123 
124 struct PutData
125 {
126  const GByte* pabyData = nullptr;
127  size_t nOff = 0;
128  size_t nTotalSize = 0;
129 
130  static size_t ReadCallBackBuffer( char *buffer, size_t size,
131  size_t nitems, void *instream )
132  {
133  PutData* poThis = static_cast<PutData *>(instream);
134  const size_t nSizeMax = size * nitems;
135  const size_t nSizeToWrite =
136  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
137  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
138  poThis->nOff += nSizeToWrite;
139  return nSizeToWrite;
140  }
141 };
142 
143 /************************************************************************/
144 /* VSICurlFilesystemHandler */
145 /************************************************************************/
146 
147 class VSICurlHandle;
148 
149 class VSICurlFilesystemHandler : public VSIFilesystemHandler
150 {
151  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
152 
153  struct FilenameOffsetPair
154  {
155  std::string filename_;
156  vsi_l_offset offset_;
157 
158  FilenameOffsetPair(const std::string& filename,
159  vsi_l_offset offset) :
160  filename_(filename), offset_(offset) {}
161 
162  bool operator==(const FilenameOffsetPair& other) const
163  {
164  return filename_ == other.filename_ &&
165  offset_ == other.offset_;
166  }
167  };
168  struct FilenameOffsetPairHasher
169  {
170  std::size_t operator()(const FilenameOffsetPair& k) const
171  {
172  return std::hash<std::string>()(k.filename_) ^
173  std::hash<vsi_l_offset>()(k.offset_);
174  }
175  };
176 
177  using RegionCacheType =
178  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
179  lru11::NullLock,
180  std::unordered_map<
181  FilenameOffsetPair,
182  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
183  std::shared_ptr<std::string>>>::iterator,
184  FilenameOffsetPairHasher>>;
185 
186  std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
187  RegionCacheType* GetRegionCache();
188 
189  lru11::Cache<std::string, FileProp> oCacheFileProp;
190 
191  int nCachedFilesInDirList = 0;
192  lru11::Cache<std::string, CachedDirList> oCacheDirList;
193 
194  char** ParseHTMLFileList(const char* pszFilename,
195  int nMaxFiles,
196  char* pszData,
197  bool* pbGotFileList);
198 
199 protected:
200  CPLMutex *hMutex = nullptr;
201 
202  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
203  virtual char** GetFileList(const char *pszFilename,
204  int nMaxFiles,
205  bool* pbGotFileList);
206 
207  void RegisterEmptyDir( const CPLString& osDirname );
208 
209  bool AnalyseS3FileList( const CPLString& osBaseURL,
210  const char* pszXML,
211  CPLStringList& osFileList,
212  int nMaxFiles,
213  bool bIgnoreGlacierStorageClass,
214  bool& bIsTruncated );
215 
216  void AnalyseSwiftFileList( const CPLString& osBaseURL,
217  const CPLString& osPrefix,
218  const char* pszJson,
219  CPLStringList& osFileList,
220  int nMaxFilesThisQuery,
221  int nMaxFiles,
222  bool& bIsTruncated,
223  CPLString& osNextMarker );
224 
225  static const char* GetOptionsStatic();
226 
227  static bool IsAllowedFilename( const char* pszFilename );
228 
229 public:
230  VSICurlFilesystemHandler();
231  ~VSICurlFilesystemHandler() override;
232 
233  VSIVirtualHandle *Open( const char *pszFilename,
234  const char *pszAccess,
235  bool bSetError ) override;
236 
237  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
238  int nFlags ) override;
239  int Unlink( const char *pszFilename ) override;
240  int Rename( const char *oldpath, const char *newpath ) override;
241  int Mkdir( const char *pszDirname, long nMode ) override;
242  int Rmdir( const char *pszDirname ) override;
243  char **ReadDir( const char *pszDirname ) override
244  { return ReadDirEx(pszDirname, 0); }
245  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
246  char **SiblingFiles( const char *pszFilename ) override;
247 
248  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
249  override { return true; }
250 
251  const char* GetActualURL(const char* pszFilename) override;
252 
253  const char* GetOptions() override;
254 
255  char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
256  CSLConstList papszOptions ) override;
257 
258  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
259  bool* pbGotFileList );
260  void InvalidateDirContent( const char *pszDirname );
261 
262  virtual CPLString GetFSPrefix() { return "/vsicurl/"; }
263  virtual bool AllowCachedDataFor(const char* pszFilename);
264 
265  std::shared_ptr<std::string> GetRegion( const char* pszURL,
266  vsi_l_offset nFileOffsetStart );
267 
268  void AddRegion( const char* pszURL,
269  vsi_l_offset nFileOffsetStart,
270  size_t nSize,
271  const char *pData );
272 
273  bool GetCachedFileProp( const char* pszURL,
274  FileProp& oFileProp );
275  void SetCachedFileProp( const char* pszURL,
276  FileProp& oFileProp );
277  void InvalidateCachedData( const char* pszURL );
278 
279  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
280 
281  virtual void ClearCache();
282  virtual void PartialClearCache(const char* pszFilename);
283 
284 
285  bool GetCachedDirList( const char* pszURL,
286  CachedDirList& oCachedDirList );
287  void SetCachedDirList( const char* pszURL,
288  CachedDirList& oCachedDirList );
289  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
290 
291  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
292 };
293 
294 /************************************************************************/
295 /* VSICurlHandle */
296 /************************************************************************/
297 
298 class VSICurlHandle : public VSIVirtualHandle
299 {
300  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
301 
302  protected:
303  VSICurlFilesystemHandler* poFS = nullptr;
304 
305  bool m_bCached = true;
306 
307  FileProp oFileProp{};
308 
309  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
310  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
311 
312  char **m_papszHTTPOptions = nullptr;
313 
314  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
315  int nBlocksToDownload = 1;
316 
317  bool bStopOnInterruptUntilUninstall = false;
318  bool bInterrupted = false;
319  VSICurlReadCbkFunc pfnReadCbk = nullptr;
320  void *pReadCbkUserData = nullptr;
321 
322  int m_nMaxRetry = 0;
323  double m_dfRetryDelay = 0.0;
324 
325  CPLStringList m_aosHeaders{};
326 
327  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
328  const int nBlocks,
329  const char* pBuffer,
330  size_t nSize );
331 
332  private:
333 
334  vsi_l_offset curOffset = 0;
335 
336  bool bEOF = false;
337 
338  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
339 
340  bool m_bUseHead = false;
341 
342  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
343  const vsi_l_offset* panOffsets,
344  const size_t* panSizes );
345  CPLString GetRedirectURLIfValid(bool& bHasExpired);
346 
347  protected:
348  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
349  const struct curl_slist* /* psExistingHeaders */)
350  { return nullptr; }
351  virtual bool AllowAutomaticRedirection() { return true; }
352  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
353  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
354  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
355  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
356  void SetURL(const char* pszURL);
357  virtual bool Authenticate() { return false; }
358 
359  public:
360 
361  VSICurlHandle( VSICurlFilesystemHandler* poFS,
362  const char* pszFilename,
363  const char* pszURLIn = nullptr );
364  ~VSICurlHandle() override;
365 
366  int Seek( vsi_l_offset nOffset, int nWhence ) override;
367  vsi_l_offset Tell() override;
368  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
369  int ReadMultiRange( int nRanges, void ** ppData,
370  const vsi_l_offset* panOffsets,
371  const size_t* panSizes ) override;
372  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
373  int Eof() override;
374  int Flush() override;
375  int Close() override;
376 
377  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
378  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
379  virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
380  bool Exists( bool bSetError );
381  bool IsDirectory() const { return oFileProp.bIsDirectory; }
382  time_t GetMTime() const { return oFileProp.mTime; }
383  const CPLStringList& GetHeaders() { return m_aosHeaders; }
384 
385  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
386  void* pfnUserData,
387  int bStopOnInterruptUntilUninstall );
388  int UninstallReadCbk();
389 
390  const char *GetURL() const { return m_pszURL; }
391 };
392 
393 /************************************************************************/
394 /* IVSIS3LikeFSHandler */
395 /************************************************************************/
396 
397 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
398 {
399  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
400 
401  bool CopyFile(VSILFILE* fpIn,
402  vsi_l_offset nSourceSize,
403  const char* pszSource,
404  const char* pszTarget,
405  GDALProgressFunc pProgressFunc,
406  void *pProgressData);
407  virtual int MkdirInternal( const char *pszDirname, bool bDoStatCheck );
408 
409  protected:
410  char** GetFileList( const char *pszFilename,
411  int nMaxFiles,
412  bool* pbGotFileList ) override;
413 
414  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
415  const char* pszURI, bool bAllowNoObject) = 0;
416 
417  virtual int CopyObject( const char *oldpath, const char *newpath,
418  CSLConstList papszMetadata );
419 
420  IVSIS3LikeFSHandler() = default;
421 
422  public:
423  int Unlink( const char *pszFilename ) override;
424  int Mkdir( const char *pszDirname, long nMode ) override;
425  int Rmdir( const char *pszDirname ) override;
426  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
427  int nFlags ) override;
428  int Rename( const char *oldpath, const char *newpath ) override;
429 
430  virtual int DeleteObject( const char *pszFilename );
431 
432  virtual const char* GetDebugKey() const = 0;
433 
434  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
435  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
436 
437  bool Sync( const char* pszSource, const char* pszTarget,
438  const char* const * papszOptions,
439  GDALProgressFunc pProgressFunc,
440  void *pProgressData,
441  char*** ppapszOutputs ) override;
442 
443  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
444  const char* const *papszOptions) override;
445 
446  // Multipart upload
447  virtual CPLString InitiateMultipartUpload(
448  const std::string& osFilename,
449  IVSIS3LikeHandleHelper *poS3HandleHelper,
450  int nMaxRetry,
451  double dfRetryDelay);
452  virtual CPLString UploadPart(const CPLString& osFilename,
453  int nPartNumber,
454  const std::string& osUploadID,
455  const void* pabyBuffer,
456  size_t nBufferSize,
457  IVSIS3LikeHandleHelper *poS3HandleHelper,
458  int nMaxRetry,
459  double dfRetryDelay);
460  virtual bool CompleteMultipart(const CPLString& osFilename,
461  const CPLString& osUploadID,
462  const std::vector<CPLString>& aosEtags,
463  IVSIS3LikeHandleHelper *poS3HandleHelper,
464  int nMaxRetry,
465  double dfRetryDelay);
466  virtual bool AbortMultipart(const CPLString& osFilename,
467  const CPLString& osUploadID,
468  IVSIS3LikeHandleHelper *poS3HandleHelper,
469  int nMaxRetry,
470  double dfRetryDelay);
471 };
472 
473 /************************************************************************/
474 /* IVSIS3LikeHandle */
475 /************************************************************************/
476 
477 class IVSIS3LikeHandle: public VSICurlHandle
478 {
479  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
480 
481  protected:
482  bool UseLimitRangeGetInsteadOfHead() override { return true; }
483  bool IsDirectoryFromExists( const char* pszVerb,
484  int response_code ) override
485  {
486  // A bit dirty, but on S3, a GET on a existing directory returns a 416
487  return response_code == 416 && EQUAL(pszVerb, "GET") &&
488  CPLString(m_pszURL).back() == '/';
489  }
490  void ProcessGetFileSizeResult( const char* pszContent ) override
491  {
492  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
493  }
494 
495  public:
496  IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
497  const char* pszFilename,
498  const char* pszURLIn ) :
499  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
500  ~IVSIS3LikeHandle() override {}
501 };
502 
503 /************************************************************************/
504 /* VSIS3WriteHandle */
505 /************************************************************************/
506 
507 class VSIS3WriteHandle final : public VSIVirtualHandle
508 {
509  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
510 
511  IVSIS3LikeFSHandler *m_poFS = nullptr;
512  CPLString m_osFilename{};
513  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
514  bool m_bUseChunked = false;
515 
516  vsi_l_offset m_nCurOffset = 0;
517  int m_nBufferOff = 0;
518  int m_nBufferSize = 0;
519  bool m_bClosed = false;
520  GByte *m_pabyBuffer = nullptr;
521  CPLString m_osUploadID{};
522  int m_nPartNumber = 0;
523  std::vector<CPLString> m_aosEtags{};
524  bool m_bError = false;
525 
526  CURLM *m_hCurlMulti = nullptr;
527  CURL *m_hCurl = nullptr;
528  const void *m_pBuffer = nullptr;
529  CPLString m_osCurlErrBuf{};
530  size_t m_nChunkedBufferOff = 0;
531  size_t m_nChunkedBufferSize = 0;
532  size_t m_nWrittenInPUT = 0;
533 
534  int m_nMaxRetry = 0;
535  double m_dfRetryDelay = 0.0;
536  WriteFuncStruct m_sWriteFuncHeaderData{};
537 
538  bool UploadPart();
539  bool DoSinglePartPUT();
540 
541  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
542  size_t nitems, void *instream );
543  size_t WriteChunked( const void *pBuffer,
544  size_t nSize, size_t nMemb );
545  int FinishChunkedTransfer();
546 
547  void InvalidateParentDirectory();
548 
549  public:
550  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
551  const char* pszFilename,
552  IVSIS3LikeHandleHelper* poS3HandleHelper,
553  bool bUseChunked );
554  ~VSIS3WriteHandle() override;
555 
556  int Seek( vsi_l_offset nOffset, int nWhence ) override;
557  vsi_l_offset Tell() override;
558  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
559  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
560  int Eof() override;
561  int Close() override;
562 
563  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
564 };
565 
566 /************************************************************************/
567 /* VSIAppendWriteHandle */
568 /************************************************************************/
569 
570 class VSIAppendWriteHandle : public VSIVirtualHandle
571 {
572  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
573 
574  protected:
575 
576  VSICurlFilesystemHandler* m_poFS = nullptr;
577  CPLString m_osFSPrefix{};
578  CPLString m_osFilename{};
579 
580  vsi_l_offset m_nCurOffset = 0;
581  int m_nBufferOff = 0;
582  int m_nBufferSize = 0;
583  int m_nBufferOffReadCallback = 0;
584  bool m_bClosed = false;
585  GByte *m_pabyBuffer = nullptr;
586  bool m_bError = false;
587 
588  static size_t ReadCallBackBuffer( char *buffer, size_t size,
589  size_t nitems, void *instream );
590  virtual bool Send(bool bIsLastBlock) = 0;
591 
592  public:
593  VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
594  const char* pszFSPrefix,
595  const char* pszFilename,
596  int nChunkSize );
597  virtual ~VSIAppendWriteHandle();
598 
599  int Seek( vsi_l_offset nOffset, int nWhence ) override;
600  vsi_l_offset Tell() override;
601  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
602  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
603  int Eof() override;
604  int Close() override;
605 
606  bool IsOK() { return m_pabyBuffer != nullptr; }
607 };
608 
609 /************************************************************************/
610 /* CurlRequestHelper */
611 /************************************************************************/
612 
613 struct CurlRequestHelper
614 {
615  WriteFuncStruct sWriteFuncData{};
616  WriteFuncStruct sWriteFuncHeaderData{};
617  char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
618 
619  CurlRequestHelper();
620  ~CurlRequestHelper();
621  long perform(CURL* hCurlHandle,
622  struct curl_slist* headers, // ownership transferred
623  VSICurlFilesystemHandler *poFS,
624  IVSIS3LikeHandleHelper *poS3HandleHelper);
625 };
626 
627 /************************************************************************/
628 /* NetworkStatisticsLogger */
629 /************************************************************************/
630 
631 class NetworkStatisticsLogger
632 {
633  static int gnEnabled;
634  static NetworkStatisticsLogger gInstance;
635 
636  NetworkStatisticsLogger() = default;
637 
638  std::mutex m_mutex{};
639 
640  struct Counters
641  {
642  GIntBig nHEAD = 0;
643  GIntBig nGET = 0;
644  GIntBig nPUT = 0;
645  GIntBig nPOST = 0;
646  GIntBig nDELETE = 0;
647  GIntBig nGETDownloadedBytes = 0;
648  GIntBig nPUTUploadedBytes = 0;
649  GIntBig nPOSTDownloadedBytes = 0;
650  GIntBig nPOSTUploadedBytes = 0;
651  };
652 
653  enum class ContextPathType
654  {
655  FILESYSTEM,
656  FILE,
657  ACTION,
658  };
659 
660  struct ContextPathItem
661  {
662  ContextPathType eType;
663  CPLString osName;
664 
665  ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
666  eType(eTypeIn), osName(osNameIn) {}
667 
668  bool operator< (const ContextPathItem& other ) const
669  {
670  if( static_cast<int>(eType) < static_cast<int>(other.eType) )
671  return true;
672  if( static_cast<int>(eType) > static_cast<int>(other.eType) )
673  return false;
674  return osName < other.osName;
675  }
676  };
677 
678  struct Stats
679  {
680  Counters counters{};
681  std::map<ContextPathItem, Stats> children{};
682 
683  void AsJSON(CPLJSONObject& oJSON) const;
684  };
685 
686  Stats m_stats{};
687  std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
688 
689  static void ReadEnabled();
690 
691  std::vector<Counters*> GetCountersForContext();
692 
693 public:
694 
695  static inline bool IsEnabled()
696  {
697  if( gnEnabled < 0)
698  {
699  ReadEnabled();
700  }
701  return gnEnabled == TRUE;
702  }
703 
704  static void EnterFileSystem(const char* pszName);
705 
706  static void LeaveFileSystem();
707 
708  static void EnterFile(const char* pszName);
709 
710  static void LeaveFile();
711 
712  static void EnterAction(const char* pszName);
713 
714  static void LeaveAction();
715 
716  static void LogHEAD();
717 
718  static void LogGET(size_t nDownloadedBytes);
719 
720  static void LogPUT(size_t nUploadedBytes);
721 
722  static void LogPOST(size_t nUploadedBytes,
723  size_t nDownloadedBytes);
724 
725  static void LogDELETE();
726 
727  static void Reset();
728 
729  static CPLString GetReportAsSerializedJSON();
730 };
731 
732 struct NetworkStatisticsFileSystem
733 {
734  inline explicit NetworkStatisticsFileSystem(const char* pszName) {
735  NetworkStatisticsLogger::EnterFileSystem(pszName);
736  }
737 
738  inline ~NetworkStatisticsFileSystem()
739  {
740  NetworkStatisticsLogger::LeaveFileSystem();
741  }
742 };
743 
744 struct NetworkStatisticsFile
745 {
746  inline explicit NetworkStatisticsFile(const char* pszName) {
747  NetworkStatisticsLogger::EnterFile(pszName);
748  }
749 
750  inline ~NetworkStatisticsFile()
751  {
752  NetworkStatisticsLogger::LeaveFile();
753  }
754 };
755 
756 struct NetworkStatisticsAction
757 {
758  inline explicit NetworkStatisticsAction(const char* pszName) {
759  NetworkStatisticsLogger::EnterAction(pszName);
760  }
761 
762  inline ~NetworkStatisticsAction()
763  {
764  NetworkStatisticsLogger::LeaveAction();
765  }
766 };
767 
768 
769 int VSICURLGetDownloadChunkSize();
770 
771 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
772  VSILFILE *fp,
773  VSICurlReadCbkFunc pfnReadCbk,
774  void *pReadCbkUserData );
775 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
776  size_t nmemb, void *req );
777 void MultiPerform(CURLM* hCurlMultiHandle,
778  CURL* hEasyHandle = nullptr);
779 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
780 
781 } // namespace cpl
782 
784 
785 #endif // HAVE_CURL
786 
787 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:576
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1007
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1216
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:318
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:194
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140